From ae1da26558d9c72f097498205b0e2ec23790bfb1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 22 Jul 2026 13:35:54 +0000 Subject: [PATCH] =?UTF-8?q?Move=20the=20PasHub=20fetcher=20into=20the=20DD?= =?UTF-8?q?D=20layer=20structure=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The service was the last one living wholly under backend/. It now follows the same layering as abri and the other newer services: domain/pashub_fetcher/ core file classification, subfolders infrastructure/pashub_fetcher/ PasHub client, token getter, wire DTOs orchestration/ PashubFetcherOrchestrator (was PashubService) applications/pashub_fetcher/ lambda handler, trigger request, dev tooling core_files.py is split along the layer boundary: the domain module keeps the filename/evidence-category classification rules and no longer imports infrastructure.postgres, while the CoreFiles -> FileTypeEnum translation moves to infrastructure/pashub_fetcher/core_file_types.py. Tests move into the tests/ tree by layer. Note this puts them in the only suite CI currently runs (unit_tests.yml is disabled), so these 73 tests now execute on PRs for the first time; they were previously reachable only via the legacy pytest.ini testpaths. sharepoint_renamer's image now copies just domain/pashub_fetcher/ rather than the whole service, since SharepointSubfolders is all it needed. Behaviour is unchanged. tests/ goes 9927 -> 10000 passed (+73, exactly the tests that moved in); the legacy suite keeps its same 17 pre-existing failures and 11 errors. tests/test_lambda_packaging.py confirms both changed Dockerfiles still copy their handler's full import closure. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy_terraform.yml | 2 +- .github/workflows/lambda_smoke_tests.yml | 2 +- ..._Coordination_Design_Board_1774881298.xlsx | Bin ...tford_Warm_Homes_Wave_3_RA Downloads .xlsx | Bin applications/pashub_fetcher/__init__.py | 0 .../pashub_fetcher}/handler.py | 10 +- .../pashub_fetcher/handler/Dockerfile | 12 +- .../pashub_fetcher/handler/requirements.txt | 0 .../pashub_fetcher/handler/test_handler.py | 0 .../local_handler/docker-compose.yml | 4 +- .../local_handler/invoke_local_lambda.py | 0 .../pashub_to_ara_trigger_request.py | 0 .../trigger_local_lambda_from_file.py | 4 +- .../trigger_pashub_sqs_from_file.py | 2 +- .../sharepoint_renamer/handler/Dockerfile | 3 +- .../HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md | 5 +- domain/pashub_fetcher/__init__.py | 0 .../pashub_fetcher/core_files.py | 30 ----- .../pashub_fetcher/evidence_categories.py | 0 {backend => domain}/pashub_fetcher/job.py | 0 .../pashub_fetcher/sharepoint_subfolders.py | 0 infrastructure/pashub_fetcher/__init__.py | 0 .../pashub_fetcher/core_file_types.py | 39 +++++++ .../pashub_fetcher/evidence_file_data.py | 0 .../pashub_fetcher/evidence_metadata.py | 0 .../pashub_fetcher/pashub_client.py | 6 +- .../pashub_fetcher/spreadsheet.py | 2 +- .../pashub_fetcher/token_getter.py | 0 .../pashub_fetcher_orchestrator.py | 0 .../sharepoint_renamer_orchestrator.py | 2 +- pytest.ini | 1 - tests/applications/pashub_fetcher/__init__.py | 0 .../test_pashub_to_ara_trigger_request.py | 2 +- tests/domain/pashub_fetcher/__init__.py | 0 .../domain/pashub_fetcher}/test_core_files.py | 109 +----------------- .../infrastructure/pashub_fetcher/__init__.py | 0 .../pashub_fetcher/test_core_file_types.py | 98 ++++++++++++++++ .../pashub_fetcher}/test_pashub_client.py | 8 +- .../pashub_fetcher}/test_token_getter.py | 8 +- .../test_pashub_fetcher_orchestrator.py | 0 tests/scripts/test_rename_sharepoint_files.py | 2 +- 41 files changed, 178 insertions(+), 173 deletions(-) rename {backend => applications}/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx (100%) rename {backend => applications}/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx (100%) create mode 100644 applications/pashub_fetcher/__init__.py rename {backend/pashub_fetcher/handler => applications/pashub_fetcher}/handler.py (88%) rename {backend => applications}/pashub_fetcher/handler/Dockerfile (69%) rename {backend => applications}/pashub_fetcher/handler/requirements.txt (100%) rename {backend => applications}/pashub_fetcher/handler/test_handler.py (100%) rename {backend => applications}/pashub_fetcher/local_handler/docker-compose.yml (65%) rename {backend => applications}/pashub_fetcher/local_handler/invoke_local_lambda.py (100%) rename {backend => applications}/pashub_fetcher/pashub_to_ara_trigger_request.py (100%) rename {backend => applications}/pashub_fetcher/trigger_local_lambda_from_file.py (93%) rename {backend => applications}/pashub_fetcher/trigger_pashub_sqs_from_file.py (98%) create mode 100644 domain/pashub_fetcher/__init__.py rename {backend => domain}/pashub_fetcher/core_files.py (60%) rename {backend => domain}/pashub_fetcher/evidence_categories.py (100%) rename {backend => domain}/pashub_fetcher/job.py (100%) rename {backend => domain}/pashub_fetcher/sharepoint_subfolders.py (100%) create mode 100644 infrastructure/pashub_fetcher/__init__.py create mode 100644 infrastructure/pashub_fetcher/core_file_types.py rename {backend => infrastructure}/pashub_fetcher/evidence_file_data.py (100%) rename {backend => infrastructure}/pashub_fetcher/evidence_metadata.py (100%) rename {backend => infrastructure}/pashub_fetcher/pashub_client.py (97%) rename {backend => infrastructure}/pashub_fetcher/spreadsheet.py (95%) rename {backend => infrastructure}/pashub_fetcher/token_getter.py (100%) rename backend/pashub_fetcher/pashub_service.py => orchestration/pashub_fetcher_orchestrator.py (100%) create mode 100644 tests/applications/pashub_fetcher/__init__.py rename {backend/pashub_fetcher/tests => tests/applications/pashub_fetcher}/test_pashub_to_ara_trigger_request.py (94%) create mode 100644 tests/domain/pashub_fetcher/__init__.py rename {backend/pashub_fetcher/tests => tests/domain/pashub_fetcher}/test_core_files.py (51%) create mode 100644 tests/infrastructure/pashub_fetcher/__init__.py create mode 100644 tests/infrastructure/pashub_fetcher/test_core_file_types.py rename {backend/pashub_fetcher/tests => tests/infrastructure/pashub_fetcher}/test_pashub_client.py (97%) rename {backend/pashub_fetcher/tests => tests/infrastructure/pashub_fetcher}/test_token_getter.py (84%) rename backend/pashub_fetcher/tests/test_pashub_service.py => tests/orchestration/test_pashub_fetcher_orchestrator.py (100%) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 87bb264a3..48158e4b3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -453,7 +453,7 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/pashub_fetcher/handler/Dockerfile + dockerfile_path: applications/pashub_fetcher/handler/Dockerfile build_context: . secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index d958abcb9..d408c46d8 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -109,7 +109,7 @@ # pashub_smoke_test: # uses: ./.github/workflows/_smoke_test_lambda.yml # with: -# dockerfile_path: backend/pashub_fetcher/handler/Dockerfile +# dockerfile_path: applications/pashub_fetcher/handler/Dockerfile # build_context: . # service_name: pashub diff --git a/backend/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx b/applications/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx similarity index 100% rename from backend/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx rename to applications/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx diff --git a/backend/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx b/applications/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx similarity index 100% rename from backend/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx rename to applications/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx diff --git a/applications/pashub_fetcher/__init__.py b/applications/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/handler/handler.py b/applications/pashub_fetcher/handler.py similarity index 88% rename from backend/pashub_fetcher/handler/handler.py rename to applications/pashub_fetcher/handler.py index 00f0ddea9..967945436 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/applications/pashub_fetcher/handler.py @@ -1,12 +1,12 @@ from typing import Any, Callable, Dict, List, Optional from backend.app.config import get_settings -from backend.pashub_fetcher.pashub_client import PashubClient -from backend.pashub_fetcher.pashub_service import PashubService -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from infrastructure.pashub_fetcher.pashub_client import PashubClient +from orchestration.pashub_fetcher_orchestrator import PashubFetcherOrchestrator +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) -from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage from backend.app.db.models.tasks import SourceEnum from backend.utils.subtasks import task_handler from utils.logger import setup_logger @@ -63,7 +63,7 @@ def handler(body: Dict[str, Any], context: Any) -> List[str]: f"Unrecognised sharepoint_site '{payload.sharepoint_site}'; skipping SharePoint upload" ) - service = PashubService( + service = PashubFetcherOrchestrator( pashub_client=get_pashub_client(pashub_email, pashub_password), sharepoint_client=sharepoint_client, s3_bucket=S3_BUCKET, diff --git a/backend/pashub_fetcher/handler/Dockerfile b/applications/pashub_fetcher/handler/Dockerfile similarity index 69% rename from backend/pashub_fetcher/handler/Dockerfile rename to applications/pashub_fetcher/handler/Dockerfile index d20d00d0c..b3c79a1f0 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/applications/pashub_fetcher/handler/Dockerfile @@ -1,6 +1,6 @@ # jammy (Ubuntu 22.04) ships Python 3.10, which lacks enum.StrEnum — used by # domain/modelling/measure_type.py, which the handler pulls in transitively -# (handler -> pashub_service -> documents_parser.parser -> ... -> domain). +# (handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> ... -> domain). # noble (Ubuntu 24.04) ships Python 3.12, matching the project's 3.11+ standard. FROM mcr.microsoft.com/playwright/python:v1.58.0-noble @@ -14,14 +14,18 @@ WORKDIR /var/task COPY utils/ utils/ COPY backend/ backend/ COPY datatypes/ datatypes/ -# handler -> pashub_service -> documents_parser.parser -> datatypes.epc.domain.mapper +# handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> datatypes.epc.domain.mapper # -> domain.sap10_calculator, and -> db_writer -> epc_property model # -> infrastructure.postgres. Without these the lambda fails at init with # "No module named 'domain'" / "'infrastructure'". COPY domain/ domain/ COPY infrastructure/ infrastructure/ +# The service now lives across the DDD layers: the handler is in applications/, +# PashubFetcherOrchestrator in orchestration/, the client in infrastructure/. +COPY applications/ applications/ +COPY orchestration/ orchestration/ -COPY backend/pashub_fetcher/handler/requirements.txt . +COPY applications/pashub_fetcher/handler/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Local lambda entrypoint @@ -33,4 +37,4 @@ ENTRYPOINT ["python", "-m", "awslambdaric"] # ----------------------------- # Lambda handler # ----------------------------- -CMD ["backend.pashub_fetcher.handler.handler.handler"] \ No newline at end of file +CMD ["applications.pashub_fetcher.handler.handler"] \ No newline at end of file diff --git a/backend/pashub_fetcher/handler/requirements.txt b/applications/pashub_fetcher/handler/requirements.txt similarity index 100% rename from backend/pashub_fetcher/handler/requirements.txt rename to applications/pashub_fetcher/handler/requirements.txt diff --git a/backend/pashub_fetcher/handler/test_handler.py b/applications/pashub_fetcher/handler/test_handler.py similarity index 100% rename from backend/pashub_fetcher/handler/test_handler.py rename to applications/pashub_fetcher/handler/test_handler.py diff --git a/backend/pashub_fetcher/local_handler/docker-compose.yml b/applications/pashub_fetcher/local_handler/docker-compose.yml similarity index 65% rename from backend/pashub_fetcher/local_handler/docker-compose.yml rename to applications/pashub_fetcher/local_handler/docker-compose.yml index 8b183d804..0bb32123e 100644 --- a/backend/pashub_fetcher/local_handler/docker-compose.yml +++ b/applications/pashub_fetcher/local_handler/docker-compose.yml @@ -4,9 +4,9 @@ services: pashub-fetcher-lambda: build: context: ../../../ - dockerfile: backend/pashub_fetcher/handler/Dockerfile + dockerfile: applications/pashub_fetcher/handler/Dockerfile entrypoint: ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] - command: ["backend.pashub_fetcher.handler.handler.handler"] + command: ["applications.pashub_fetcher.handler.handler"] ports: - "9000:8080" env_file: diff --git a/backend/pashub_fetcher/local_handler/invoke_local_lambda.py b/applications/pashub_fetcher/local_handler/invoke_local_lambda.py similarity index 100% rename from backend/pashub_fetcher/local_handler/invoke_local_lambda.py rename to applications/pashub_fetcher/local_handler/invoke_local_lambda.py diff --git a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py b/applications/pashub_fetcher/pashub_to_ara_trigger_request.py similarity index 100% rename from backend/pashub_fetcher/pashub_to_ara_trigger_request.py rename to applications/pashub_fetcher/pashub_to_ara_trigger_request.py diff --git a/backend/pashub_fetcher/trigger_local_lambda_from_file.py b/applications/pashub_fetcher/trigger_local_lambda_from_file.py similarity index 93% rename from backend/pashub_fetcher/trigger_local_lambda_from_file.py rename to applications/pashub_fetcher/trigger_local_lambda_from_file.py index ad3097da1..cfc8ad697 100644 --- a/backend/pashub_fetcher/trigger_local_lambda_from_file.py +++ b/applications/pashub_fetcher/trigger_local_lambda_from_file.py @@ -5,10 +5,10 @@ from typing import Any, Dict, List from openpyxl import load_workbook -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) -from backend.pashub_fetcher.handler.handler import handler +from applications.pashub_fetcher.handler import handler if __name__ == "__main__": BASE_DIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py b/applications/pashub_fetcher/trigger_pashub_sqs_from_file.py similarity index 98% rename from backend/pashub_fetcher/trigger_pashub_sqs_from_file.py rename to applications/pashub_fetcher/trigger_pashub_sqs_from_file.py index b647c6a76..aa36aa7a5 100644 --- a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py +++ b/applications/pashub_fetcher/trigger_pashub_sqs_from_file.py @@ -7,7 +7,7 @@ import boto3 from openpyxl import load_workbook from backend.app.config import get_settings -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) diff --git a/applications/sharepoint_renamer/handler/Dockerfile b/applications/sharepoint_renamer/handler/Dockerfile index 6a0a28b46..40373e7b0 100644 --- a/applications/sharepoint_renamer/handler/Dockerfile +++ b/applications/sharepoint_renamer/handler/Dockerfile @@ -8,7 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt COPY utils/ utils/ COPY utilities/ utilities/ COPY backend/__init__.py backend/__init__.py -COPY backend/pashub_fetcher/ backend/pashub_fetcher/ +# SharepointSubfolders only — the rest of the PasHub service is not needed here. +COPY domain/pashub_fetcher/ domain/pashub_fetcher/ COPY orchestration/ orchestration/ COPY applications/sharepoint_renamer/ applications/sharepoint_renamer/ CMD ["applications.sharepoint_renamer.handler.handler"] diff --git a/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md index e13f81c84..f79faf188 100644 --- a/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md +++ b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md @@ -29,8 +29,9 @@ PDF) parses of 754881 are identical field-for-field — so the harness fixtures faithful and the ~2.7 MAE is a true calculator gap, not a fixture artifact. `parse_site_notes_pdf` never extracts the SAP rating from the PDF at all (`energy_rating_current` is populated by Dan's separate pashub-API fetch, -`pashub_service.py:45` ← `preSapRating`), which is why the rating is `None` on a -raw parse but present in the DB — unrelated to stripping. +`orchestration/pashub_fetcher_orchestrator.py` ← `preSapRating`), which is why +the rating is `None` on a raw parse but present in the DB — unrelated to +stripping. ## Properties that did not run (need PasHub re-extraction — data, not code) diff --git a/domain/pashub_fetcher/__init__.py b/domain/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/core_files.py b/domain/pashub_fetcher/core_files.py similarity index 60% rename from backend/pashub_fetcher/core_files.py rename to domain/pashub_fetcher/core_files.py index c6392e861..0f0bea34a 100644 --- a/backend/pashub_fetcher/core_files.py +++ b/domain/pashub_fetcher/core_files.py @@ -1,8 +1,6 @@ from enum import Enum from typing import Optional -from infrastructure.postgres.uploaded_file_table import FileTypeEnum - class CoreFiles(Enum): PHOTOPACK = "Photopack" @@ -20,23 +18,6 @@ class CoreFiles(Enum): MCS_COMPLIANCE_CERTIFICATE = "MCS Compliance Certificate" -_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = { - CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value, - CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value, - CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value, - CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value, - CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value, - CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value, - CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value, - CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value, - CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value, - CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value, - CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value, - CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value, - CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value, -} - - def get_core_file_type( filename: str, evidence_category: Optional[str] = None ) -> Optional[CoreFiles]: @@ -75,14 +56,3 @@ def get_core_file_type( return core_file return None - - -def get_file_type_string( - filename: str, evidence_category: Optional[str] = None -) -> Optional[str]: - core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category) - - if core_file is None: - return None - - return _CORE_FILE_TO_FILE_TYPE[core_file] diff --git a/backend/pashub_fetcher/evidence_categories.py b/domain/pashub_fetcher/evidence_categories.py similarity index 100% rename from backend/pashub_fetcher/evidence_categories.py rename to domain/pashub_fetcher/evidence_categories.py diff --git a/backend/pashub_fetcher/job.py b/domain/pashub_fetcher/job.py similarity index 100% rename from backend/pashub_fetcher/job.py rename to domain/pashub_fetcher/job.py diff --git a/backend/pashub_fetcher/sharepoint_subfolders.py b/domain/pashub_fetcher/sharepoint_subfolders.py similarity index 100% rename from backend/pashub_fetcher/sharepoint_subfolders.py rename to domain/pashub_fetcher/sharepoint_subfolders.py diff --git a/infrastructure/pashub_fetcher/__init__.py b/infrastructure/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infrastructure/pashub_fetcher/core_file_types.py b/infrastructure/pashub_fetcher/core_file_types.py new file mode 100644 index 000000000..9168dadec --- /dev/null +++ b/infrastructure/pashub_fetcher/core_file_types.py @@ -0,0 +1,39 @@ +"""Maps the domain's CoreFiles onto the persisted ``uploaded_file.file_type``. + +The classification rules — which filename/evidence category means which core +file — are domain policy and live in ``domain.pashub_fetcher.core_files``. Only +the translation to the storage enum belongs here, so the domain stays free of +any import of the persistence layer. +""" + +from typing import Optional + +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type +from infrastructure.postgres.uploaded_file_table import FileTypeEnum + +_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = { + CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value, + CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value, + CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value, + CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value, + CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value, + CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value, + CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value, + CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value, + CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value, + CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value, + CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value, + CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value, + CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value, +} + + +def get_file_type_string( + filename: str, evidence_category: Optional[str] = None +) -> Optional[str]: + core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category) + + if core_file is None: + return None + + return _CORE_FILE_TO_FILE_TYPE[core_file] diff --git a/backend/pashub_fetcher/evidence_file_data.py b/infrastructure/pashub_fetcher/evidence_file_data.py similarity index 100% rename from backend/pashub_fetcher/evidence_file_data.py rename to infrastructure/pashub_fetcher/evidence_file_data.py diff --git a/backend/pashub_fetcher/evidence_metadata.py b/infrastructure/pashub_fetcher/evidence_metadata.py similarity index 100% rename from backend/pashub_fetcher/evidence_metadata.py rename to infrastructure/pashub_fetcher/evidence_metadata.py diff --git a/backend/pashub_fetcher/pashub_client.py b/infrastructure/pashub_fetcher/pashub_client.py similarity index 97% rename from backend/pashub_fetcher/pashub_client.py rename to infrastructure/pashub_fetcher/pashub_client.py index aadaed41f..667756bd4 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/infrastructure/pashub_fetcher/pashub_client.py @@ -5,9 +5,9 @@ from datetime import datetime import requests -from backend.pashub_fetcher.core_files import CoreFiles, get_core_file_type -from backend.pashub_fetcher.evidence_file_data import EvidenceFileData -from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type +from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData +from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata from utils.logger import setup_logger logger = setup_logger() diff --git a/backend/pashub_fetcher/spreadsheet.py b/infrastructure/pashub_fetcher/spreadsheet.py similarity index 95% rename from backend/pashub_fetcher/spreadsheet.py rename to infrastructure/pashub_fetcher/spreadsheet.py index 5f8f74f6d..172fefc9a 100644 --- a/backend/pashub_fetcher/spreadsheet.py +++ b/infrastructure/pashub_fetcher/spreadsheet.py @@ -2,7 +2,7 @@ import re from typing import Dict, List from openpyxl import load_workbook -from backend.pashub_fetcher.job import Job +from domain.pashub_fetcher.job import Job def extract_jobs(filepath: str) -> List[Job]: diff --git a/backend/pashub_fetcher/token_getter.py b/infrastructure/pashub_fetcher/token_getter.py similarity index 100% rename from backend/pashub_fetcher/token_getter.py rename to infrastructure/pashub_fetcher/token_getter.py diff --git a/backend/pashub_fetcher/pashub_service.py b/orchestration/pashub_fetcher_orchestrator.py similarity index 100% rename from backend/pashub_fetcher/pashub_service.py rename to orchestration/pashub_fetcher_orchestrator.py diff --git a/orchestration/sharepoint_renamer_orchestrator.py b/orchestration/sharepoint_renamer_orchestrator.py index 3c7d3e8d3..a73373188 100644 --- a/orchestration/sharepoint_renamer_orchestrator.py +++ b/orchestration/sharepoint_renamer_orchestrator.py @@ -2,7 +2,7 @@ import csv import os from typing import Optional -from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders +from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from utilities.logger import setup_logger from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient diff --git a/pytest.ini b/pytest.ini index 2bcd61785..7cc9a255c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,7 +15,6 @@ testpaths = backend/ecmk_fetcher/tests backend/export/tests backend/onboarders/tests - backend/pashub_fetcher/tests datatypes/epc/domain/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests diff --git a/tests/applications/pashub_fetcher/__init__.py b/tests/applications/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py b/tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py similarity index 94% rename from backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py rename to tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py index 56187350b..d1c2d91ea 100644 --- a/backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py +++ b/tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py @@ -1,6 +1,6 @@ import pytest -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) diff --git a/tests/domain/pashub_fetcher/__init__.py b/tests/domain/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/tests/test_core_files.py b/tests/domain/pashub_fetcher/test_core_files.py similarity index 51% rename from backend/pashub_fetcher/tests/test_core_files.py rename to tests/domain/pashub_fetcher/test_core_files.py index a2047ecee..e29cf1997 100644 --- a/backend/pashub_fetcher/tests/test_core_files.py +++ b/tests/domain/pashub_fetcher/test_core_files.py @@ -1,96 +1,4 @@ -from backend.pashub_fetcher.core_files import ( - CoreFiles, - get_core_file_type, - get_file_type_string, -) - - -def test_file_type_for_photopack(): - assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack" - - -def test_file_type_for_sitenote(): - assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note" - - -def test_file_type_for_rdsap_sitenote(): - assert ( - get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf") - == "rd_sap_site_note" - ) - - -def test_file_type_for_pas2023_ventilation(): - assert ( - get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf") - == "pas_2023_ventilation" - ) - - -def test_file_type_for_pas2023_condition(): - assert ( - get_file_type_string("PAS 2023 Condition Report_123456.pdf") - == "pas_2023_condition" - ) - - -def test_file_type_for_pas_significance(): - assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance" - - -def test_file_type_for_par_photopack(): - assert ( - get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf") - == "par_photo_pack" - ) - - -def test_file_type_for_pas2023_property(): - assert ( - get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf") - == "pas_2023_property" - ) - - -def test_file_type_for_pas2023_occupancy(): - assert ( - get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf") - == "pas_2023_occupancy" - ) - - -def test_file_type_for_improvement_option_evaluation(): - # filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf" - assert ( - get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf") - == "improvement_option_evaluation" - ) - - -def test_file_type_for_medium_term_improvement_plan(): - # filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf" - assert ( - get_file_type_string( - "60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf" - ) - == "medium_term_improvement_plan" - ) - - -def test_file_type_for_retrofit_design_doc(): - assert ( - get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf") - == "retrofit_design_doc" - ) - assert ( - get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf") - == "retrofit_design_doc" - ) - - -# --------------------------------------------------------------------------- -# core_file_for -# --------------------------------------------------------------------------- +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type def test_core_file_for_evidence_category_match_is_case_insensitive() -> None: @@ -209,18 +117,3 @@ def test_core_file_for_mcs_compliance_certificate_is_case_insensitive() -> None: # Assert assert result == CoreFiles.MCS_COMPLIANCE_CERTIFICATE - - -def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> ( - None -): - # Arrange - filename = "some_cert.pdf" - - # Act - result = get_file_type_string( - filename, evidence_category="MCS Compliance Certificate" - ) - - # Assert - assert result == "mcs_compliance_certificate" diff --git a/tests/infrastructure/pashub_fetcher/__init__.py b/tests/infrastructure/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/infrastructure/pashub_fetcher/test_core_file_types.py b/tests/infrastructure/pashub_fetcher/test_core_file_types.py new file mode 100644 index 000000000..9b95c9b3e --- /dev/null +++ b/tests/infrastructure/pashub_fetcher/test_core_file_types.py @@ -0,0 +1,98 @@ +from infrastructure.pashub_fetcher.core_file_types import get_file_type_string + + +def test_file_type_for_photopack(): + assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack" + + +def test_file_type_for_sitenote(): + assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note" + + +def test_file_type_for_rdsap_sitenote(): + assert ( + get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf") + == "rd_sap_site_note" + ) + + +def test_file_type_for_pas2023_ventilation(): + assert ( + get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf") + == "pas_2023_ventilation" + ) + + +def test_file_type_for_pas2023_condition(): + assert ( + get_file_type_string("PAS 2023 Condition Report_123456.pdf") + == "pas_2023_condition" + ) + + +def test_file_type_for_pas_significance(): + assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance" + + +def test_file_type_for_par_photopack(): + assert ( + get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf") + == "par_photo_pack" + ) + + +def test_file_type_for_pas2023_property(): + assert ( + get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf") + == "pas_2023_property" + ) + + +def test_file_type_for_pas2023_occupancy(): + assert ( + get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf") + == "pas_2023_occupancy" + ) + + +def test_file_type_for_improvement_option_evaluation(): + # filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf" + assert ( + get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf") + == "improvement_option_evaluation" + ) + + +def test_file_type_for_medium_term_improvement_plan(): + # filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf" + assert ( + get_file_type_string( + "60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf" + ) + == "medium_term_improvement_plan" + ) + + +def test_file_type_for_retrofit_design_doc(): + assert ( + get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf") + == "retrofit_design_doc" + ) + assert ( + get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf") + == "retrofit_design_doc" + ) + +def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> ( + None +): + # Arrange + filename = "some_cert.pdf" + + # Act + result = get_file_type_string( + filename, evidence_category="MCS Compliance Certificate" + ) + + # Assert + assert result == "mcs_compliance_certificate" diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/tests/infrastructure/pashub_fetcher/test_pashub_client.py similarity index 97% rename from backend/pashub_fetcher/tests/test_pashub_client.py rename to tests/infrastructure/pashub_fetcher/test_pashub_client.py index ecc92cde4..c1ea1ee29 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/tests/infrastructure/pashub_fetcher/test_pashub_client.py @@ -5,10 +5,10 @@ from unittest.mock import MagicMock, patch import pytest import requests -from backend.pashub_fetcher.core_files import CoreFiles -from backend.pashub_fetcher.evidence_file_data import EvidenceFileData -from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata -from backend.pashub_fetcher.pashub_client import ( +from domain.pashub_fetcher.core_files import CoreFiles +from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData +from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata +from infrastructure.pashub_fetcher.pashub_client import ( DownloadedFiles, PashubClient, RdSapSummary, diff --git a/backend/pashub_fetcher/tests/test_token_getter.py b/tests/infrastructure/pashub_fetcher/test_token_getter.py similarity index 84% rename from backend/pashub_fetcher/tests/test_token_getter.py rename to tests/infrastructure/pashub_fetcher/test_token_getter.py index 2c847d167..8e8b0c447 100644 --- a/backend/pashub_fetcher/tests/test_token_getter.py +++ b/tests/infrastructure/pashub_fetcher/test_token_getter.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock, patch -from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None: @@ -20,9 +20,9 @@ def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None: mock_sync_playwright.return_value.__enter__.return_value = mock_p -@patch("backend.pashub_fetcher.token_getter.shutil.rmtree") -@patch("backend.pashub_fetcher.token_getter.glob.glob") -@patch("backend.pashub_fetcher.token_getter.sync_playwright") +@patch("infrastructure.pashub_fetcher.token_getter.shutil.rmtree") +@patch("infrastructure.pashub_fetcher.token_getter.glob.glob") +@patch("infrastructure.pashub_fetcher.token_getter.sync_playwright") def test_playwright_tmp_dirs_are_cleaned_up_after_browser_close( mock_sync_playwright: MagicMock, mock_glob: MagicMock, diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/tests/orchestration/test_pashub_fetcher_orchestrator.py similarity index 100% rename from backend/pashub_fetcher/tests/test_pashub_service.py rename to tests/orchestration/test_pashub_fetcher_orchestrator.py diff --git a/tests/scripts/test_rename_sharepoint_files.py b/tests/scripts/test_rename_sharepoint_files.py index c732383c7..cd27d8a29 100644 --- a/tests/scripts/test_rename_sharepoint_files.py +++ b/tests/scripts/test_rename_sharepoint_files.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest -from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders +from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from orchestration.sharepoint_renamer_orchestrator import ( ASSESSMENT_SUBFOLDER, BASE_PATH,