Move the PasHub fetcher into the DDD layer structure 🟪

The service was the last one living wholly under backend/. It now follows
the same layering as abri and the other newer services:

  domain/pashub_fetcher/          core file classification, subfolders
  infrastructure/pashub_fetcher/  PasHub client, token getter, wire DTOs
  orchestration/                  PashubFetcherOrchestrator (was PashubService)
  applications/pashub_fetcher/    lambda handler, trigger request, dev tooling

core_files.py is split along the layer boundary: the domain module keeps the
filename/evidence-category classification rules and no longer imports
infrastructure.postgres, while the CoreFiles -> FileTypeEnum translation moves
to infrastructure/pashub_fetcher/core_file_types.py.

Tests move into the tests/ tree by layer. Note this puts them in the only
suite CI currently runs (unit_tests.yml is disabled), so these 73 tests now
execute on PRs for the first time; they were previously reachable only via
the legacy pytest.ini testpaths.

sharepoint_renamer's image now copies just domain/pashub_fetcher/ rather than
the whole service, since SharepointSubfolders is all it needed.

Behaviour is unchanged. tests/ goes 9927 -> 10000 passed (+73, exactly the
tests that moved in); the legacy suite keeps its same 17 pre-existing failures
and 11 errors. tests/test_lambda_packaging.py confirms both changed
Dockerfiles still copy their handler's full import closure.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Roth 2026-07-22 13:35:54 +00:00
parent 98faeb0b76
commit ae1da26558
41 changed files with 178 additions and 173 deletions

View file

@ -453,7 +453,7 @@ jobs:
uses: ./.github/workflows/_build_image.yml
with:
ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }}
dockerfile_path: backend/pashub_fetcher/handler/Dockerfile
dockerfile_path: applications/pashub_fetcher/handler/Dockerfile
build_context: .
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}

View file

@ -109,7 +109,7 @@
# pashub_smoke_test:
# uses: ./.github/workflows/_smoke_test_lambda.yml
# with:
# dockerfile_path: backend/pashub_fetcher/handler/Dockerfile
# dockerfile_path: applications/pashub_fetcher/handler/Dockerfile
# build_context: .
# service_name: pashub

View file

View file

@ -1,12 +1,12 @@
from typing import Any, Callable, Dict, List, Optional
from backend.app.config import get_settings
from backend.pashub_fetcher.pashub_client import PashubClient
from backend.pashub_fetcher.pashub_service import PashubService
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
from infrastructure.pashub_fetcher.pashub_client import PashubClient
from orchestration.pashub_fetcher_orchestrator import PashubFetcherOrchestrator
from applications.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)
from backend.pashub_fetcher.token_getter import get_token_from_local_storage
from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage
from backend.app.db.models.tasks import SourceEnum
from backend.utils.subtasks import task_handler
from utils.logger import setup_logger
@ -63,7 +63,7 @@ def handler(body: Dict[str, Any], context: Any) -> List[str]:
f"Unrecognised sharepoint_site '{payload.sharepoint_site}'; skipping SharePoint upload"
)
service = PashubService(
service = PashubFetcherOrchestrator(
pashub_client=get_pashub_client(pashub_email, pashub_password),
sharepoint_client=sharepoint_client,
s3_bucket=S3_BUCKET,

View file

@ -1,6 +1,6 @@
# jammy (Ubuntu 22.04) ships Python 3.10, which lacks enum.StrEnum — used by
# domain/modelling/measure_type.py, which the handler pulls in transitively
# (handler -> pashub_service -> documents_parser.parser -> ... -> domain).
# (handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> ... -> domain).
# noble (Ubuntu 24.04) ships Python 3.12, matching the project's 3.11+ standard.
FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
@ -14,14 +14,18 @@ WORKDIR /var/task
COPY utils/ utils/
COPY backend/ backend/
COPY datatypes/ datatypes/
# handler -> pashub_service -> documents_parser.parser -> datatypes.epc.domain.mapper
# handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> datatypes.epc.domain.mapper
# -> domain.sap10_calculator, and -> db_writer -> epc_property model
# -> infrastructure.postgres. Without these the lambda fails at init with
# "No module named 'domain'" / "'infrastructure'".
COPY domain/ domain/
COPY infrastructure/ infrastructure/
# The service now lives across the DDD layers: the handler is in applications/,
# PashubFetcherOrchestrator in orchestration/, the client in infrastructure/.
COPY applications/ applications/
COPY orchestration/ orchestration/
COPY backend/pashub_fetcher/handler/requirements.txt .
COPY applications/pashub_fetcher/handler/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Local lambda entrypoint
@ -33,4 +37,4 @@ ENTRYPOINT ["python", "-m", "awslambdaric"]
# -----------------------------
# Lambda handler
# -----------------------------
CMD ["backend.pashub_fetcher.handler.handler.handler"]
CMD ["applications.pashub_fetcher.handler.handler"]

View file

@ -4,9 +4,9 @@ services:
pashub-fetcher-lambda:
build:
context: ../../../
dockerfile: backend/pashub_fetcher/handler/Dockerfile
dockerfile: applications/pashub_fetcher/handler/Dockerfile
entrypoint: ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"]
command: ["backend.pashub_fetcher.handler.handler.handler"]
command: ["applications.pashub_fetcher.handler.handler"]
ports:
- "9000:8080"
env_file:

View file

@ -5,10 +5,10 @@ from typing import Any, Dict, List
from openpyxl import load_workbook
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
from applications.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)
from backend.pashub_fetcher.handler.handler import handler
from applications.pashub_fetcher.handler import handler
if __name__ == "__main__":
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

View file

@ -7,7 +7,7 @@ import boto3
from openpyxl import load_workbook
from backend.app.config import get_settings
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
from applications.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)

View file

@ -8,7 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt
COPY utils/ utils/
COPY utilities/ utilities/
COPY backend/__init__.py backend/__init__.py
COPY backend/pashub_fetcher/ backend/pashub_fetcher/
# SharepointSubfolders only — the rest of the PasHub service is not needed here.
COPY domain/pashub_fetcher/ domain/pashub_fetcher/
COPY orchestration/ orchestration/
COPY applications/sharepoint_renamer/ applications/sharepoint_renamer/
CMD ["applications.sharepoint_renamer.handler.handler"]

View file

@ -29,8 +29,9 @@ PDF) parses of 754881 are identical field-for-field — so the harness fixtures
faithful and the ~2.7 MAE is a true calculator gap, not a fixture artifact.
`parse_site_notes_pdf` never extracts the SAP rating from the PDF at all
(`energy_rating_current` is populated by Dan's separate pashub-API fetch,
`pashub_service.py:45``preSapRating`), which is why the rating is `None` on a
raw parse but present in the DB — unrelated to stripping.
`orchestration/pashub_fetcher_orchestrator.py``preSapRating`), which is why
the rating is `None` on a raw parse but present in the DB — unrelated to
stripping.
## Properties that did not run (need PasHub re-extraction — data, not code)

View file

View file

@ -1,8 +1,6 @@
from enum import Enum
from typing import Optional
from infrastructure.postgres.uploaded_file_table import FileTypeEnum
class CoreFiles(Enum):
PHOTOPACK = "Photopack"
@ -20,23 +18,6 @@ class CoreFiles(Enum):
MCS_COMPLIANCE_CERTIFICATE = "MCS Compliance Certificate"
_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = {
CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value,
CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value,
CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value,
CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value,
CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value,
CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value,
CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value,
CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value,
CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value,
CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value,
CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value,
CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value,
CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value,
}
def get_core_file_type(
filename: str, evidence_category: Optional[str] = None
) -> Optional[CoreFiles]:
@ -75,14 +56,3 @@ def get_core_file_type(
return core_file
return None
def get_file_type_string(
filename: str, evidence_category: Optional[str] = None
) -> Optional[str]:
core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category)
if core_file is None:
return None
return _CORE_FILE_TO_FILE_TYPE[core_file]

View file

@ -0,0 +1,39 @@
"""Maps the domain's CoreFiles onto the persisted ``uploaded_file.file_type``.
The classification rules which filename/evidence category means which core
file are domain policy and live in ``domain.pashub_fetcher.core_files``. Only
the translation to the storage enum belongs here, so the domain stays free of
any import of the persistence layer.
"""
from typing import Optional
from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type
from infrastructure.postgres.uploaded_file_table import FileTypeEnum
_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = {
CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value,
CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value,
CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value,
CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value,
CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value,
CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value,
CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value,
CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value,
CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value,
CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value,
CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value,
CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value,
CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value,
}
def get_file_type_string(
filename: str, evidence_category: Optional[str] = None
) -> Optional[str]:
core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category)
if core_file is None:
return None
return _CORE_FILE_TO_FILE_TYPE[core_file]

View file

@ -5,9 +5,9 @@ from datetime import datetime
import requests
from backend.pashub_fetcher.core_files import CoreFiles, get_core_file_type
from backend.pashub_fetcher.evidence_file_data import EvidenceFileData
from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata
from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type
from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData
from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata
from utils.logger import setup_logger
logger = setup_logger()

View file

@ -2,7 +2,7 @@ import re
from typing import Dict, List
from openpyxl import load_workbook
from backend.pashub_fetcher.job import Job
from domain.pashub_fetcher.job import Job
def extract_jobs(filepath: str) -> List[Job]:

View file

@ -2,7 +2,7 @@ import csv
import os
from typing import Optional
from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from utilities.logger import setup_logger
from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient

View file

@ -15,7 +15,6 @@ testpaths =
backend/ecmk_fetcher/tests
backend/export/tests
backend/onboarders/tests
backend/pashub_fetcher/tests
datatypes/epc/domain/tests
datatypes/epc/schema/tests
datatypes/epc/surveys/tests

View file

@ -1,6 +1,6 @@
import pytest
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
from applications.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)

View file

View file

@ -1,96 +1,4 @@
from backend.pashub_fetcher.core_files import (
CoreFiles,
get_core_file_type,
get_file_type_string,
)
def test_file_type_for_photopack():
assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack"
def test_file_type_for_sitenote():
assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note"
def test_file_type_for_rdsap_sitenote():
assert (
get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf")
== "rd_sap_site_note"
)
def test_file_type_for_pas2023_ventilation():
assert (
get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf")
== "pas_2023_ventilation"
)
def test_file_type_for_pas2023_condition():
assert (
get_file_type_string("PAS 2023 Condition Report_123456.pdf")
== "pas_2023_condition"
)
def test_file_type_for_pas_significance():
assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance"
def test_file_type_for_par_photopack():
assert (
get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf")
== "par_photo_pack"
)
def test_file_type_for_pas2023_property():
assert (
get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf")
== "pas_2023_property"
)
def test_file_type_for_pas2023_occupancy():
assert (
get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf")
== "pas_2023_occupancy"
)
def test_file_type_for_improvement_option_evaluation():
# filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf"
assert (
get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf")
== "improvement_option_evaluation"
)
def test_file_type_for_medium_term_improvement_plan():
# filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf"
assert (
get_file_type_string(
"60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf"
)
== "medium_term_improvement_plan"
)
def test_file_type_for_retrofit_design_doc():
assert (
get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf")
== "retrofit_design_doc"
)
assert (
get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf")
== "retrofit_design_doc"
)
# ---------------------------------------------------------------------------
# core_file_for
# ---------------------------------------------------------------------------
from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type
def test_core_file_for_evidence_category_match_is_case_insensitive() -> None:
@ -209,18 +117,3 @@ def test_core_file_for_mcs_compliance_certificate_is_case_insensitive() -> None:
# Assert
assert result == CoreFiles.MCS_COMPLIANCE_CERTIFICATE
def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> (
None
):
# Arrange
filename = "some_cert.pdf"
# Act
result = get_file_type_string(
filename, evidence_category="MCS Compliance Certificate"
)
# Assert
assert result == "mcs_compliance_certificate"

View file

@ -0,0 +1,98 @@
from infrastructure.pashub_fetcher.core_file_types import get_file_type_string
def test_file_type_for_photopack():
assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack"
def test_file_type_for_sitenote():
assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note"
def test_file_type_for_rdsap_sitenote():
assert (
get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf")
== "rd_sap_site_note"
)
def test_file_type_for_pas2023_ventilation():
assert (
get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf")
== "pas_2023_ventilation"
)
def test_file_type_for_pas2023_condition():
assert (
get_file_type_string("PAS 2023 Condition Report_123456.pdf")
== "pas_2023_condition"
)
def test_file_type_for_pas_significance():
assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance"
def test_file_type_for_par_photopack():
assert (
get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf")
== "par_photo_pack"
)
def test_file_type_for_pas2023_property():
assert (
get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf")
== "pas_2023_property"
)
def test_file_type_for_pas2023_occupancy():
assert (
get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf")
== "pas_2023_occupancy"
)
def test_file_type_for_improvement_option_evaluation():
# filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf"
assert (
get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf")
== "improvement_option_evaluation"
)
def test_file_type_for_medium_term_improvement_plan():
# filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf"
assert (
get_file_type_string(
"60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf"
)
== "medium_term_improvement_plan"
)
def test_file_type_for_retrofit_design_doc():
assert (
get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf")
== "retrofit_design_doc"
)
assert (
get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf")
== "retrofit_design_doc"
)
def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> (
None
):
# Arrange
filename = "some_cert.pdf"
# Act
result = get_file_type_string(
filename, evidence_category="MCS Compliance Certificate"
)
# Assert
assert result == "mcs_compliance_certificate"

View file

@ -5,10 +5,10 @@ from unittest.mock import MagicMock, patch
import pytest
import requests
from backend.pashub_fetcher.core_files import CoreFiles
from backend.pashub_fetcher.evidence_file_data import EvidenceFileData
from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata
from backend.pashub_fetcher.pashub_client import (
from domain.pashub_fetcher.core_files import CoreFiles
from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData
from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata
from infrastructure.pashub_fetcher.pashub_client import (
DownloadedFiles,
PashubClient,
RdSapSummary,

View file

@ -1,6 +1,6 @@
from unittest.mock import MagicMock, patch
from backend.pashub_fetcher.token_getter import get_token_from_local_storage
from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage
def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None:
@ -20,9 +20,9 @@ def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None:
mock_sync_playwright.return_value.__enter__.return_value = mock_p
@patch("backend.pashub_fetcher.token_getter.shutil.rmtree")
@patch("backend.pashub_fetcher.token_getter.glob.glob")
@patch("backend.pashub_fetcher.token_getter.sync_playwright")
@patch("infrastructure.pashub_fetcher.token_getter.shutil.rmtree")
@patch("infrastructure.pashub_fetcher.token_getter.glob.glob")
@patch("infrastructure.pashub_fetcher.token_getter.sync_playwright")
def test_playwright_tmp_dirs_are_cleaned_up_after_browser_close(
mock_sync_playwright: MagicMock,
mock_glob: MagicMock,

View file

@ -4,7 +4,7 @@ from unittest.mock import MagicMock
import pytest
from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from orchestration.sharepoint_renamer_orchestrator import (
ASSESSMENT_SUBFOLDER,
BASE_PATH,