From 5178cd02c59b695fc4d1402c97886cd1c8fc0b94 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:50:51 +0000 Subject: [PATCH] =?UTF-8?q?UploadedFile,=20FileTypeEnum,=20FileSourceEnum?= =?UTF-8?q?=20importable=20from=20infrastructure.postgres.uploaded=5Ffile?= =?UTF-8?q?=5Ftable=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../db/functions/uploaded_files_functions.py | 2 +- backend/app/db/models/uploaded_file.py | 69 ------------- backend/ecmk_fetcher/ecmk_service.py | 2 +- backend/ecmk_fetcher/reports.py | 2 +- .../ecmk_fetcher/tests/test_ecmk_service.py | 2 +- backend/ecmk_fetcher/tests/test_upload.py | 2 +- backend/ecmk_fetcher/upload.py | 2 +- backend/pashub_fetcher/core_files.py | 2 +- backend/pashub_fetcher/pashub_service.py | 2 +- .../tests/test_pashub_service.py | 2 +- .../postgres/uploaded_file_table.py | 96 +++++++++++++++++++ orchestration/magic_plan_orchestrator.py | 2 +- tests/conftest.py | 40 +++++++- .../test_magic_plan_orchestrator.py | 2 +- 14 files changed, 146 insertions(+), 81 deletions(-) delete mode 100644 backend/app/db/models/uploaded_file.py create mode 100644 infrastructure/postgres/uploaded_file_table.py diff --git a/backend/app/db/functions/uploaded_files_functions.py b/backend/app/db/functions/uploaded_files_functions.py index 3708813a..44af55ea 100644 --- a/backend/app/db/functions/uploaded_files_functions.py +++ b/backend/app/db/functions/uploaded_files_functions.py @@ -3,7 +3,7 @@ from typing import Optional from sqlalchemy import select from backend.app.db.connection import db_read_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/app/db/models/uploaded_file.py b/backend/app/db/models/uploaded_file.py deleted file mode 100644 index e00acbe1..00000000 --- a/backend/app/db/models/uploaded_file.py +++ /dev/null @@ -1,69 +0,0 @@ -import enum -from sqlalchemy import TIMESTAMP, BigInteger, Column, Text, Enum as SqlEnum - -from backend.app.db.base import Base - - -class FileTypeEnum(enum.Enum): - PHOTO_PACK = "photo_pack" - SITE_NOTE = "site_note" - RD_SAP_SITE_NOTE = "rd_sap_site_note" - PAS_2023_VENTILATION = "pas_2023_ventilation" - PAS_2023_CONDITION = "pas_2023_condition" - PAS_SIGNIFICANCE = "pas_significance" - PAR_PHOTO_PACK = "par_photo_pack" - PAS_2023_PROPERTY = "pas_2023_property" - PAS_2023_OCCUPANCY = "pas_2023_occupancy" - ECMK_SITE_NOTE = "ecmk_site_note" - ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" - ECMK_SURVEY_XML = "ecmk_survey_xml" - MAGIC_PLAN_JSON = "magic_plan_json" - IMPROVEMENT_OPTION_EVALUATION = "improvement_option_evaluation" - MEDIUM_TERM_IMPROVEMENT_PLAN = "medium_term_improvement_plan" - RETROFIT_DESIGN_DOC = "retrofit_design_doc" - MCS_COMPLIANCE_CERTIFICATE = "mcs_compliance_certificate" - OTHER = "other" - - -class FileSourceEnum(enum.Enum): - PAS_HUB = "pas hub" - COORDINATION_HUB = "coordination_hub" - SHAREPOINT = "sharepoint" - HUBSPOT = "hubspot" - ECMK = "ecmk" - MAGIC_PLAN = "magic_plan" - - -class UploadedFile(Base): - __tablename__ = "uploaded_files" - - id = Column(BigInteger, primary_key=True, autoincrement=True) - - s3_file_bucket = Column(Text, nullable=False) - s3_file_key = Column(Text, nullable=False) - s3_upload_timestamp = Column(TIMESTAMP(timezone=True), nullable=False) - - landlord_property_id = Column(Text, nullable=True) - uprn = Column(BigInteger, nullable=True) - hubspot_listing_id = Column(BigInteger, nullable=True) - hubspot_deal_id = Column(Text, nullable=True) - - file_type = Column( - SqlEnum( - FileTypeEnum, - name="file_type", - create_type=False, - values_callable=lambda enum_cls: [e.value for e in enum_cls], - ), - nullable=True, - ) - - file_source = Column( - SqlEnum( - FileSourceEnum, - name="file_source", - create_type=False, - values_callable=lambda enum_cls: [e.value for e in enum_cls], - ), - nullable=True, - ) diff --git a/backend/ecmk_fetcher/ecmk_service.py b/backend/ecmk_fetcher/ecmk_service.py index 35b8f552..fa613e4b 100644 --- a/backend/ecmk_fetcher/ecmk_service.py +++ b/backend/ecmk_fetcher/ecmk_service.py @@ -7,7 +7,7 @@ from backend.app.db.connection import db_session from backend.app.db.functions.uploaded_files_functions import ( get_uploaded_file_by_listing_type_and_source, ) -from backend.app.db.models.uploaded_file import FileSourceEnum, FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum from backend.documents_parser.db_writer import save_epc_property_data from backend.documents_parser.parser import parse_site_notes_pdf from backend.ecmk_fetcher.address_list import ( diff --git a/backend/ecmk_fetcher/reports.py b/backend/ecmk_fetcher/reports.py index d2f8ea52..2d77ab45 100644 --- a/backend/ecmk_fetcher/reports.py +++ b/backend/ecmk_fetcher/reports.py @@ -1,6 +1,6 @@ from enum import Enum -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum class FileDownloadButtonType(Enum): diff --git a/backend/ecmk_fetcher/tests/test_ecmk_service.py b/backend/ecmk_fetcher/tests/test_ecmk_service.py index 703bc4c5..b8c8cc53 100644 --- a/backend/ecmk_fetcher/tests/test_ecmk_service.py +++ b/backend/ecmk_fetcher/tests/test_ecmk_service.py @@ -1,7 +1,7 @@ from typing import Dict from unittest.mock import MagicMock, call, patch -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum from backend.ecmk_fetcher.address_list import PropertyRow from backend.ecmk_fetcher.ecmk_service import EcmkService from backend.ecmk_fetcher.reports import FileDownloadButtonType diff --git a/backend/ecmk_fetcher/tests/test_upload.py b/backend/ecmk_fetcher/tests/test_upload.py index 79823e8e..7cb6e8a9 100644 --- a/backend/ecmk_fetcher/tests/test_upload.py +++ b/backend/ecmk_fetcher/tests/test_upload.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, call, patch import pytest -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum from backend.ecmk_fetcher.upload import upload_file_to_s3_and_record diff --git a/backend/ecmk_fetcher/upload.py b/backend/ecmk_fetcher/upload.py index fc05363c..bf13a36b 100644 --- a/backend/ecmk_fetcher/upload.py +++ b/backend/ecmk_fetcher/upload.py @@ -3,7 +3,7 @@ import os from typing import cast from backend.app.db.connection import db_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/pashub_fetcher/core_files.py b/backend/pashub_fetcher/core_files.py index c387e0b8..c6392e86 100644 --- a/backend/pashub_fetcher/core_files.py +++ b/backend/pashub_fetcher/core_files.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Optional -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum class CoreFiles(Enum): diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 86a553f0..64c2f6ae 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from typing import Callable, List, NamedTuple, Optional, cast from backend.app.db.connection import db_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index ccb80ac4..09635c92 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -4,7 +4,7 @@ from typing import Any, Callable, Optional from unittest.mock import MagicMock, call, patch -from backend.app.db.models.uploaded_file import FileSourceEnum, FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, diff --git a/infrastructure/postgres/uploaded_file_table.py b/infrastructure/postgres/uploaded_file_table.py new file mode 100644 index 00000000..8f1a7e64 --- /dev/null +++ b/infrastructure/postgres/uploaded_file_table.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import enum +from typing import ClassVar, Optional + +from sqlalchemy import TIMESTAMP, BigInteger, Column, Text +from sqlalchemy import Enum as SqlEnum +from sqlmodel import Field, SQLModel + + +class FileTypeEnum(enum.Enum): + PHOTO_PACK = "photo_pack" + SITE_NOTE = "site_note" + RD_SAP_SITE_NOTE = "rd_sap_site_note" + PAS_2023_VENTILATION = "pas_2023_ventilation" + PAS_2023_CONDITION = "pas_2023_condition" + PAS_SIGNIFICANCE = "pas_significance" + PAR_PHOTO_PACK = "par_photo_pack" + PAS_2023_PROPERTY = "pas_2023_property" + PAS_2023_OCCUPANCY = "pas_2023_occupancy" + ECMK_SITE_NOTE = "ecmk_site_note" + ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" + ECMK_SURVEY_XML = "ecmk_survey_xml" + MAGIC_PLAN_JSON = "magic_plan_json" + IMPROVEMENT_OPTION_EVALUATION = "improvement_option_evaluation" + MEDIUM_TERM_IMPROVEMENT_PLAN = "medium_term_improvement_plan" + RETROFIT_DESIGN_DOC = "retrofit_design_doc" + MCS_COMPLIANCE_CERTIFICATE = "mcs_compliance_certificate" + OTHER = "other" + VENTILATION_AUDIT = "ventilation_audit" + + +class FileSourceEnum(enum.Enum): + PAS_HUB = "pas hub" + COORDINATION_HUB = "coordination_hub" + SHAREPOINT = "sharepoint" + HUBSPOT = "hubspot" + ECMK = "ecmk" + MAGIC_PLAN = "magic_plan" + AUDIT_GENERATOR = "audit_generator" + + +def _enum_values(enum_cls: type[enum.Enum]) -> list[str]: + return [e.value for e in enum_cls] + + +class UploadedFile(SQLModel, table=True): + __tablename__: ClassVar[str] = "uploaded_files" # pyright: ignore[reportIncompatibleVariableOverride] + + id: Optional[int] = Field( + default=None, + sa_column=Column(BigInteger, primary_key=True, autoincrement=True), + ) + s3_file_bucket: str = Field(sa_column=Column(Text, nullable=False)) + s3_file_key: str = Field(sa_column=Column(Text, nullable=False)) + s3_upload_timestamp: object = Field( + sa_column=Column(TIMESTAMP(timezone=True), nullable=False) + ) + + landlord_property_id: Optional[str] = Field( + default=None, sa_column=Column(Text, nullable=True) + ) + uprn: Optional[int] = Field( + default=None, sa_column=Column(BigInteger, nullable=True) + ) + hubspot_listing_id: Optional[int] = Field( + default=None, sa_column=Column(BigInteger, nullable=True) + ) + hubspot_deal_id: Optional[str] = Field( + default=None, sa_column=Column(Text, nullable=True) + ) + + file_type: Optional[str] = Field( + default=None, + sa_column=Column( + SqlEnum( + FileTypeEnum, + name="file_type", + create_type=False, + values_callable=_enum_values, + ), + nullable=True, + ), + ) + file_source: Optional[str] = Field( + default=None, + sa_column=Column( + SqlEnum( + FileSourceEnum, + name="file_source", + create_type=False, + values_callable=_enum_values, + ), + nullable=True, + ), + ) diff --git a/orchestration/magic_plan_orchestrator.py b/orchestration/magic_plan_orchestrator.py index a3191985..9cbd9129 100644 --- a/orchestration/magic_plan_orchestrator.py +++ b/orchestration/magic_plan_orchestrator.py @@ -8,7 +8,7 @@ from domain.magicplan.api.response import MagicPlanPlan, PlanSummary from domain.magicplan.mapper import map_plan from domain.magicplan.models import Plan -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/tests/conftest.py b/tests/conftest.py index 0a246372..17c2eefd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,12 +17,13 @@ from typing import Any import pytest from psycopg import Connection from pytest_postgresql import factories -from sqlalchemy import Engine +from sqlalchemy import Engine, text from sqlmodel import SQLModel, create_engine # Importing the SQLModel row modules registers their tables on # SQLModel.metadata so ``create_all`` builds the full schema. Imports look # unused; they aren't. +import infrastructure.postgres.uploaded_file_table as _uf_table # pyright: ignore[reportUnusedImport] # pg_ctl ships under a versioned path and is not on PATH in the dev container. @@ -34,12 +35,49 @@ postgresql_proc = factories.postgresql_proc( postgresql = factories.postgresql("postgresql_proc") +def _create_pg_enum_types(engine: Engine) -> None: + """Emit CREATE TYPE for PostgreSQL enum types used by UploadedFile. + + SQLModel.metadata.create_all uses create_type=False for these enums + (they are normally created by Alembic migrations). Tests need them upfront. + A DO block swallows duplicate_object so the fixture is safe to call on a + pre-seeded database. + """ + from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum + + ft_values = ", ".join(f"'{e.value}'" for e in FileTypeEnum) + fs_values = ", ".join(f"'{e.value}'" for e in FileSourceEnum) + with engine.connect() as conn: + conn.execute( + text( + f""" + DO $$ BEGIN + CREATE TYPE file_type AS ENUM ({ft_values}); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + """ + ) + ) + conn.execute( + text( + f""" + DO $$ BEGIN + CREATE TYPE file_source AS ENUM ({fs_values}); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + """ + ) + ) + conn.commit() + + @pytest.fixture def db_engine(postgresql: Connection[Any]) -> Iterator[Engine]: """A SQLModel engine bound to a fresh, ephemeral PostgreSQL database.""" info = postgresql.info url = f"postgresql+psycopg://{info.user}:@{info.host}:{info.port}/{info.dbname}" engine = create_engine(url) + _create_pg_enum_types(engine) SQLModel.metadata.create_all(engine) try: yield engine diff --git a/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py b/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py index c688008e..9c65b6c9 100644 --- a/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py +++ b/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py @@ -9,7 +9,7 @@ from domain.magicplan.api.response import MagicPlanPlan, PlanSummary from domain.magicplan.mapper import map_plan from domain.magicplan.models import Plan -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile,