working on integration test

.idea/Model.iml

@@ -10,4 +10,7 @@
     <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
 </module>

backend/app/db/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()

backend/app/db/models/addresses.py

@@ -7,9 +7,7 @@ from sqlalchemy import (
     func,
     UniqueConstraint,
 )
-from sqlalchemy.orm import declarative_base
-
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class PostcodeSearch(Base):

backend/app/db/models/condition.py

@@ -7,12 +7,12 @@ from sqlalchemy import (
     String,
     Enum as SqlEnum,
 )
-from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy.orm import relationship
 
 from backend.condition.domain.aspect_type import AspectType
 from backend.condition.domain.element_type import ElementType
 
-Base = declarative_base()
+from backend.app.db.base import Base
 
 ElementTypeDb = SqlEnum(
     ElementType,

backend/app/db/models/energy_assessments.py

@@ -1,10 +1,8 @@
-from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.postgresql import ENUM as PgEnum
 import enum
 from datetime import datetime
-
-Base = declarative_base()
+from backend.app.db.base import Base
+from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey
+from sqlalchemy.dialects.postgresql import ENUM as PgEnum
 
 
 class EnergyAssessment(Base):

backend/app/db/models/epc.py

@@ -4,11 +4,8 @@ from sqlalchemy import (
     String,
     JSON,
     TIMESTAMP,
-    UniqueConstraint,
 )
-from sqlalchemy.orm import declarative_base
-
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class EpcStore(Base):

backend/app/db/models/funding.py

@@ -3,20 +3,17 @@ import enum
 from sqlalchemy import (
     Column,
     Integer,
-    String,
     Float,
     Enum,
     TIMESTAMP,
     BigInteger,
     ForeignKey,
 )
-from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func
+from backend.app.db.base import Base
 from backend.app.db.models.recommendations import PlanModel
 from backend.app.db.models.materials import MaterialType, Material
 
-Base = declarative_base()
-
 
 class SchemeEnum(enum.Enum):
     eco4 = "eco4"

backend/app/db/models/inspections.py

@@ -9,11 +9,9 @@ from sqlalchemy import (
     Enum,
     ForeignKey,
 )
-from sqlalchemy.ext.declarative import declarative_base
+from backend.app.db.base import Base
 from backend.app.db.models.portfolio import PropertyModel
 
-Base = declarative_base()
-
 
 # -------------------------------------------------------------------
 # ENUM DEFINITIONS (equivalent to drizzle pgEnum calls)

backend/app/db/models/materials.py

@@ -1,10 +1,9 @@
 import enum
 
 from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP, Boolean
-from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func
 
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class MaterialType(enum.Enum):

backend/app/db/models/non_intrusive_surveys.py

@@ -1,7 +1,5 @@
 from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer
-from sqlalchemy.orm import declarative_base
-
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class NonIntrusiveSurvey(Base):

backend/app/db/models/portfolio.py

@@ -4,6 +4,7 @@ import datetime
 from sqlalchemy import (
     Column,
     Integer,
+    BigInteger,
     Text,
     Boolean,
     Float,
@@ -12,12 +13,10 @@ from sqlalchemy import (
     ForeignKey,
     CheckConstraint,
 )
-from sqlalchemy.ext.declarative import declarative_base
+from backend.app.db.base import Base
 from backend.app.db.models.users import UserModel  # noqa
 from backend.app.db.models.materials import MaterialType
 
-Base = declarative_base()
-
 
 class PortfolioStatus(enum.Enum):
     SCOPING = "scoping"
@@ -32,7 +31,7 @@ class PortfolioStatus(enum.Enum):
     NEEDS_REVIEW = "needs review"
 
 
-class PortfolioGoal(enum.Enum): # TODO: Move to domain?
+class PortfolioGoal(enum.Enum):  # TODO: Move to domain?
     VALUATION_IMPROVEMENT = "Valuation Improvement"
     INCREASING_EPC = "Increasing EPC"
     REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions"
@@ -116,9 +115,9 @@ class PropertyModel(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     portfolio_id = Column(Integer, ForeignKey("portfolio.id"), nullable=False)
     creation_status = Column(Enum(PropertyCreationStatus), nullable=False)
-    uprn = Column(Integer)
+    uprn = Column(BigInteger)
     landlord_property_id = Column(Text)
-    building_reference_number = Column(Integer)
+    building_reference_number = Column(BigInteger)
     status = Column(
         Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]),
         nullable=False,

backend/app/db/models/recommendations.py

@@ -1,3 +1,4 @@
+import enum
 from typing import Iterable, List, NamedTuple, Optional, Type
 from sqlalchemy import (
     Column,
@@ -9,17 +10,15 @@ from sqlalchemy import (
     ForeignKey,
     Enum,
 )
-from sqlalchemy.orm import declarative_base, Mapped, mapped_column
+from sqlalchemy.orm import Mapped, mapped_column
 from sqlalchemy.sql import func
 from datetime import datetime
 
+from backend.app.db.base import Base
 from backend.app.db.models.portfolio import Portfolio, PortfolioGoal, PropertyModel
 from backend.app.db.models.materials import Material
 from backend.app.db.models.portfolio import Epc
 from datatypes.enums import QuantityUnits
-import enum
-
-Base = declarative_base()
 
 
 def portfolio_goal_values(enum_cls: Type[PortfolioGoal]) -> List[str]:

backend/app/db/models/solar.py

@@ -2,9 +2,7 @@ import datetime
 import pytz
 from enum import Enum as PyEnum
 from sqlalchemy import Column, Integer, Float, DateTime, JSON, BigInteger, ForeignKey, Enum, Boolean
-from sqlalchemy.ext.declarative import declarative_base
-
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class Solar(Base):

backend/app/db/models/users.py

@@ -1,8 +1,6 @@
 from sqlalchemy import Column, Integer, String, DateTime
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.sql import func
-
-Base = declarative_base()
+from backend.app.db.base import Base
 
 
 class UserModel(Base):

backend/app/db/models/whlg.py

@@ -1,4 +1,3 @@
-import uuid
 from typing import Optional
 from sqlmodel import SQLModel, Field
 
@@ -12,4 +11,4 @@ class Whlg(SQLModel, table=True):
         index=True,
     )
 
-    postcode: str = Field(nullable=False)
+    postcode: str = Field(nullable=False)

backend/export/README.md

@@ -0,0 +1,155 @@
+# 🧪 Running Tests in PyCharm (macOS + pytest-postgresql)
+
+Our test suite uses `pytest` and `pytest-postgresql`, which
+automatically spins up a temporary PostgreSQL instance.
+
+On Linux (including GitHub Actions), PostgreSQL binaries are installed
+in standard system locations.\
+On macOS (Homebrew), they are not --- so PyCharm needs a small
+configuration tweak to locate `pg_ctl`.
+
+This guide explains how to run and debug tests locally in PyCharm
+without modifying test code.
+
+------------------------------------------------------------------------
+
+## ✅ Prerequisites
+
+1. Install PostgreSQL via Homebrew:
+
+``` bash
+brew install postgresql
+```
+
+2. Confirm `pg_ctl` exists:
+
+``` bash
+which pg_ctl
+```
+
+Typical output:
+
+    /opt/homebrew/bin/pg_ctl
+
+------------------------------------------------------------------------
+
+# 🚀 Running Tests in PyCharm
+
+## Step 1 --- Create a PyCharm pytest Run Configuration
+
+1. Open the test file.
+2. Click the green ▶ next to the test.
+3. Choose **"Edit Run Configuration..."**
+
+You should see something like:
+
+- **Target:** `backend/export/tests/test_export.py`
+- **Working directory:** Project root (e.g.`Model/`)
+
+------------------------------------------------------------------------
+
+## Step 2 --- Add Required Override (macOS Only)
+
+In the Run Configuration:
+
+### ➜ "Additional Arguments"
+
+Add:
+
+    --override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl
+
+This tells `pytest-postgresql` where `pg_ctl` lives on macOS.
+
+Without this, PyCharm may fail with:
+
+    ExecutableMissingException: Could not found pg_config executable
+
+------------------------------------------------------------------------
+
+## Step 3 --- Run or Debug
+
+You can now:
+
+- Click ▶ Run\
+- Click 🐞 Debug\
+- Set breakpoints normally
+
+The temporary PostgreSQL instance will start automatically.
+
+------------------------------------------------------------------------
+
+# 🔍 Why This Is Needed
+
+`pytest-postgresql` defaults to a Linux-style path:
+
+    /usr/lib/postgresql/<version>/bin/pg_ctl
+
+That path exists on Ubuntu (CI), but not on macOS.
+
+On macOS, Homebrew installs PostgreSQL in:
+
+    /opt/homebrew/bin/
+
+The `--override-ini` flag safely overrides the executable path
+**locally**, without modifying:
+
+- test files\
+- `conftest.py`\
+- `pytest.ini`\
+- CI configuration
+
+This ensures:
+
+- ✅ Tests still work in GitHub Actions\
+- ✅ Tests still work for Linux users\
+- ✅ macOS developers can debug in PyCharm\
+- ✅ No repository-specific hacks are required
+
+------------------------------------------------------------------------
+
+# 🛠 Optional: Using a Local `.env` File
+
+If you prefer not to hardcode the override in the run configuration:
+
+1. Create a local file:
+
+```{=html}
+<!-- -->
+```
+
+    .env.local
+
+2. Add:
+
+```{=html}
+<!-- -->
+```
+
+    PYTEST_ADDOPTS=--override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl
+
+3. In PyCharm:
+    - Open the Run Configuration
+    - Add `.env.local` under **"Paths to .env files"**
+
+------------------------------------------------------------------------
+
+# 🧪 Running Tests via Terminal (Recommended for CI Parity)
+
+For normal execution outside PyCharm:
+
+``` bash
+make test
+```
+
+These already work without additional configuration.
+
+------------------------------------------------------------------------
+
+# 🧠 Summary
+
+Environment Works Without Override? Needs `--override-ini`?
+  ------------------------ ------------------------- -------------------------
+GitHub Actions (Linux)   ✅ Yes ❌ No
+Linux local ✅ Yes ❌ No
+macOS terminal (tox)     ✅ Yes ❌ No
+macOS PyCharm debugger ❌ No ✅ Yes

backend/export/property_scenarios/db_functions.py

@@ -1,6 +1,5 @@
 from typing import List, Any, Dict, Optional
 import pandas as pd
-from sqlalchemy import func
 from sqlalchemy.orm import Session
 from collections import defaultdict
 
@@ -95,7 +94,10 @@ class DbMethods:
 
             plans_query = (
                 self.session.query(PlanModel)
-                .filter(PlanModel.is_default.is_(True))
+                .filter(
+                    PlanModel.portfolio_id == portfolio_id,
+                    PlanModel.is_default.is_(True)
+                )
                 .distinct(PlanModel.property_id)
                 .order_by(
                     PlanModel.property_id,
@@ -110,7 +112,10 @@ class DbMethods:
 
             plans_query = (
                 self.session.query(PlanModel)
-                .filter(PlanModel.scenario_id.in_(scenario_ids))
+                .filter(
+                    PlanModel.portfolio_id == portfolio_id,
+                    PlanModel.scenario_id.in_(scenario_ids)
+                )
                 .distinct(
                     PlanModel.scenario_id,
                     PlanModel.property_id,
@@ -138,6 +143,7 @@ class DbMethods:
     def get_recommendations(self, plan_ids: List[int]) -> pd.DataFrame:
 
         if not plan_ids:
+            logger.info("No plan ids provided")
             return pd.DataFrame()
 
         recs_query = (

backend/export/property_scenarios/main.py

@@ -1,96 +1,98 @@
 import json
-from typing import List, Optional, Any, Mapping
+from typing import Optional, Any, Mapping, Dict, Union
 
 import pandas as pd
 from sqlalchemy.orm import Session
 
 from backend.export.property_scenarios.input_schema import ExportRequest
 from backend.export.property_scenarios.db_functions import DbMethods
-from backend.app.db.connection import db_engine
+from backend.app.db.connection import db_read_session
 from backend.app.utils import sap_to_epc
 from utils.logger import setup_logger
 
 logger = setup_logger()
 
 
-def process_export(config: ExportRequest) -> List[str]:
-    exported_files: List[str] = []
+def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]:
+    export_files: Dict[Union[str, int], pd.DataFrame] = {}
 
-    with Session(bind=db_engine) as session:
+    db_methods = DbMethods(session)
 
-        db_methods = DbMethods(session)
+    properties_df = db_methods.get_properties(payload.portfolio_id)
 
-        properties_df = db_methods.get_properties(config.portfolio_id)
+    logger.info("Retrieved %s properties for export", len(properties_df))
 
-        plans_df = db_methods.get_latest_plans(
-            portfolio_id=config.portfolio_id,
-            scenario_ids=config.scenario_ids,
-            default_only=config.default_plans_only,
-        )
+    plans_df = db_methods.get_latest_plans(
+        portfolio_id=payload.portfolio_id,
+        scenario_ids=payload.scenario_ids,
+        default_only=payload.default_plans_only,
+    )
 
-        if plans_df.empty:
-            return exported_files
+    logger.info("Retrieved %s plans for export", len(plans_df))
 
-        recommendations_df = db_methods.get_recommendations(
-            plans_df["id"].tolist()
-        )
+    if plans_df.empty:
+        return export_files
 
-        recommendations_df = db_methods.attach_materials(recommendations_df)
+    recommendations_df = db_methods.get_recommendations(
+        plans_df["id"].tolist()
+    )
 
-        for scenario_id in config.scenario_ids:
+    recommendations_df = db_methods.attach_materials(recommendations_df)
 
+    if payload.default_plans_only:
+        group_keys = [None]  # Single export, no scenario grouping
+    else:
+        group_keys = payload.scenario_ids
+
+    for group_key in group_keys:
+
+        if payload.default_plans_only:
+            scenario_recs = recommendations_df
+            export_label = "default_plans"
+        else:
             scenario_recs = recommendations_df[
-                recommendations_df["scenario_id"] == scenario_id
+                recommendations_df["scenario_id"] == group_key
                 ]
+            export_label = group_key
 
-            if scenario_recs.empty:
-                continue
+        if scenario_recs.empty:
+            continue
 
-            measures_df = scenario_recs[
-                ["property_id", "measure_type", "estimated_cost"]
-            ].drop_duplicates()
+        measures_df: pd.DataFrame = scenario_recs[
+            ["property_id", "measure_type", "estimated_cost"]
+        ].drop_duplicates()
 
-            pivot = measures_df.pivot(
-                index="property_id",
-                columns="measure_type",
-                values="estimated_cost",
-            ).reset_index()
+        pivot = measures_df.pivot(
+            index="property_id",
+            columns="measure_type",
+            values="estimated_cost",
+        ).reset_index()
 
-            pivot["total_retrofit_cost"] = (
-                pivot.drop(columns=["property_id"]).sum(axis=1)
-            )
+        pivot["total_retrofit_cost"] = (
+            pivot.drop(columns=["property_id"]).sum(axis=1)
+        )
 
-            post_sap = (
-                scenario_recs.groupby("property_id")[["sap_points"]]
-                .sum()
-                .reset_index()
-            )
+        post_sap = (
+            scenario_recs.groupby("property_id")[["sap_points"]]
+            .sum()
+            .reset_index()
+        )
 
-            df = (
-                properties_df
-                .merge(pivot, how="left", on="property_id")
-                .merge(post_sap, how="left", on="property_id")
-            )
+        df = (
+            properties_df.rename(columns={"solar_pv": "existing_solar_pv"})
+            .merge(pivot, how="left", on="property_id")
+            .merge(post_sap, how="left", on="property_id")
+        )
 
-            df["sap_points"] = df["sap_points"].fillna(0)
-            df["predicted_post_works_sap"] = (
-                df["current_sap_points"] + df["sap_points"]
-            )
-            df["predicted_post_works_epc"] = df[
-                "predicted_post_works_sap"
-            ].apply(sap_to_epc)
+        df["sap_points"] = df["sap_points"].fillna(0)
+        df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
+        df["predicted_post_works_epc"] = df[
+            "predicted_post_works_sap"
+        ].apply(sap_to_epc)
 
-            filename = (
-                f"/tmp/{config.scenario_names[scenario_id]} - "
-                f"{config.project_name}.xlsx"
-            )
+        export_files[export_label] = df
 
-            with pd.ExcelWriter(filename) as writer:
-                df.to_excel(writer, sheet_name="properties", index=False)
-
-            exported_files.append(filename)
-
-    return exported_files
+    return export_files
 
 
 # ============================================================
@@ -106,22 +108,23 @@ def handler(event: dict, context: Optional[Any]) -> Mapping[str, int | str]:
     3) scenario ids - list of scenario ids to export
     4) default_plans_only - flag indicating if we should only consider default plans for export (optional,
     defaults to False)
-    :param event:
-    :param context:
-    :return:
+
+    Exxample event:
+    body_dict = {
+        "task_id": "test",
+        "subtask_id": "test",
+        "portfolio_id": 569,
+        "scenario_ids": [],
+        "default_plans_only": True,
+    }
+    :param event: Lambda event containing export request details
+    :param context: Lambda context (not used in this handler but included for completeness)
+    :return: HTTP response indicating success or failure of the export operation
     """
     for record in event.get("Records", []):
         try:
             body_dict = json.loads(record["body"])
 
-            # body_dict = {
-            #     "task_id": "test",
-            #     "subtask_id": "test",
-            #     "portfolio_id": 569,
-            #     "scenario_ids": [],
-            #     "default_plans_only": True,
-            # }
-
             logger.debug("Validating request body")
             payload = ExportRequest.model_validate(body_dict)
 
@@ -132,7 +135,8 @@ def handler(event: dict, context: Optional[Any]) -> Mapping[str, int | str]:
                 )
 
             logger.debug("Successfully validated request body")
-            process_export(payload)
+            with db_read_session() as session:
+                exported_files = process_export(payload, session)
 
             # TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url
 

backend/export/tests/conftest.py

@@ -0,0 +1,55 @@
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from backend.app.db.base import Base
+
+
+@pytest.fixture(scope="function")
+def engine(postgresql):
+    """
+    Create a SQLAlchemy engine bound to the ephemeral
+    pytest-postgresql database.
+    """
+
+    # Build SQLAlchemy URL from psycopg connection info
+    connection_string = (
+        f"postgresql+psycopg://"
+        f"{postgresql.info.user}:"
+        f"{postgresql.info.password}@"
+        f"{postgresql.info.host}:"
+        f"{postgresql.info.port}/"
+        f"{postgresql.info.dbname}"
+    )
+
+    engine = create_engine(connection_string)
+
+    # Create tables once per test session
+    Base.metadata.create_all(engine)
+
+    # Yeild will split this function into two phase. 1) setup and 2) teardown, the latter of which will run after all
+    # tests have completed
+    yield engine
+
+    # Clean-up after entire test session
+    Base.metadata.drop_all(engine)
+    engine.dispose()
+
+
+@pytest.fixture(scope="function")
+def db_session(engine):
+    """
+    Provides a clean transactional session per test.
+
+    Rolls back after each test to keep isolation.
+    """
+
+    connection = engine.connect()
+    transaction = connection.begin()
+
+    session = sessionmaker(bind=connection)()
+
+    yield session
+
+    session.close()
+    transaction.rollback()
+    connection.close()

backend/export/tests/test_export.py

@@ -0,0 +1,274 @@
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import time
+
+from backend.export.property_scenarios.main import process_export
+from backend.export.property_scenarios.input_schema import ExportRequest
+from backend.app.db.models.portfolio import PropertyModel, Epc, Portfolio, PortfolioStatus, PortfolioGoal, \
+    PropertyCreationStatus, PropertyDetailsEpcModel
+from backend.app.db.models.recommendations import PlanModel, Recommendation, PlanRecommendations
+from utils.logger import setup_logger
+
+FIXTURE_PATH = Path("backend/export/tests/fixtures")
+logger = setup_logger()
+
+
+def load_csv(name: str) -> pd.DataFrame:
+    df = pd.read_csv(FIXTURE_PATH / name)
+    df = df.replace({np.nan: None})
+    return df
+
+
+def test_default_export_integration(db_session):
+    # ----------------------------------------
+    # 1) Load csvs
+    # ----------------------------------------
+    t0 = time.perf_counter()
+    portfolio_df = load_csv("portfolio_569.csv")
+    properties_df = load_csv("properties_569.csv")
+    property_details_epc_df = load_csv("property_details_epc_569.csv")
+    plans_df = load_csv("plans_569.csv")
+    plan_recs_df = load_csv("plan_recs_569.csv")
+    recommendations_df = load_csv("recommendations_569.csv")
+
+    # Shrink down recommendations_df to speed up the data load. For this test, we only need
+    # default recommendations so let's focus on those. We filter on where default is true
+    recommendations_df = recommendations_df[
+        recommendations_df["default"]
+    ]
+    valid_rec_ids = recommendations_df["id"].unique()
+
+    plan_recs_df = plan_recs_df[
+        plan_recs_df["recommendation_id"].isin(valid_rec_ids)
+    ]
+
+    logger.info(
+        "Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s",
+        time.perf_counter() - t0,
+        len(properties_df),
+        len(plans_df),
+        len(recommendations_df),
+    )
+
+    logger.info("Starting database load")
+    db_load_t0 = time.perf_counter()
+
+    # ----------------------------------------
+    # 2) Insert test portfolio
+    # ----------------------------------------
+
+    portfolios = []
+    for row in portfolio_df.itertuples(index=False):
+        portfolios.append(
+            Portfolio(
+                id=row.id,
+                name=row.name,
+                status=PortfolioStatus[row.status.split(".")[-1]],
+                goal=PortfolioGoal[row.goal.split(".")[-1]] if row.goal else None,
+            )
+        )
+
+    db_session.bulk_save_objects(portfolios)
+    db_session.flush()
+    # ----------------------------------------
+    # 3) Insert test property
+    # ----------------------------------------
+
+    properties = []
+
+    for row in properties_df.itertuples(index=False):
+        row_dict = row._asdict()
+
+        row_dict["uprn"] = int(row_dict["uprn"]) if row_dict.get("uprn") else None
+        row_dict["building_reference_number"] = (
+            int(row_dict["building_reference_number"])
+            if row_dict.get("building_reference_number")
+            else None
+        )
+
+        prop = PropertyModel(**{
+            col: row_dict[col]
+            for col in PropertyModel.__table__.columns.keys()
+            if col in row_dict
+        })
+
+        prop.creation_status = PropertyCreationStatus[
+            row_dict["creation_status"].split(".")[-1]
+        ]
+        prop.status = PortfolioStatus[row_dict["status"].split(".")[-1]]
+
+        if row_dict.get("current_epc_rating"):
+            prop.current_epc_rating = Epc[
+                row_dict["current_epc_rating"].split(".")[-1]
+            ]
+
+        properties.append(prop)
+
+    db_session.bulk_save_objects(properties)
+    db_session.flush()
+
+    # ----------------------------------------
+    # 4) Insert property details - EPC
+    # ----------------------------------------
+
+    property_lookup = {
+        prop.uprn: prop
+        for prop in db_session.query(PropertyModel).all()
+    }
+
+    epc_rows = []
+
+    for row in property_details_epc_df.itertuples(index=False):
+        row_dict = row._asdict()
+
+        uprn = int(row_dict["uprn"]) if row_dict.get("uprn") else None
+        property_obj = property_lookup.get(uprn)
+
+        if not property_obj:
+            continue  # skip if property not found
+
+        # Build only fields that exist on the model
+        epc_data = {
+            col.name: row_dict[col.name]
+            for col in PropertyDetailsEpcModel.__table__.columns
+            if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"]
+        }
+
+        epc = PropertyDetailsEpcModel(
+            property_id=property_obj.id,
+            portfolio_id=property_obj.portfolio_id,
+            **epc_data,
+        )
+
+        epc_rows.append(epc)
+
+    db_session.bulk_save_objects(epc_rows)
+    db_session.flush()
+
+    # ----------------------------------------
+    # 4) Insert default plan
+    # ----------------------------------------
+
+    plans = []
+
+    for row in plans_df.itertuples(index=False):
+        row_dict = row._asdict()
+
+        if row_dict.get("post_epc_rating"):
+            row_dict["post_epc_rating"] = Epc[
+                row_dict["post_epc_rating"].split(".")[-1]
+            ]
+
+        row_dict["scenario_id"] = None
+
+        plan = PlanModel(**{
+            col: row_dict[col]
+            for col in PlanModel.__table__.columns.keys()
+            if col in row_dict
+        })
+
+        plans.append(plan)
+
+    db_session.bulk_save_objects(plans)
+    db_session.flush()
+
+    # ----------------------------------------
+    # 5) Insert recommendation
+    # ----------------------------------------
+
+    recs = [
+        Recommendation(**{
+            col: row[col]
+            for col in Recommendation.__table__.columns.keys()
+            if col in row
+        })
+        for _, row in recommendations_df.iterrows()
+    ]
+
+    db_session.bulk_save_objects(recs)
+    db_session.flush()
+
+    # ----------------------------------------
+    # 6) Insert PlanRecommendations
+    # ----------------------------------------
+    links = [
+        PlanRecommendations(
+            plan_id=row.plan_id,
+            recommendation_id=row.recommendation_id,
+        )
+        for row in plan_recs_df.itertuples(index=False)
+    ]
+
+    db_session.bulk_save_objects(links)
+    db_session.commit()
+    logger.info("Inserted all data in %.2f seconds", time.perf_counter() - db_load_t0)
+
+    # ----------------------------------------
+    # 6) Build payload
+    # ----------------------------------------
+
+    body_dict = {
+        "task_id": "test",
+        "subtask_id": "test",
+        "portfolio_id": 569,
+        "scenario_ids": [],
+        "default_plans_only": True,
+    }
+
+    payload = ExportRequest.model_validate(body_dict)
+
+    # ----------------------------------------
+    # 7) Call process_export
+    # ----------------------------------------
+
+    logger.info(
+        "Recommendation count in DB: %s",
+        db_session.query(Recommendation).count()
+    )
+
+    logger.info(
+        "Default + not installed count: %s",
+        db_session.query(Recommendation)
+        .filter(
+            Recommendation.default.is_(True),
+            Recommendation.already_installed.is_(False)
+        )
+        .count()
+    )
+
+    logger.info("Starting process_export")
+    process_t0 = time.perf_counter()
+
+    result = process_export(payload, session=db_session)
+
+    logger.info("process_export finished in %.2f seconds", time.perf_counter() - process_t0)
+
+    # ----------------------------------------
+    # 8) Assertions
+    # ----------------------------------------
+
+    assert "default_plans" in result
+
+    df = result["default_plans"]
+
+    assert not df.empty
+
+    # This test was generated on a real portfolio and so we check the things we expect to do
+
+    # 1) All packages are "compliant", where in this case, the properties should get to EPC C
+
+    failed = df[df["predicted_post_works_sap"] < 69]
+    failed_property_types = failed["property_type"].value_counts().to_dict()
+    assert failed_property_types["Flat"] == 113
+    assert failed_property_types["House"] == 8
+    assert failed_property_types["Bungalow"] == 4
+    assert failed_property_types["Maisonette"] == 1
+    # Check the houses
+
+    assert failed.shape[0]
+
+    # Errors for me:
+    # - should get to EPC C: https://ara.domna.homes/portfolio/569/building-passport/661051/plans
+    # - Why doesn't this get to a C, under the plan?:
+    # https://ara.domna.homes/portfolio/569/building-passport/660447/plans/1603913

pytest.ini

@@ -1,4 +1,6 @@
 [pytest]
 pythonpath = .
+log_cli = true
+log_cli_level = INFO
 addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial
 testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests

test.requirements.txt

@@ -2,4 +2,6 @@ pytest
 mock
 pytest-cov
 pytest-mock
-dotenv
+dotenv
+psycopg[binary]
+pytest-postgresql