diff --git a/.idea/Model.iml b/.idea/Model.iml index c6561970..1e51ede4 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -10,4 +10,7 @@ + + \ No newline at end of file diff --git a/backend/app/db/base.py b/backend/app/db/base.py new file mode 100644 index 00000000..59be7030 --- /dev/null +++ b/backend/app/db/base.py @@ -0,0 +1,3 @@ +from sqlalchemy.orm import declarative_base + +Base = declarative_base() diff --git a/backend/app/db/models/addresses.py b/backend/app/db/models/addresses.py index 51e9540f..a813f58d 100644 --- a/backend/app/db/models/addresses.py +++ b/backend/app/db/models/addresses.py @@ -7,9 +7,7 @@ from sqlalchemy import ( func, UniqueConstraint, ) -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class PostcodeSearch(Base): diff --git a/backend/app/db/models/condition.py b/backend/app/db/models/condition.py index 77043366..96f601a7 100644 --- a/backend/app/db/models/condition.py +++ b/backend/app/db/models/condition.py @@ -7,12 +7,12 @@ from sqlalchemy import ( String, Enum as SqlEnum, ) -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import relationship from backend.condition.domain.aspect_type import AspectType from backend.condition.domain.element_type import ElementType -Base = declarative_base() +from backend.app.db.base import Base ElementTypeDb = SqlEnum( ElementType, diff --git a/backend/app/db/models/energy_assessments.py b/backend/app/db/models/energy_assessments.py index 46912c9b..65879c39 100644 --- a/backend/app/db/models/energy_assessments.py +++ b/backend/app/db/models/energy_assessments.py @@ -1,10 +1,8 @@ -from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.dialects.postgresql import ENUM as PgEnum import enum from datetime import datetime - -Base = declarative_base() +from backend.app.db.base import Base +from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey +from sqlalchemy.dialects.postgresql import ENUM as PgEnum class EnergyAssessment(Base): diff --git a/backend/app/db/models/epc.py b/backend/app/db/models/epc.py index 5a216040..ff0b40a0 100644 --- a/backend/app/db/models/epc.py +++ b/backend/app/db/models/epc.py @@ -4,11 +4,8 @@ from sqlalchemy import ( String, JSON, TIMESTAMP, - UniqueConstraint, ) -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class EpcStore(Base): diff --git a/backend/app/db/models/funding.py b/backend/app/db/models/funding.py index a7417e14..19e8203d 100644 --- a/backend/app/db/models/funding.py +++ b/backend/app/db/models/funding.py @@ -3,20 +3,17 @@ import enum from sqlalchemy import ( Column, Integer, - String, Float, Enum, TIMESTAMP, BigInteger, ForeignKey, ) -from sqlalchemy.orm import declarative_base from sqlalchemy.sql import func +from backend.app.db.base import Base from backend.app.db.models.recommendations import PlanModel from backend.app.db.models.materials import MaterialType, Material -Base = declarative_base() - class SchemeEnum(enum.Enum): eco4 = "eco4" diff --git a/backend/app/db/models/inspections.py b/backend/app/db/models/inspections.py index 473f8a02..2a42f589 100644 --- a/backend/app/db/models/inspections.py +++ b/backend/app/db/models/inspections.py @@ -9,11 +9,9 @@ from sqlalchemy import ( Enum, ForeignKey, ) -from sqlalchemy.ext.declarative import declarative_base +from backend.app.db.base import Base from backend.app.db.models.portfolio import PropertyModel -Base = declarative_base() - # ------------------------------------------------------------------- # ENUM DEFINITIONS (equivalent to drizzle pgEnum calls) diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 8a524491..101ac021 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -1,10 +1,9 @@ import enum from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP, Boolean -from sqlalchemy.orm import declarative_base from sqlalchemy.sql import func -Base = declarative_base() +from backend.app.db.base import Base class MaterialType(enum.Enum): diff --git a/backend/app/db/models/non_intrusive_surveys.py b/backend/app/db/models/non_intrusive_surveys.py index bc2d8adc..bbfb7a54 100644 --- a/backend/app/db/models/non_intrusive_surveys.py +++ b/backend/app/db/models/non_intrusive_surveys.py @@ -1,7 +1,5 @@ from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class NonIntrusiveSurvey(Base): diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index f6a99a97..9eb26597 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -4,6 +4,7 @@ import datetime from sqlalchemy import ( Column, Integer, + BigInteger, Text, Boolean, Float, @@ -12,12 +13,10 @@ from sqlalchemy import ( ForeignKey, CheckConstraint, ) -from sqlalchemy.ext.declarative import declarative_base +from backend.app.db.base import Base from backend.app.db.models.users import UserModel # noqa from backend.app.db.models.materials import MaterialType -Base = declarative_base() - class PortfolioStatus(enum.Enum): SCOPING = "scoping" @@ -32,7 +31,7 @@ class PortfolioStatus(enum.Enum): NEEDS_REVIEW = "needs review" -class PortfolioGoal(enum.Enum): # TODO: Move to domain? +class PortfolioGoal(enum.Enum): # TODO: Move to domain? VALUATION_IMPROVEMENT = "Valuation Improvement" INCREASING_EPC = "Increasing EPC" REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions" @@ -116,9 +115,9 @@ class PropertyModel(Base): id = Column(Integer, primary_key=True, autoincrement=True) portfolio_id = Column(Integer, ForeignKey("portfolio.id"), nullable=False) creation_status = Column(Enum(PropertyCreationStatus), nullable=False) - uprn = Column(Integer) + uprn = Column(BigInteger) landlord_property_id = Column(Text) - building_reference_number = Column(Integer) + building_reference_number = Column(BigInteger) status = Column( Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]), nullable=False, diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index 538b11e3..9352eeb2 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -1,3 +1,4 @@ +import enum from typing import Iterable, List, NamedTuple, Optional, Type from sqlalchemy import ( Column, @@ -9,17 +10,15 @@ from sqlalchemy import ( ForeignKey, Enum, ) -from sqlalchemy.orm import declarative_base, Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql import func from datetime import datetime +from backend.app.db.base import Base from backend.app.db.models.portfolio import Portfolio, PortfolioGoal, PropertyModel from backend.app.db.models.materials import Material from backend.app.db.models.portfolio import Epc from datatypes.enums import QuantityUnits -import enum - -Base = declarative_base() def portfolio_goal_values(enum_cls: Type[PortfolioGoal]) -> List[str]: diff --git a/backend/app/db/models/solar.py b/backend/app/db/models/solar.py index 88372bd3..dc1846f3 100644 --- a/backend/app/db/models/solar.py +++ b/backend/app/db/models/solar.py @@ -2,9 +2,7 @@ import datetime import pytz from enum import Enum as PyEnum from sqlalchemy import Column, Integer, Float, DateTime, JSON, BigInteger, ForeignKey, Enum, Boolean -from sqlalchemy.ext.declarative import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class Solar(Base): diff --git a/backend/app/db/models/users.py b/backend/app/db/models/users.py index 6e243815..7952b9b7 100644 --- a/backend/app/db/models/users.py +++ b/backend/app/db/models/users.py @@ -1,8 +1,6 @@ from sqlalchemy import Column, Integer, String, DateTime -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import func - -Base = declarative_base() +from backend.app.db.base import Base class UserModel(Base): diff --git a/backend/app/db/models/whlg.py b/backend/app/db/models/whlg.py index 29d907e4..5c5b7172 100644 --- a/backend/app/db/models/whlg.py +++ b/backend/app/db/models/whlg.py @@ -1,4 +1,3 @@ -import uuid from typing import Optional from sqlmodel import SQLModel, Field @@ -12,4 +11,4 @@ class Whlg(SQLModel, table=True): index=True, ) - postcode: str = Field(nullable=False) \ No newline at end of file + postcode: str = Field(nullable=False) diff --git a/backend/export/README.md b/backend/export/README.md new file mode 100644 index 00000000..a98154fc --- /dev/null +++ b/backend/export/README.md @@ -0,0 +1,155 @@ +# ๐Ÿงช Running Tests in PyCharm (macOS + pytest-postgresql) + +Our test suite uses `pytest` and `pytest-postgresql`, which +automatically spins up a temporary PostgreSQL instance. + +On Linux (including GitHub Actions), PostgreSQL binaries are installed +in standard system locations.\ +On macOS (Homebrew), they are not --- so PyCharm needs a small +configuration tweak to locate `pg_ctl`. + +This guide explains how to run and debug tests locally in PyCharm +without modifying test code. + +------------------------------------------------------------------------ + +## โœ… Prerequisites + +1. Install PostgreSQL via Homebrew: + +``` bash +brew install postgresql +``` + +2. Confirm `pg_ctl` exists: + +``` bash +which pg_ctl +``` + +Typical output: + + /opt/homebrew/bin/pg_ctl + +------------------------------------------------------------------------ + +# ๐Ÿš€ Running Tests in PyCharm + +## Step 1 --- Create a PyCharm pytest Run Configuration + +1. Open the test file. +2. Click the green โ–ถ next to the test. +3. Choose **"Edit Run Configuration..."** + +You should see something like: + +- **Target:** `backend/export/tests/test_export.py` +- **Working directory:** Project root (e.g.`Model/`) + +------------------------------------------------------------------------ + +## Step 2 --- Add Required Override (macOS Only) + +In the Run Configuration: + +### โžœ "Additional Arguments" + +Add: + + --override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl + +This tells `pytest-postgresql` where `pg_ctl` lives on macOS. + +Without this, PyCharm may fail with: + + ExecutableMissingException: Could not found pg_config executable + +------------------------------------------------------------------------ + +## Step 3 --- Run or Debug + +You can now: + +- Click โ–ถ Run\ +- Click ๐Ÿž Debug\ +- Set breakpoints normally + +The temporary PostgreSQL instance will start automatically. + +------------------------------------------------------------------------ + +# ๐Ÿ” Why This Is Needed + +`pytest-postgresql` defaults to a Linux-style path: + + /usr/lib/postgresql//bin/pg_ctl + +That path exists on Ubuntu (CI), but not on macOS. + +On macOS, Homebrew installs PostgreSQL in: + + /opt/homebrew/bin/ + +The `--override-ini` flag safely overrides the executable path +**locally**, without modifying: + +- test files\ +- `conftest.py`\ +- `pytest.ini`\ +- CI configuration + +This ensures: + +- โœ… Tests still work in GitHub Actions\ +- โœ… Tests still work for Linux users\ +- โœ… macOS developers can debug in PyCharm\ +- โœ… No repository-specific hacks are required + +------------------------------------------------------------------------ + +# ๐Ÿ›  Optional: Using a Local `.env` File + +If you prefer not to hardcode the override in the run configuration: + +1. Create a local file: + +```{=html} + +``` + + .env.local + +2. Add: + +```{=html} + +``` + + PYTEST_ADDOPTS=--override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl + +3. In PyCharm: + - Open the Run Configuration + - Add `.env.local` under **"Paths to .env files"** + +------------------------------------------------------------------------ + +# ๐Ÿงช Running Tests via Terminal (Recommended for CI Parity) + +For normal execution outside PyCharm: + +``` bash +make test +``` + +These already work without additional configuration. + +------------------------------------------------------------------------ + +# ๐Ÿง  Summary + +Environment Works Without Override? Needs `--override-ini`? + ------------------------ ------------------------- ------------------------- +GitHub Actions (Linux) โœ… Yes โŒ No +Linux local โœ… Yes โŒ No +macOS terminal (tox) โœ… Yes โŒ No +macOS PyCharm debugger โŒ No โœ… Yes diff --git a/backend/export/property_scenarios/db_functions.py b/backend/export/property_scenarios/db_functions.py index f527e738..8b29ab0e 100644 --- a/backend/export/property_scenarios/db_functions.py +++ b/backend/export/property_scenarios/db_functions.py @@ -1,6 +1,5 @@ from typing import List, Any, Dict, Optional import pandas as pd -from sqlalchemy import func from sqlalchemy.orm import Session from collections import defaultdict @@ -95,7 +94,10 @@ class DbMethods: plans_query = ( self.session.query(PlanModel) - .filter(PlanModel.is_default.is_(True)) + .filter( + PlanModel.portfolio_id == portfolio_id, + PlanModel.is_default.is_(True) + ) .distinct(PlanModel.property_id) .order_by( PlanModel.property_id, @@ -110,7 +112,10 @@ class DbMethods: plans_query = ( self.session.query(PlanModel) - .filter(PlanModel.scenario_id.in_(scenario_ids)) + .filter( + PlanModel.portfolio_id == portfolio_id, + PlanModel.scenario_id.in_(scenario_ids) + ) .distinct( PlanModel.scenario_id, PlanModel.property_id, @@ -138,6 +143,7 @@ class DbMethods: def get_recommendations(self, plan_ids: List[int]) -> pd.DataFrame: if not plan_ids: + logger.info("No plan ids provided") return pd.DataFrame() recs_query = ( diff --git a/backend/export/property_scenarios/main.py b/backend/export/property_scenarios/main.py index 88ebf326..d2d89916 100644 --- a/backend/export/property_scenarios/main.py +++ b/backend/export/property_scenarios/main.py @@ -1,96 +1,98 @@ import json -from typing import List, Optional, Any, Mapping +from typing import Optional, Any, Mapping, Dict, Union import pandas as pd from sqlalchemy.orm import Session from backend.export.property_scenarios.input_schema import ExportRequest from backend.export.property_scenarios.db_functions import DbMethods -from backend.app.db.connection import db_engine +from backend.app.db.connection import db_read_session from backend.app.utils import sap_to_epc from utils.logger import setup_logger logger = setup_logger() -def process_export(config: ExportRequest) -> List[str]: - exported_files: List[str] = [] +def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]: + export_files: Dict[Union[str, int], pd.DataFrame] = {} - with Session(bind=db_engine) as session: + db_methods = DbMethods(session) - db_methods = DbMethods(session) + properties_df = db_methods.get_properties(payload.portfolio_id) - properties_df = db_methods.get_properties(config.portfolio_id) + logger.info("Retrieved %s properties for export", len(properties_df)) - plans_df = db_methods.get_latest_plans( - portfolio_id=config.portfolio_id, - scenario_ids=config.scenario_ids, - default_only=config.default_plans_only, - ) + plans_df = db_methods.get_latest_plans( + portfolio_id=payload.portfolio_id, + scenario_ids=payload.scenario_ids, + default_only=payload.default_plans_only, + ) - if plans_df.empty: - return exported_files + logger.info("Retrieved %s plans for export", len(plans_df)) - recommendations_df = db_methods.get_recommendations( - plans_df["id"].tolist() - ) + if plans_df.empty: + return export_files - recommendations_df = db_methods.attach_materials(recommendations_df) + recommendations_df = db_methods.get_recommendations( + plans_df["id"].tolist() + ) - for scenario_id in config.scenario_ids: + recommendations_df = db_methods.attach_materials(recommendations_df) + if payload.default_plans_only: + group_keys = [None] # Single export, no scenario grouping + else: + group_keys = payload.scenario_ids + + for group_key in group_keys: + + if payload.default_plans_only: + scenario_recs = recommendations_df + export_label = "default_plans" + else: scenario_recs = recommendations_df[ - recommendations_df["scenario_id"] == scenario_id + recommendations_df["scenario_id"] == group_key ] + export_label = group_key - if scenario_recs.empty: - continue + if scenario_recs.empty: + continue - measures_df = scenario_recs[ - ["property_id", "measure_type", "estimated_cost"] - ].drop_duplicates() + measures_df: pd.DataFrame = scenario_recs[ + ["property_id", "measure_type", "estimated_cost"] + ].drop_duplicates() - pivot = measures_df.pivot( - index="property_id", - columns="measure_type", - values="estimated_cost", - ).reset_index() + pivot = measures_df.pivot( + index="property_id", + columns="measure_type", + values="estimated_cost", + ).reset_index() - pivot["total_retrofit_cost"] = ( - pivot.drop(columns=["property_id"]).sum(axis=1) - ) + pivot["total_retrofit_cost"] = ( + pivot.drop(columns=["property_id"]).sum(axis=1) + ) - post_sap = ( - scenario_recs.groupby("property_id")[["sap_points"]] - .sum() - .reset_index() - ) + post_sap = ( + scenario_recs.groupby("property_id")[["sap_points"]] + .sum() + .reset_index() + ) - df = ( - properties_df - .merge(pivot, how="left", on="property_id") - .merge(post_sap, how="left", on="property_id") - ) + df = ( + properties_df.rename(columns={"solar_pv": "existing_solar_pv"}) + .merge(pivot, how="left", on="property_id") + .merge(post_sap, how="left", on="property_id") + ) - df["sap_points"] = df["sap_points"].fillna(0) - df["predicted_post_works_sap"] = ( - df["current_sap_points"] + df["sap_points"] - ) - df["predicted_post_works_epc"] = df[ - "predicted_post_works_sap" - ].apply(sap_to_epc) + df["sap_points"] = df["sap_points"].fillna(0) + df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] + df["predicted_post_works_epc"] = df[ + "predicted_post_works_sap" + ].apply(sap_to_epc) - filename = ( - f"/tmp/{config.scenario_names[scenario_id]} - " - f"{config.project_name}.xlsx" - ) + export_files[export_label] = df - with pd.ExcelWriter(filename) as writer: - df.to_excel(writer, sheet_name="properties", index=False) - - exported_files.append(filename) - - return exported_files + return export_files # ============================================================ @@ -106,22 +108,23 @@ def handler(event: dict, context: Optional[Any]) -> Mapping[str, int | str]: 3) scenario ids - list of scenario ids to export 4) default_plans_only - flag indicating if we should only consider default plans for export (optional, defaults to False) - :param event: - :param context: - :return: + + Exxample event: + body_dict = { + "task_id": "test", + "subtask_id": "test", + "portfolio_id": 569, + "scenario_ids": [], + "default_plans_only": True, + } + :param event: Lambda event containing export request details + :param context: Lambda context (not used in this handler but included for completeness) + :return: HTTP response indicating success or failure of the export operation """ for record in event.get("Records", []): try: body_dict = json.loads(record["body"]) - # body_dict = { - # "task_id": "test", - # "subtask_id": "test", - # "portfolio_id": 569, - # "scenario_ids": [], - # "default_plans_only": True, - # } - logger.debug("Validating request body") payload = ExportRequest.model_validate(body_dict) @@ -132,7 +135,8 @@ def handler(event: dict, context: Optional[Any]) -> Mapping[str, int | str]: ) logger.debug("Successfully validated request body") - process_export(payload) + with db_read_session() as session: + exported_files = process_export(payload, session) # TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url diff --git a/backend/export/tests/conftest.py b/backend/export/tests/conftest.py new file mode 100644 index 00000000..10bfa971 --- /dev/null +++ b/backend/export/tests/conftest.py @@ -0,0 +1,55 @@ +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from backend.app.db.base import Base + + +@pytest.fixture(scope="function") +def engine(postgresql): + """ + Create a SQLAlchemy engine bound to the ephemeral + pytest-postgresql database. + """ + + # Build SQLAlchemy URL from psycopg connection info + connection_string = ( + f"postgresql+psycopg://" + f"{postgresql.info.user}:" + f"{postgresql.info.password}@" + f"{postgresql.info.host}:" + f"{postgresql.info.port}/" + f"{postgresql.info.dbname}" + ) + + engine = create_engine(connection_string) + + # Create tables once per test session + Base.metadata.create_all(engine) + + # Yeild will split this function into two phase. 1) setup and 2) teardown, the latter of which will run after all + # tests have completed + yield engine + + # Clean-up after entire test session + Base.metadata.drop_all(engine) + engine.dispose() + + +@pytest.fixture(scope="function") +def db_session(engine): + """ + Provides a clean transactional session per test. + + Rolls back after each test to keep isolation. + """ + + connection = engine.connect() + transaction = connection.begin() + + session = sessionmaker(bind=connection)() + + yield session + + session.close() + transaction.rollback() + connection.close() diff --git a/backend/export/tests/test_export.py b/backend/export/tests/test_export.py new file mode 100644 index 00000000..eb82333d --- /dev/null +++ b/backend/export/tests/test_export.py @@ -0,0 +1,274 @@ +import pandas as pd +import numpy as np +from pathlib import Path +import time + +from backend.export.property_scenarios.main import process_export +from backend.export.property_scenarios.input_schema import ExportRequest +from backend.app.db.models.portfolio import PropertyModel, Epc, Portfolio, PortfolioStatus, PortfolioGoal, \ + PropertyCreationStatus, PropertyDetailsEpcModel +from backend.app.db.models.recommendations import PlanModel, Recommendation, PlanRecommendations +from utils.logger import setup_logger + +FIXTURE_PATH = Path("backend/export/tests/fixtures") +logger = setup_logger() + + +def load_csv(name: str) -> pd.DataFrame: + df = pd.read_csv(FIXTURE_PATH / name) + df = df.replace({np.nan: None}) + return df + + +def test_default_export_integration(db_session): + # ---------------------------------------- + # 1) Load csvs + # ---------------------------------------- + t0 = time.perf_counter() + portfolio_df = load_csv("portfolio_569.csv") + properties_df = load_csv("properties_569.csv") + property_details_epc_df = load_csv("property_details_epc_569.csv") + plans_df = load_csv("plans_569.csv") + plan_recs_df = load_csv("plan_recs_569.csv") + recommendations_df = load_csv("recommendations_569.csv") + + # Shrink down recommendations_df to speed up the data load. For this test, we only need + # default recommendations so let's focus on those. We filter on where default is true + recommendations_df = recommendations_df[ + recommendations_df["default"] + ] + valid_rec_ids = recommendations_df["id"].unique() + + plan_recs_df = plan_recs_df[ + plan_recs_df["recommendation_id"].isin(valid_rec_ids) + ] + + logger.info( + "Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s", + time.perf_counter() - t0, + len(properties_df), + len(plans_df), + len(recommendations_df), + ) + + logger.info("Starting database load") + db_load_t0 = time.perf_counter() + + # ---------------------------------------- + # 2) Insert test portfolio + # ---------------------------------------- + + portfolios = [] + for row in portfolio_df.itertuples(index=False): + portfolios.append( + Portfolio( + id=row.id, + name=row.name, + status=PortfolioStatus[row.status.split(".")[-1]], + goal=PortfolioGoal[row.goal.split(".")[-1]] if row.goal else None, + ) + ) + + db_session.bulk_save_objects(portfolios) + db_session.flush() + # ---------------------------------------- + # 3) Insert test property + # ---------------------------------------- + + properties = [] + + for row in properties_df.itertuples(index=False): + row_dict = row._asdict() + + row_dict["uprn"] = int(row_dict["uprn"]) if row_dict.get("uprn") else None + row_dict["building_reference_number"] = ( + int(row_dict["building_reference_number"]) + if row_dict.get("building_reference_number") + else None + ) + + prop = PropertyModel(**{ + col: row_dict[col] + for col in PropertyModel.__table__.columns.keys() + if col in row_dict + }) + + prop.creation_status = PropertyCreationStatus[ + row_dict["creation_status"].split(".")[-1] + ] + prop.status = PortfolioStatus[row_dict["status"].split(".")[-1]] + + if row_dict.get("current_epc_rating"): + prop.current_epc_rating = Epc[ + row_dict["current_epc_rating"].split(".")[-1] + ] + + properties.append(prop) + + db_session.bulk_save_objects(properties) + db_session.flush() + + # ---------------------------------------- + # 4) Insert property details - EPC + # ---------------------------------------- + + property_lookup = { + prop.uprn: prop + for prop in db_session.query(PropertyModel).all() + } + + epc_rows = [] + + for row in property_details_epc_df.itertuples(index=False): + row_dict = row._asdict() + + uprn = int(row_dict["uprn"]) if row_dict.get("uprn") else None + property_obj = property_lookup.get(uprn) + + if not property_obj: + continue # skip if property not found + + # Build only fields that exist on the model + epc_data = { + col.name: row_dict[col.name] + for col in PropertyDetailsEpcModel.__table__.columns + if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"] + } + + epc = PropertyDetailsEpcModel( + property_id=property_obj.id, + portfolio_id=property_obj.portfolio_id, + **epc_data, + ) + + epc_rows.append(epc) + + db_session.bulk_save_objects(epc_rows) + db_session.flush() + + # ---------------------------------------- + # 4) Insert default plan + # ---------------------------------------- + + plans = [] + + for row in plans_df.itertuples(index=False): + row_dict = row._asdict() + + if row_dict.get("post_epc_rating"): + row_dict["post_epc_rating"] = Epc[ + row_dict["post_epc_rating"].split(".")[-1] + ] + + row_dict["scenario_id"] = None + + plan = PlanModel(**{ + col: row_dict[col] + for col in PlanModel.__table__.columns.keys() + if col in row_dict + }) + + plans.append(plan) + + db_session.bulk_save_objects(plans) + db_session.flush() + + # ---------------------------------------- + # 5) Insert recommendation + # ---------------------------------------- + + recs = [ + Recommendation(**{ + col: row[col] + for col in Recommendation.__table__.columns.keys() + if col in row + }) + for _, row in recommendations_df.iterrows() + ] + + db_session.bulk_save_objects(recs) + db_session.flush() + + # ---------------------------------------- + # 6) Insert PlanRecommendations + # ---------------------------------------- + links = [ + PlanRecommendations( + plan_id=row.plan_id, + recommendation_id=row.recommendation_id, + ) + for row in plan_recs_df.itertuples(index=False) + ] + + db_session.bulk_save_objects(links) + db_session.commit() + logger.info("Inserted all data in %.2f seconds", time.perf_counter() - db_load_t0) + + # ---------------------------------------- + # 6) Build payload + # ---------------------------------------- + + body_dict = { + "task_id": "test", + "subtask_id": "test", + "portfolio_id": 569, + "scenario_ids": [], + "default_plans_only": True, + } + + payload = ExportRequest.model_validate(body_dict) + + # ---------------------------------------- + # 7) Call process_export + # ---------------------------------------- + + logger.info( + "Recommendation count in DB: %s", + db_session.query(Recommendation).count() + ) + + logger.info( + "Default + not installed count: %s", + db_session.query(Recommendation) + .filter( + Recommendation.default.is_(True), + Recommendation.already_installed.is_(False) + ) + .count() + ) + + logger.info("Starting process_export") + process_t0 = time.perf_counter() + + result = process_export(payload, session=db_session) + + logger.info("process_export finished in %.2f seconds", time.perf_counter() - process_t0) + + # ---------------------------------------- + # 8) Assertions + # ---------------------------------------- + + assert "default_plans" in result + + df = result["default_plans"] + + assert not df.empty + + # This test was generated on a real portfolio and so we check the things we expect to do + + # 1) All packages are "compliant", where in this case, the properties should get to EPC C + + failed = df[df["predicted_post_works_sap"] < 69] + failed_property_types = failed["property_type"].value_counts().to_dict() + assert failed_property_types["Flat"] == 113 + assert failed_property_types["House"] == 8 + assert failed_property_types["Bungalow"] == 4 + assert failed_property_types["Maisonette"] == 1 + # Check the houses + + assert failed.shape[0] + + # Errors for me: + # - should get to EPC C: https://ara.domna.homes/portfolio/569/building-passport/661051/plans + # - Why doesn't this get to a C, under the plan?: + # https://ara.domna.homes/portfolio/569/building-passport/660447/plans/1603913 diff --git a/pytest.ini b/pytest.ini index 9c9f8234..7bef3884 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,6 @@ [pytest] pythonpath = . +log_cli = true +log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests diff --git a/test.requirements.txt b/test.requirements.txt index d31371a6..d8b8b777 100644 --- a/test.requirements.txt +++ b/test.requirements.txt @@ -2,4 +2,6 @@ pytest mock pytest-cov pytest-mock -dotenv \ No newline at end of file +dotenv +psycopg[binary] +pytest-postgresql \ No newline at end of file