mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
540 lines
21 KiB
Python
540 lines
21 KiB
Python
import pandas as pd
|
|
import numpy as np
|
|
from pathlib import Path
|
|
import time
|
|
|
|
from backend.export.property_scenarios.main import process_export
|
|
from backend.export.property_scenarios.input_schema import ExportRequest
|
|
from backend.app.db.models.portfolio import PropertyModel, Epc, Portfolio, PortfolioStatus, PortfolioGoal, \
|
|
PropertyCreationStatus, PropertyDetailsEpcModel
|
|
from backend.app.db.models.recommendations import PlanModel, Recommendation, PlanRecommendations, \
|
|
RecommendationMaterials
|
|
from backend.app.db.models.materials import Material
|
|
from utils.logger import setup_logger
|
|
|
|
FIXTURE_PATH = Path("backend/export/tests/fixtures")
|
|
logger = setup_logger()
|
|
|
|
|
|
def load_csv(name: str) -> pd.DataFrame:
|
|
df = pd.read_csv(FIXTURE_PATH / name)
|
|
df = df.replace({np.nan: None})
|
|
return df
|
|
|
|
|
|
def test_default_export_integration(db_session):
|
|
# ----------------------------------------
|
|
# 1) Load csvs
|
|
# ----------------------------------------
|
|
t0 = time.perf_counter()
|
|
portfolio_df = load_csv("portfolio_569.csv")
|
|
properties_df = load_csv("properties_569.csv")
|
|
property_details_epc_df = load_csv("property_details_epc_569.csv")
|
|
plans_df = load_csv("plans_569.csv")
|
|
plan_recs_df = load_csv("plan_recs_569.csv")
|
|
recommendations_df = load_csv("recommendations_569.csv")
|
|
|
|
logger.info(
|
|
"Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s",
|
|
time.perf_counter() - t0,
|
|
len(properties_df),
|
|
len(plans_df),
|
|
len(recommendations_df),
|
|
)
|
|
|
|
logger.info("Starting database load")
|
|
db_load_t0 = time.perf_counter()
|
|
|
|
# ----------------------------------------
|
|
# 2) Insert test portfolio
|
|
# ----------------------------------------
|
|
|
|
portfolios = []
|
|
for row in portfolio_df.itertuples(index=False):
|
|
portfolios.append(
|
|
Portfolio(
|
|
id=row.id,
|
|
name=row.name,
|
|
status=PortfolioStatus[row.status.split(".")[-1]],
|
|
goal=PortfolioGoal[row.goal.split(".")[-1]] if row.goal else None,
|
|
)
|
|
)
|
|
|
|
db_session.bulk_save_objects(portfolios)
|
|
db_session.flush()
|
|
# ----------------------------------------
|
|
# 3) Insert test property
|
|
# ----------------------------------------
|
|
|
|
properties = []
|
|
|
|
for row in properties_df.itertuples(index=False):
|
|
row_dict = {field: getattr(row, field) for field in row._fields}
|
|
|
|
row_dict["uprn"] = int(row_dict["uprn"]) if row_dict.get("uprn") else None
|
|
row_dict["building_reference_number"] = (
|
|
int(row_dict["building_reference_number"])
|
|
if row_dict.get("building_reference_number")
|
|
else None
|
|
)
|
|
|
|
prop = PropertyModel(**{
|
|
col: row_dict[col]
|
|
for col in PropertyModel.__table__.columns.keys()
|
|
if col in row_dict
|
|
})
|
|
|
|
prop.creation_status = PropertyCreationStatus[
|
|
row_dict["creation_status"].split(".")[-1]
|
|
]
|
|
prop.status = PortfolioStatus[row_dict["status"].split(".")[-1]]
|
|
|
|
if row_dict.get("current_epc_rating"):
|
|
prop.current_epc_rating = Epc[
|
|
row_dict["current_epc_rating"].split(".")[-1]
|
|
]
|
|
|
|
properties.append(prop)
|
|
|
|
db_session.bulk_save_objects(properties)
|
|
db_session.flush()
|
|
|
|
# ----------------------------------------
|
|
# 4) Insert property details - EPC
|
|
# ----------------------------------------
|
|
|
|
epc_rows = []
|
|
|
|
for row in property_details_epc_df.itertuples(index=False):
|
|
row_dict = {field: getattr(row, field) for field in row._fields}
|
|
|
|
# Build only fields that exist on the model
|
|
epc_data = {
|
|
col.name: row_dict[col.name]
|
|
for col in PropertyDetailsEpcModel.__table__.columns.values()
|
|
if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"]
|
|
}
|
|
|
|
epc = PropertyDetailsEpcModel(
|
|
property_id=row.property_id,
|
|
portfolio_id=row.portfolio_id,
|
|
**epc_data,
|
|
)
|
|
|
|
epc_rows.append(epc)
|
|
|
|
db_session.bulk_save_objects(epc_rows)
|
|
db_session.flush()
|
|
|
|
# ----------------------------------------
|
|
# 4) Insert default plan
|
|
# ----------------------------------------
|
|
|
|
plans = []
|
|
|
|
for row in plans_df.itertuples(index=False):
|
|
row_dict = {field: getattr(row, field) for field in row._fields}
|
|
|
|
if row_dict.get("post_epc_rating"):
|
|
row_dict["post_epc_rating"] = Epc[
|
|
row_dict["post_epc_rating"].split(".")[-1]
|
|
]
|
|
|
|
row_dict["scenario_id"] = None
|
|
|
|
plan = PlanModel(**{
|
|
col: row_dict[col]
|
|
for col in PlanModel.__table__.columns.keys()
|
|
if col in row_dict
|
|
})
|
|
|
|
plans.append(plan)
|
|
|
|
db_session.bulk_save_objects(plans)
|
|
db_session.flush()
|
|
|
|
# ----------------------------------------
|
|
# 5) Insert recommendation
|
|
# ----------------------------------------
|
|
|
|
recs = [
|
|
Recommendation(**{
|
|
col: row[col]
|
|
for col in Recommendation.__table__.columns.keys()
|
|
if col in row
|
|
})
|
|
for _, row in recommendations_df.iterrows()
|
|
]
|
|
|
|
db_session.bulk_save_objects(recs)
|
|
db_session.flush()
|
|
|
|
# ----------------------------------------
|
|
# 6) Insert PlanRecommendations
|
|
# ----------------------------------------
|
|
links = [
|
|
PlanRecommendations(
|
|
plan_id=row.plan_id,
|
|
recommendation_id=row.recommendation_id,
|
|
)
|
|
for row in plan_recs_df.itertuples(index=False)
|
|
]
|
|
|
|
db_session.bulk_save_objects(links)
|
|
db_session.commit()
|
|
logger.info("Inserted all data in %.2f seconds", time.perf_counter() - db_load_t0)
|
|
|
|
# ----------------------------------------
|
|
# 6) Build payload
|
|
# ----------------------------------------
|
|
|
|
body_dict = {
|
|
"task_id": "test",
|
|
"subtask_id": "test",
|
|
"portfolio_id": 569,
|
|
"scenario_ids": [],
|
|
"default_plans_only": True,
|
|
}
|
|
|
|
payload = ExportRequest.model_validate(body_dict)
|
|
|
|
# ----------------------------------------
|
|
# 7) Call process_export
|
|
# ----------------------------------------
|
|
|
|
logger.info(
|
|
"Recommendation count in DB: %s",
|
|
db_session.query(Recommendation).count()
|
|
)
|
|
|
|
logger.info(
|
|
"Property count in DB: %s",
|
|
db_session.query(PropertyModel).count()
|
|
)
|
|
|
|
logger.info(
|
|
"Property EPC in DB: %s",
|
|
db_session.query(PropertyDetailsEpcModel).count()
|
|
)
|
|
|
|
logger.info(
|
|
"Plan count in DB: %s",
|
|
db_session.query(PlanModel).count()
|
|
)
|
|
|
|
logger.info(
|
|
"PlanRecommendatons count in DB: %s",
|
|
db_session.query(PlanModel).count()
|
|
)
|
|
|
|
logger.info("Starting process_export")
|
|
process_t0 = time.perf_counter()
|
|
|
|
result = process_export(payload, session=db_session)
|
|
|
|
logger.info("process_export finished in %.2f seconds", time.perf_counter() - process_t0)
|
|
|
|
# ----------------------------------------
|
|
# 8) Assertions
|
|
# ----------------------------------------
|
|
|
|
assert "default_plans" in result, "Expected 'default_plans' in export result, got {}".format(result.keys())
|
|
|
|
df = result["default_plans"]
|
|
|
|
assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0])
|
|
|
|
failed = df[df["predicted_post_works_sap"] < 69]
|
|
failed_property_types = failed["property_type"].value_counts().to_dict()
|
|
assert failed_property_types["Flat"] == 2
|
|
# Check the houses
|
|
|
|
assert failed.shape[0]
|
|
|
|
assert df["total_retrofit_cost"].sum() == 41706.585999999996, (
|
|
"Expected total retrofit cost to be 10000, got {}".format(df["total_retrofit_cost"].sum())
|
|
)
|
|
|
|
assert df["predicted_post_works_sap"].sum() == 698.1, (
|
|
"Expected total predicted post works SAP to be 698.1, got {}".format(df["predicted_post_works_sap"].sum())
|
|
)
|
|
|
|
assert df["sap_points"].sum() == 100.10000000000001, (
|
|
"Expected total SAP points increase to be 100.10000000000001, got {}".format(df["sap_points"].sum())
|
|
)
|
|
|
|
assert df.shape == (10, 100), "Expected dataframe shape to be (10, 100), got {}".format(df.shape)
|
|
|
|
|
|
def test_solar_with_battery_example(db_session):
|
|
test_portfolio_id = 1
|
|
test_property_id = 1
|
|
|
|
portfolio_df = pd.DataFrame(
|
|
[{'id': test_portfolio_id, 'name': 'Example', 'budget': None,
|
|
'status': 'PortfolioStatus.SCOPING', 'goal': 'PortfolioGoal.NONE', 'cost': None, 'number_of_properties': None,
|
|
'co2_equivalent_savings': None, 'energy_savings': None, 'energy_cost_savings': None,
|
|
'property_valuation_increase': None, 'rental_yield_increase': None, 'total_work_hours': None,
|
|
'labour_days': None, 'created_at': '2026-02-12 21:23:37.862000+00:00',
|
|
'updated_at': '2026-02-12 21:23:37.862000+00:00', 'epc_breakdown_pre_retrofit': None,
|
|
'epc_breakdown_post_retrofit': None, 'n_units_to_retrofit': None, 'co2_per_unit_pre_retrofit': None,
|
|
'co2_per_unit_post_retrofit': None, 'energy_bill_per_unit_pre_retrofit': None,
|
|
'energy_bill_per_unit_post_retrofit': None, 'energy_consumption_per_unit_pre_retrofit': None,
|
|
'energy_consumption_per_unit_post_retrofit': None, 'valuation_improvement_per_unit': None,
|
|
'cost_per_unit': None, 'cost_per_co2_saved': None, 'cost_per_sap_point': None,
|
|
'valuation_return_on_investment': None}]
|
|
)
|
|
|
|
properties_df = pd.DataFrame(
|
|
[{'id': test_property_id, 'portfolio_id': test_portfolio_id, 'creation_status': 'PropertyCreationStatus.READY',
|
|
'uprn': 100090438731, 'landlord_property_id': 'BARR052', 'building_reference_number': 3460742868.0,
|
|
'status': 'PortfolioStatus.ASSESSMENT', 'address': '52, Barrack Street', 'postcode': 'CO1 2LR',
|
|
'has_pre_condition_report': True, 'has_recommendations': True, 'created_at': '2026-02-12 21:59:02.744427',
|
|
'updated_at': '2026-02-19 16:18:57.941443', 'property_type': 'House', 'built_form': 'End-Terrace',
|
|
'local_authority': 'Colchester', 'constituency': 'Colchester', 'number_of_rooms': 4.0, 'year_built': 1900.0,
|
|
'tenure': 'rental (private)', 'current_epc_rating': 'Epc.E', 'current_sap_points': 53.0,
|
|
'current_valuation': 0.0, 'installed_measures_sap_point_adjustment': 0.0,
|
|
'is_sap_points_adjusted_for_installed_measures': False, 'original_sap_points': 53.0}]
|
|
)
|
|
|
|
property_details_epc_df = pd.DataFrame(
|
|
[
|
|
{'id': 1534934, 'property_id': test_property_id, 'portfolio_id': test_portfolio_id,
|
|
'full_address': '48, Medcalf Road', 'lodgement_date': '2018-09-05', 'is_expired': False,
|
|
'total_floor_area': 68.0, 'walls': 'Solid brick, as built, no insulation', 'walls_rating': 1,
|
|
'roof': 'Pitched, no insulation', 'roof_rating': 1.0, 'floor': 'Solid, no insulation',
|
|
'floor_rating': None,
|
|
'windows': 'Fully double glazed', 'windows_rating': 4, 'heating': 'Boiler and radiators, mains gas',
|
|
'heating_rating': 4, 'heating_controls': 'Programmer, room thermostat and trvs',
|
|
'heating_controls_rating': 4,
|
|
'hot_water': 'From main system', 'hot_water_rating': 4,
|
|
'lighting': 'Low energy lighting in all fixed outlets', 'lighting_rating': 5,
|
|
'mainfuel': 'Mains gas not community', 'ventilation': 'natural', 'solar_pv': 0.0, 'solar_hot_water': False,
|
|
'wind_turbine': 0.0, 'floor_height': 2.55, 'number_heated_rooms': None, 'heat_loss_corridor': False,
|
|
'unheated_corridor_length': None, 'number_of_open_fireplaces': 0, 'number_of_extensions': 0,
|
|
'number_of_storeys': None, 'mains_gas': True, 'energy_tariff': 'Single',
|
|
'primary_energy_consumption': 278.0,
|
|
'co2_emissions': 3.81, 'current_energy_demand': 14643.366,
|
|
'current_energy_demand_heating_hotwater': 12185.6,
|
|
'estimated': False, 'sap_05_overwritten': False, 'sap_05_score': None, 'sap_05_epc_rating': None,
|
|
'heating_cost_current': 711.0628, 'hot_water_cost_current': 139.06198, 'lighting_cost_current': 70.770935,
|
|
'appliances_cost_current': 609.7844, 'gas_standing_charge': 128.0785,
|
|
'electricity_standing_charge': 199.8375,
|
|
'original_co2_emissions': 3.81, 'original_primary_energy_consumption': 278.0,
|
|
'original_current_energy_demand': 14643.366, 'original_current_energy_demand_heating_hotwater': 12185.6,
|
|
'installed_measures_co2_adjustment': 0.0, 'installed_measures_energy_demand_adjustment': 0.0,
|
|
'installed_measures_total_energy_bill_adjustment': 0.0, 'installed_measures_heat_demand_adjustment': 0.0,
|
|
'is_epc_adjusted_for_installed_measures': False}
|
|
]
|
|
)
|
|
|
|
plans_df = pd.DataFrame(
|
|
[
|
|
{'id': 0, 'name': None, 'portfolio_id': test_portfolio_id, 'property_id': test_property_id,
|
|
'scenario_id': 1060, 'created_at': '2026-02-19 16:14:45.560816', 'is_default': True,
|
|
'valuation_increase_lower_bound': 0.0302,
|
|
'valuation_increase_upper_bound': 0.07, 'valuation_increase_average': 0.048226666, 'plan_type': None,
|
|
'post_sap_points': 71.5, 'post_epc_rating': 'Epc.C', 'post_co2_emissions': 4.1813498,
|
|
'co2_savings': 0.71865046, 'post_energy_bill': 1447.5204, 'energy_bill_savings': 691.6662,
|
|
'post_energy_consumption': 15303.688, 'energy_consumption_savings': 3276.7622,
|
|
'valuation_post_retrofit': None, 'valuation_increase': None, 'cost_of_works': 6984.568,
|
|
'contingency_cost': 1003.9568}
|
|
]
|
|
)
|
|
|
|
plan_recs_df = pd.DataFrame(
|
|
[{'id': 0, 'plan_id': 0, 'recommendation_id': 0}]
|
|
)
|
|
|
|
recommendations_df = pd.DataFrame(
|
|
[{'id': 0, 'property_id': test_property_id, 'created_at': '2026-02-19 16:14:45.560816',
|
|
'type': 'solar_pv', 'measure_type': 'solar_pv',
|
|
'description': 'Fit solar',
|
|
'estimated_cost': 10000, 'default': True, 'starting_u_value': None, 'new_u_value': None, 'sap_points': 1.5,
|
|
'heat_demand': 14.9, 'kwh_savings': 1041.2, 'co2_equivalent_savings': 0.2, 'energy_savings': 14.9,
|
|
'energy_cost_savings': 72.639015, 'property_valuation_increase': None, 'rental_yield_increase': None,
|
|
'total_work_hours': 4.16, 'labour_days': 1.0, 'already_installed': False, 'plan_name': 'whatever'}
|
|
]
|
|
)
|
|
|
|
recommendations_materials_df = pd.DataFrame(
|
|
[
|
|
{
|
|
"id": 0, "recommendation_id": 0, "material_id": 0, "depth": None, "quantity": 1.0,
|
|
"quantity_unit": "part",
|
|
"estimated_cost": 10000, "created_at": '2026-02-19 16:14:45.560816',
|
|
"updated_at": '2026-02-19 16:14:45.560816',
|
|
}
|
|
]
|
|
)
|
|
|
|
materials_df = pd.DataFrame(
|
|
[
|
|
{'id': 0, 'type': 'solar_pv', 'description': 'Some solar product',
|
|
'depth': 75.0,
|
|
'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
|
|
'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
|
|
'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'Test',
|
|
'created_at': "'2026-02-19 16:14:45.560816", 'is_active': True,
|
|
'prime_material_cost': None,
|
|
'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0,
|
|
'total_cost': 10000,
|
|
'notes': None, 'is_installer_quote': True, 'innovation_rate': 0.25, 'size': None, 'size_unit': None,
|
|
'includes_scaffolding': True, 'includes_battery': True, 'battery_size': 5.8}
|
|
]
|
|
)
|
|
|
|
# Load into db
|
|
# -------------------------------------------------
|
|
# Insert Portfolio
|
|
# -------------------------------------------------
|
|
for row in portfolio_df.itertuples(index=False):
|
|
db_session.add(
|
|
Portfolio(
|
|
id=row.id,
|
|
name=row.name,
|
|
status=PortfolioStatus[row.status.split(".")[-1]],
|
|
goal=PortfolioGoal[row.goal.split(".")[-1]],
|
|
)
|
|
)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Insert Property
|
|
# -------------------------------------------------
|
|
for row in properties_df.itertuples(index=False):
|
|
prop = PropertyModel(
|
|
id=row.id,
|
|
portfolio_id=row.portfolio_id,
|
|
creation_status=PropertyCreationStatus[row.creation_status.split(".")[-1]],
|
|
status=PortfolioStatus[row.status.split(".")[-1]],
|
|
uprn=row.uprn,
|
|
property_type=row.property_type,
|
|
current_sap_points=row.current_sap_points,
|
|
current_epc_rating=Epc[row.current_epc_rating.split(".")[-1]],
|
|
)
|
|
db_session.add(prop)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Insert EPC Details
|
|
# -------------------------------------------------
|
|
for row in property_details_epc_df.itertuples(index=False):
|
|
epc = PropertyDetailsEpcModel(
|
|
property_id=row.property_id,
|
|
portfolio_id=row.portfolio_id,
|
|
full_address=row.full_address,
|
|
total_floor_area=row.total_floor_area,
|
|
walls=row.walls,
|
|
roof=row.roof,
|
|
windows=row.windows,
|
|
heating=row.heating,
|
|
solar_pv=row.solar_pv,
|
|
)
|
|
db_session.add(epc)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Insert Plan (default)
|
|
# -------------------------------------------------
|
|
for row in plans_df.itertuples(index=False):
|
|
plan = PlanModel(
|
|
id=row.id,
|
|
portfolio_id=row.portfolio_id,
|
|
property_id=row.property_id,
|
|
scenario_id=None, # default mode
|
|
is_default=row.is_default,
|
|
)
|
|
db_session.add(plan)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# IMPORTANT: Force recommendation to be solar_pv
|
|
# -------------------------------------------------
|
|
recommendations_df.loc[0, "measure_type"] = "solar_pv"
|
|
|
|
for row in recommendations_df.itertuples(index=False):
|
|
rec = Recommendation(
|
|
id=row.id,
|
|
property_id=row.property_id,
|
|
measure_type=row.measure_type,
|
|
estimated_cost=row.estimated_cost,
|
|
default=row.default,
|
|
already_installed=row.already_installed,
|
|
sap_points=row.sap_points,
|
|
type=row.type,
|
|
description=row.description
|
|
)
|
|
db_session.add(rec)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Link Plan -> Recommendation
|
|
# -------------------------------------------------
|
|
for row in plan_recs_df.itertuples(index=False):
|
|
db_session.add(
|
|
PlanRecommendations(
|
|
plan_id=row.plan_id,
|
|
recommendation_id=row.recommendation_id,
|
|
)
|
|
)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Insert Material (includes_battery=True)
|
|
# -------------------------------------------------
|
|
for row in materials_df.itertuples(index=False):
|
|
material = Material(
|
|
id=row.id,
|
|
type=row.type,
|
|
description=row.description,
|
|
depth_unit=row.depth_unit,
|
|
cost_unit=row.cost_unit,
|
|
r_value_unit=row.r_value_unit,
|
|
thermal_conductivity_unit=row.thermal_conductivity_unit,
|
|
includes_battery=row.includes_battery,
|
|
is_active=row.is_active,
|
|
)
|
|
db_session.add(material)
|
|
db_session.flush()
|
|
|
|
# -------------------------------------------------
|
|
# Link Recommendation -> Material
|
|
# -------------------------------------------------
|
|
for row in recommendations_materials_df.itertuples(index=False):
|
|
db_session.add(
|
|
RecommendationMaterials(
|
|
recommendation_id=row.recommendation_id,
|
|
material_id=row.material_id,
|
|
depth=row.depth or 0.0,
|
|
quantity=row.quantity,
|
|
quantity_unit=row.quantity_unit,
|
|
estimated_cost=row.estimated_cost,
|
|
)
|
|
)
|
|
|
|
db_session.commit()
|
|
|
|
payload = ExportRequest.model_validate({
|
|
"task_id": "test",
|
|
"subtask_id": "test",
|
|
"portfolio_id": test_portfolio_id,
|
|
"scenario_ids": [],
|
|
"default_plans_only": True,
|
|
})
|
|
|
|
result = process_export(payload, session=db_session)
|
|
|
|
assert "default_plans" in result
|
|
|
|
df = result["default_plans"]
|
|
|
|
assert "solar_pv_with_battery" in df.columns
|
|
|
|
# solar_pv should NOT exist
|
|
assert "solar_pv" not in df.columns
|
|
|
|
assert df.shape[0] == 1, "Expected 1 property in the export, got {}".format(df.shape[0])
|
|
|
|
# Cost should land in correct column
|
|
assert df["solar_pv_with_battery"].iloc[0] == 10000
|