From a820274d4fc9d70883c8a0adbad8479cf605cd54 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 16 Jul 2026 01:26:32 +0000 Subject: [PATCH] ARA export: newest plan per (property, scenario) + uprn-matched Effective-EPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes so a multi-scenario export over a PasHub-fetched portfolio is correct: 1. Plan selection: take the NEWEST plan per (property, scenario), not the is_default one. is_default is one-per-property (not per-scenario), so it cannot scope a multi-scenario export — a property's default sits under a single scenario, leaving every other scenario with zero rows. _ROWS_SQL and _MEASURES_SQL now DISTINCT ON (property_id) ORDER BY created_at DESC, id DESC. 2. Effective-EPC descriptive block: match the header facts + current performance by UPRN (the latest epc_property for the source), because the PasHub re-fetch lands records with property_id = NULL — a property_id join silently fell back to a stale gov-EPC cert (wrong 2013-2025 lodgement dates / int property_type) or nothing. Lodgement date coalesces registration -> completion -> inspection (PasHub surveys carry only inspection_date). Descriptive prose elements stay on the property_id gov-EPC fallback (a survey lodges structured fields, not gov-EPC prose). Result on portfolio 838 / scenarios 1303+1304: both sheets populate 205 rows; property_type text on 204/205; lodgement dates 2026 on 200/205. Repository tests updated (uprn fixtures, newest-plan selection + a newest-wins case); 22 export tests pass, pyright clean. --- .../scenario_export_postgres_repository.py | 85 ++++++++++++------- ...est_scenario_export_postgres_repository.py | 66 +++++++++++--- 2 files changed, 110 insertions(+), 41 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 4029b41eb..a2674b20f 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -11,50 +11,69 @@ from repositories.scenario_export.scenario_export_repository import ( ScenarioExportRepository, ) -# The default Plan per (scenario, property), scoped to the requested Properties. -# The JOIN to `plan` is the model-A scenario-scoping (ADR-0065): a Property with -# no default Plan under the Scenario yields no row and drops out. +# The NEWEST Plan per (property, scenario), scoped to the requested Properties. +# `is_default` is one-per-property (not per-scenario), so it cannot scope a +# multi-scenario export — the freshest Plan under the Scenario (created_at DESC, +# id tiebreak) is the export's selection. A Property with no Plan under the +# Scenario yields no row and drops out. _ROWS_SQL = text( "SELECT p.id, p.landlord_property_id, p.uprn, p.address, p.postcode," - " pl.post_sap_points, pl.post_epc_rating, pl.cost_of_works," - " pl.contingency_cost, pl.co2_savings, pl.energy_bill_savings," - " pl.energy_consumption_savings, pl.valuation_increase" + " np.post_sap_points, np.post_epc_rating, np.cost_of_works," + " np.contingency_cost, np.co2_savings, np.energy_bill_savings," + " np.energy_consumption_savings, np.valuation_increase" " FROM property p" - " JOIN plan pl ON pl.property_id = p.id" + " JOIN (" + " SELECT DISTINCT ON (property_id) property_id, post_sap_points," + " post_epc_rating, cost_of_works, contingency_cost, co2_savings," + " energy_bill_savings, energy_consumption_savings, valuation_increase" + " FROM plan" + " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" + " AND property_id = ANY(:property_ids)" + " ORDER BY property_id, created_at DESC, id DESC" + " ) np ON np.property_id = p.id" " WHERE p.portfolio_id = :portfolio_id" - " AND pl.scenario_id = :scenario_id" - " AND pl.is_default = TRUE" - " AND p.id = ANY(:property_ids)" " ORDER BY p.id" ) -# The selected measures on each Property's default Plan: the recommendations -# kept in the Optimised Package (``default`` and not ``already_installed``), -# with the installed material's battery flag for the solar pivot (ADR-0065). -# ``default`` is a reserved word, hence the quoting. +# The selected measures on each Property's NEWEST Plan under the Scenario: the +# recommendations kept in the Optimised Package (``default`` and not +# ``already_installed``), with the installed material's battery flag for the +# solar pivot (ADR-0065). ``default`` is a reserved word, hence the quoting. _MEASURES_SQL = text( "SELECT r.property_id, r.measure_type, r.estimated_cost, r.sap_points," " r.co2_equivalent_savings, r.kwh_savings, r.energy_cost_savings," " COALESCE(m.includes_battery, FALSE) AS includes_battery" " FROM recommendation r" - " JOIN plan pl ON pl.id = r.plan_id" + " JOIN (" + " SELECT DISTINCT ON (property_id) id" + " FROM plan" + " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" + " AND property_id = ANY(:property_ids)" + " ORDER BY property_id, created_at DESC, id DESC" + " ) np ON np.id = r.plan_id" " LEFT JOIN material m ON m.id = r.material_id" - " WHERE pl.portfolio_id = :portfolio_id" - " AND pl.scenario_id = :scenario_id" - " AND pl.is_default = TRUE" - " AND pl.property_id = ANY(:property_ids)" - ' AND r."default" = TRUE' + ' WHERE r."default" = TRUE' " AND r.already_installed = FALSE" ) -# The Effective-EPC descriptive picture, read from the persisted lodged EPC slot -# (ADR-0065). Header facts, descriptive energy elements, and current performance -# are read separately and composed per Property. +# The Effective-EPC descriptive picture (ADR-0065). Header facts and current +# performance are matched by UPRN (the latest epc_property for the source), +# because the PasHub re-fetch lands records with property_id = NULL — a +# property_id join would silently fall back to a stale gov-EPC cert (wrong +# lodgement dates / property_type) or nothing. The descriptive prose elements +# have no PasHub equivalent (a survey lodges structured fields, not gov-EPC +# prose), so they stay on the property_id join as the best-available gov-EPC +# fallback. All three are read separately and composed per Property. _EPC_HEADER_SQL = text( - "SELECT property_id, property_type, total_floor_area_m2, registration_date," - " habitable_rooms_count" - " FROM epc_property" - " WHERE property_id = ANY(:property_ids) AND source = :source" + "WITH latest AS (" + " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" + " FROM property p JOIN epc_property e ON e.uprn = p.uprn" + " WHERE p.id = ANY(:property_ids) AND e.source = :source" + " ORDER BY p.id, e.id DESC)" + " SELECT l.property_id, e.property_type, e.total_floor_area_m2," + " COALESCE(e.registration_date, e.completion_date, e.inspection_date)," + " e.habitable_rooms_count" + " FROM latest l JOIN epc_property e ON e.id = l.epc_id" ) _EPC_ELEMENTS_SQL = text( "SELECT ep.property_id, e.element_type, e.description" @@ -63,11 +82,15 @@ _EPC_ELEMENTS_SQL = text( " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" ) _EPC_PERF_SQL = text( - "SELECT ep.property_id, perf.energy_rating_current," + "WITH latest AS (" + " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" + " FROM property p JOIN epc_property e ON e.uprn = p.uprn" + " WHERE p.id = ANY(:property_ids) AND e.source = :source" + " ORDER BY p.id, e.id DESC)" + " SELECT l.property_id, perf.energy_rating_current," " perf.current_energy_efficiency_band" - " FROM epc_property_energy_performance perf" - " JOIN epc_property ep ON ep.id = perf.epc_property_id" - " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" + " FROM latest l" + " JOIN epc_property_energy_performance perf ON perf.epc_property_id = l.epc_id" ) # A landlord property_type override wins over any EPC-derived value (ADR-0065); diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index b5940eb33..8d6f77bdd 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -34,6 +34,7 @@ def _add_epc( *, property_id: int, epc_id: int, + uprn: int = 100001, source: str = "lodged", property_type: Optional[str] = None, total_floor_area_m2: float = 90.0, @@ -50,6 +51,7 @@ def _add_epc( EpcPropertyModel( id=epc_id, property_id=property_id, + uprn=uprn, source=source, property_type=property_type, total_floor_area_m2=total_floor_area_m2, @@ -147,12 +149,13 @@ def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( assert row.cost_of_works == 4200.0 -def test_excludes_requested_properties_without_a_default_plan_under_the_scenario( +def test_excludes_requested_properties_without_a_plan_under_the_scenario( db_engine: Engine, ) -> None: - # arrange — four requested Properties: only #1 has a default Plan under - # scenario 5. #2 has a non-default Plan there, #3 a default Plan under a - # different scenario, #4 no Plan at all. + # arrange — four requested Properties: #1 and #2 both have a Plan under + # scenario 5 (#2's is non-default — is_default no longer gates, since it is + # one-per-property not per-scenario), #3 a Plan under a different scenario, + # #4 no Plan at all. with Session(db_engine) as session: for pid in (1, 2, 3, 4): session.add( @@ -175,8 +178,43 @@ def test_excludes_requested_properties_without_a_default_plan_under_the_scenario portfolio_id=100, scenario_id=5, property_ids=[1, 2, 3, 4] ) - # assert — only #1 (a default Plan under scenario 5) survives (model A). - assert [r.property_id for r in rows] == [1] + # assert — #1 and #2 (both have a Plan under scenario 5) survive; #3 (wrong + # scenario) and #4 (no Plan) drop out. + assert [r.property_id for r in rows] == [1, 2] + + +def test_selects_the_newest_plan_per_property_under_the_scenario( + db_engine: Engine, +) -> None: + # arrange — a Property with an older and a newer Plan under scenario 5; the + # newest Plan wins regardless of is_default (a re-model appends a new Plan). + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel( + id=1, portfolio_id=100, property_id=1, scenario_id=5, + is_default=True, post_sap_points=60.0, + ) + ) + session.add( + PlanModel( + id=2, portfolio_id=100, property_id=1, scenario_id=5, + is_default=False, post_sap_points=72.0, + ) + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the newer Plan (id=2, non-default) wins over the older default. + assert len(rows) == 1 + assert rows[0].post_sap_points == 72.0 def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( @@ -186,7 +224,9 @@ def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( # material carries a battery), plus a non-default and an already-installed # recommendation that must be excluded. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel( id=10, portfolio_id=100, property_id=1, scenario_id=5, is_default=True @@ -273,7 +313,9 @@ def test_effective_epc_descriptive_fields_come_from_the_lodged_epc( # arrange — a Property with a default Plan under scenario 5 and a lodged EPC # carrying descriptive energy elements and current performance. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) ) @@ -323,7 +365,9 @@ def test_property_type_override_beats_the_lodged_epc(db_engine: Engine) -> None: # arrange — the lodged EPC says Flat, but a landlord override says House # (Effective-EPC precedence: override → lodged, ADR-0065). with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) ) @@ -355,7 +399,9 @@ def test_falls_back_to_the_predicted_epc_when_there_is_no_lodged_epc( ) -> None: # arrange — the Property has only a predicted EPC slot, no lodged cert. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) )