From ef934f6b7c13918e014182ee043514a18604c019 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 28 Oct 2025 19:21:53 +0000 Subject: [PATCH] debugged funding test --- backend/SearchEpc.py | 15 +- .../test_data/innovation_measure_fixtures.py | 40 +- backend/tests/test_funding.py | 102 +- backend/tests/test_integration.py | 1061 +++++++++-------- backend/tests/test_search_epc.py | 3 +- recommendations/tests/test_optimisers.py | 142 ++- 6 files changed, 756 insertions(+), 607 deletions(-) diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index 1a14e87a..60999e94 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -418,7 +418,20 @@ class SearchEpc: address, [", ".join([r["address"]]) for r in rows], score_cutoff=0 ) # Pick the largest score - if best_match1[1] >= best_match2[1]: + if best_match1[1] == best_match2[1]: + # if thery're the same, we'll work under the assumption that the addresses are the same and we'll + # take whichever has the newest EPC + rows_filtered = [ + r for r in rows + if (", ".join([r["address"], r["posttown"]]) == best_match1[0]) or + (r["address"] == best_match2[0]) + ] + rows_filtered = [ + r for r in rows_filtered + if r["lodgement-datetime"] == max([x["lodgement-datetime"] for x in rows_filtered]) + ] + + elif best_match1[1] > best_match2[1]: # Get all of the scores rows_filtered = [r for r in rows if ", ".join([r["address"], r["posttown"]]) == best_match1[0]] else: diff --git a/backend/tests/test_data/innovation_measure_fixtures.py b/backend/tests/test_data/innovation_measure_fixtures.py index 886421c4..a66cc7ec 100644 --- a/backend/tests/test_data/innovation_measure_fixtures.py +++ b/backend/tests/test_data/innovation_measure_fixtures.py @@ -4,7 +4,7 @@ innovation_scenarios = [ # 1) Innovation PV, non-eligible heating system in place, EPC D - not eligible { "description": "Innovation PV, non-eligible heating system in place, EPC D", - "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}], + "measures": [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}], "starting_sap": 60, "mainheat_description": "Electric storage heaters", "heating_control_description": "Manual charge control", @@ -16,7 +16,7 @@ innovation_scenarios = [ # 2) Innovation PV, eligible heating system in place, EPC D - eligible { "description": "Innovation PV, eligible heating system in place, EPC D", - "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}], + "measures": [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", "heating_control_description": "Programmer, room thermostat and TRVs", @@ -29,8 +29,8 @@ innovation_scenarios = [ { "description": "Innovation PV + HHRSH upgrade, EPC E", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "high_heat_retention_storage_heater", "is_innovation": False, "innovation_uplift": 0} ], "starting_sap": 50, "mainheat_description": "Electric storage heaters", @@ -44,8 +44,8 @@ innovation_scenarios = [ { "description": "Innovation PV + HHRSH upgrade, EPC E", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "high_heat_retention_storage_heater", "is_innovation": False, "innovation_uplift": 0} ], "starting_sap": 50, "mainheat_description": "Electric storage heaters", @@ -58,7 +58,7 @@ innovation_scenarios = [ # 5) Innovation PV, needs wall insulation, no wall insulation measure - not eligible { "description": "Innovation PV, wall insulation recommended, but not installed", - "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}], + "measures": [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", "heating_control_description": "Programmer, room thermostat and TRVs", @@ -71,8 +71,8 @@ innovation_scenarios = [ { "description": "Innovation PV, wall insulation recommended and installed", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0.25} ], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", @@ -85,7 +85,7 @@ innovation_scenarios = [ # 7) Innovation PV, needs roof insulation, no roof insulation measure - not eligible { "description": "Innovation PV, roof insulation recommended, not installed", - "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}], + "measures": [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", "heating_control_description": "Programmer, room thermostat and TRVs", @@ -98,8 +98,8 @@ innovation_scenarios = [ { "description": "Innovation PV, roof insulation recommended and installed", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0} ], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", @@ -112,7 +112,7 @@ innovation_scenarios = [ # 9) Innovation PV, needs both roof + wall insulation, no insulation - not eligible { "description": "Innovation PV, both insulations recommended, none installed", - "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}], + "measures": [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", "heating_control_description": "Programmer, room thermostat and TRVs", @@ -125,8 +125,8 @@ innovation_scenarios = [ { "description": "Innovation PV, both insulations recommended, only wall done", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0.25} ], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", @@ -140,8 +140,8 @@ innovation_scenarios = [ { "description": "Innovation PV, both insulations recommended, only roof done", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0} ], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", @@ -155,9 +155,9 @@ innovation_scenarios = [ { "description": "Innovation PV, both insulations recommended and installed", "measures": [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0.25}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0} ], "starting_sap": 60, "mainheat_description": "Air source heat pump, radiators", diff --git a/backend/tests/test_funding.py b/backend/tests/test_funding.py index 59d65a28..d84480ce 100644 --- a/backend/tests/test_funding.py +++ b/backend/tests/test_funding.py @@ -120,7 +120,7 @@ def test_eco4_prs_eligible_with_swi( # 3) is getting a solid was measure # so it's eligible for ECO4 - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=50, # EPC E @@ -162,7 +162,7 @@ def test_eco4_prs_not_eligible_high_epc( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=72, # EPC C (too high) @@ -203,7 +203,7 @@ def test_gbis_prs_general_eligibility( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=65, # EPC D @@ -244,7 +244,7 @@ def test_gbis_prs_low_income_caveat( tenure="Private", ) - measures = [{"type": "cavity_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "cavity_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=60, # EPC D @@ -290,7 +290,7 @@ def test_eco4_sh_epc_e_eligible( tenure="Social", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=50, # EPC E @@ -330,7 +330,7 @@ def test_eco4_sh_epc_d_requires_innovation( tenure="Social", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] funding.check_funding( measures=measures, starting_sap=60, # EPC D @@ -365,7 +365,7 @@ def test_eco4_sh_epc_d_requires_innovation( gbis_private_solid_abs_rate=28, tenure="Social", ) - measures2 = [{"type": "internal_wall_insulation", "is_innovation": True, "uplift": 0.25}] + measures2 = [{"type": "internal_wall_insulation", "is_innovation": True, "innovation_uplift": 0.25}] funding2.check_funding( measures=measures2, starting_sap=60, # EPC D @@ -403,7 +403,7 @@ def test_eco4_sh_epc_d_requires_innovation( gbis_private_solid_abs_rate=28, tenure="Social", ) - measures3 = [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}] + measures3 = [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}] funding3.check_funding( measures=measures3, starting_sap=60, # EPC D @@ -439,7 +439,7 @@ def test_eco4_sh_epc_d_requires_innovation( tenure="Social", ) - measures4 = [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, ] + measures4 = [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, ] funding4.check_funding( measures=measures4, starting_sap=60, # EPC D @@ -476,8 +476,8 @@ def test_eco4_sh_epc_d_requires_innovation( ) measures5 = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "high_heat_retention_storage_heater", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "high_heat_retention_storage_heater", "is_innovation": False, "innovation_uplift": 0} ] funding5.check_funding( measures=measures5, @@ -516,7 +516,7 @@ def test_eco4_sh_epc_d_requires_innovation( ) measures6 = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, ] funding6.check_funding( measures=measures6, @@ -556,9 +556,9 @@ def test_eco4_sh_epc_d_requires_innovation( tenure="Social", ) measures7 = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "cavity_wall_insulation", "is_innovation": False, "uplift": 0.25}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "cavity_wall_insulation", "is_innovation": False, "innovation_uplift": 0.25}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0} ] funding7.check_funding( measures=measures7, @@ -599,7 +599,7 @@ def test_eco4_sh_solar_pv_requires_heating( tenure="Social", ) - measures = [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}] + measures = [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}] funding.check_funding( measures=measures, starting_sap=60, # EPC D @@ -641,8 +641,8 @@ def test_eco4_sh_solar_pv_with_heating_is_ok( ) measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0} ] funding.check_funding( measures=measures, @@ -684,7 +684,7 @@ def test_eco4_upgrade_requirement_e_to_c_pass( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] # E (SAP 50) → C (SAP 70) meets upgrade rule funding.check_funding( @@ -727,7 +727,7 @@ def test_eco4_upgrade_requirement_e_to_d_fail( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] # E (SAP 50) → D (SAP 65) does NOT meet ECO4 upgrade rule funding.check_funding( @@ -770,7 +770,7 @@ def test_eco4_upgrade_requirement_f_to_d_pass( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] # F (SAP 35) → D (SAP 60) is OK for ECO4 funding.check_funding( @@ -813,7 +813,7 @@ def test_eco4_upgrade_requirement_f_to_e_fail( tenure="Private", ) - measures = [{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}] + measures = [{"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}] # F (SAP 35) → E (SAP 50) does NOT meet ECO4 rule funding.check_funding( @@ -859,7 +859,7 @@ def test_epc_d_social_no_innovation_no_heating( ) measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45} ] funding.check_funding( @@ -905,10 +905,10 @@ def test_epc_d_social_with_heating_and_insulation( # Should NOT be eligible as the ASHP is not an innovation measure measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0} ] funding.check_funding( @@ -954,9 +954,9 @@ def test_epc_d_social_solar_with_only_minimum_insulation_should_fail( # Solar PV innovation with insulation, but no heating system upgrade => not eligible measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0} ] funding.check_funding( @@ -1002,8 +1002,8 @@ def test_epc_d_social_solar_with_ashp_and_no_insulation_should_fail( # Solar PV innovation with heating, but no insulation when insulation is recommended => not eligible measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0} ] funding.check_funding( @@ -1050,10 +1050,10 @@ def test_epc_d_social_solar_with_heating_and_minimum_insulation_should_pass( # Innovation solar + insulation measures + eligible heating upgrade = not valid because the heat pump isn;t # an innovation measure measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0} ] funding.check_funding( @@ -1095,10 +1095,10 @@ def test_epc_d_social_solar_with_heating_and_minimum_insulation_should_pass( # Innovation solar + insulation measures + eligible heating upgrade = should be valid because the # heat pump is an innovation measure measures2 = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0}, - {"type": "air_source_heat_pump", "is_innovation": True, "uplift": 0.25} + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "air_source_heat_pump", "is_innovation": True, "innovation_uplift": 0.25} ] funding2.check_funding( @@ -1203,11 +1203,11 @@ def test_uplift( # # TODO: Add a scenario with multiple measures, where some are innovation, some are not and we have # TODO: Make sure private works too measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0}, - {"type": "cavity_wall_insulation", "is_innovation": False, "uplift": 0.25}, + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "internal_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0}, + {"type": "cavity_wall_insulation", "is_innovation": False, "innovation_uplift": 0.25}, ] funding.check_funding( @@ -1229,7 +1229,7 @@ def test_uplift( ) assert funding.eco4_funding == 5302.3949999999995 - assert funding.full_project_abs == 392.77 # is 280 + the 112.77 innovation uplift + assert funding.full_project_abs == 280 # Doesn't include the eco4 uplift assert funding.eco4_uplift == 112.77 @@ -1311,7 +1311,7 @@ def test_private_epc_e_solar_needs_heating( tenure="Private", ) - measures = [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}] + measures = [{"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}] funding.check_funding( measures=measures, starting_sap=54, # EPC E - eligible for private on EPC @@ -1360,10 +1360,10 @@ def test_private_epc_e_solar_with_heating_and_minimum_insulation_produces_uplift ) measures = [ - {"type": "solar_pv", "is_innovation": True, "uplift": 0.45}, - {"type": "air_source_heat_pump", "is_innovation": False, "uplift": 0}, - {"type": "cavity_wall_insulation", "is_innovation": False, "uplift": 0}, - {"type": "loft_insulation", "is_innovation": False, "uplift": 0}, + {"type": "solar_pv", "is_innovation": True, "innovation_uplift": 0.45}, + {"type": "air_source_heat_pump", "is_innovation": False, "innovation_uplift": 0}, + {"type": "cavity_wall_insulation", "is_innovation": False, "innovation_uplift": 0}, + {"type": "loft_insulation", "is_innovation": False, "innovation_uplift": 0}, ] funding.check_funding( diff --git a/backend/tests/test_integration.py b/backend/tests/test_integration.py index e6bcfce8..60778132 100644 --- a/backend/tests/test_integration.py +++ b/backend/tests/test_integration.py @@ -1,531 +1,532 @@ -import ast -import json -from copy import deepcopy -from dataclasses import replace -from datetime import datetime - -import random -from tqdm import tqdm -import pandas as pd -import numpy as np -from etl.epc.Record import EPCRecord -from backend.SearchEpc import SearchEpc -from sqlalchemy.exc import IntegrityError, OperationalError -from sqlalchemy.orm import sessionmaker -from starlette.responses import Response - -from backend.app.config import get_settings, get_prediction_buckets -from backend.app.db.connection import db_engine -from backend.app.db.functions.materials_functions import get_materials -from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations -from backend.app.db.functions.property_functions import ( - create_property, create_property_details_epc, create_property_targets, update_property_data, - update_or_create_property_spatial_details -) -from backend.app.db.functions.recommendations_functions import ( - create_plan, upload_recommendations, create_scenario -) -from backend.app.db.functions.funding_functions import upload_funding -from backend.app.db.functions.energy_assessment_functions import get_latest_assessment_by_uprn -from backend.app.db.models.portfolio import rating_lookup -from backend.app.plan.schemas import PlanTriggerRequest, WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES -from backend.app.plan.utils import get_cleaned -from backend.app.utils import sap_to_epc -import backend.app.assumptions as assumptions - -from backend.ml_models.api import ModelApi -from backend.Property import Property -from backend.apis.GoogleSolarApi import GoogleSolarApi - -from recommendations.optimiser.CostOptimiser import CostOptimiser -from recommendations.optimiser.GainOptimiser import GainOptimiser -import recommendations.optimiser.optimiser_functions as optimiser_functions -from recommendations.Recommendations import Recommendations -from utils.logger import setup_logger -from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3, read_excel_from_s3 -from backend.ml_models.Valuation import PropertyValuation - -from etl.bill_savings.KwhData import KwhData -from etl.spatial.OpenUprnClient import OpenUprnClient -from etl.find_my_epc.RetrieveFindMyEpc import RetrieveFindMyEpc - -from backend.Funding import Funding -from recommendations.optimiser.funding_optimiser import optimise_with_funding_paths -from recommendations.recommendation_utils import convert_thickness_to_numeric, get_wall_u_value - -# Input data (temp) -import pickle - -import pandas as pd - -with open("local_data_for_deletion.pkl", 'rb') as f: - local_data = pickle.load(f) - -cleaning_data = local_data["cleaning_data"] -materials = local_data["materials"] -cleaned = local_data["cleaned"] -project_scores_matrix = local_data["project_scores_matrix"] -partial_project_scores_matrix = local_data["partial_project_scores_matrix"] -whlg_eligible_postcodes = local_data["whlg_eligible_postcodes"] - -with open("kwh_client_for_deletion.pkl", "rb") as f: - kwh_client = pickle.load(f) - -epc_data = pd.read_csv( - "/Users/khalimconn-kowlessar/Downloads/all-domestic-certificates/domestic-E06000002-Middlesbrough/certificates.csv", - low_memory=False -) - -# TODO: Store this for cleaning -costs_by_floor_area = epc_data[ - pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2024-01-01" - ][["TOTAL_FLOOR_AREA", "CURRENT_ENERGY_EFFICIENCY", "LIGHTING_COST_CURRENT", "HEATING_COST_CURRENT", - "HOT_WATER_COST_CURRENT"]].copy() - -costs_by_floor_area.columns = [c.lower().replace("_", "-") for c in costs_by_floor_area.columns] -for c in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]: - costs_by_floor_area[c + "_scaled"] = costs_by_floor_area[c] / costs_by_floor_area["total-floor-area"] - -costs_by_floor_area = costs_by_floor_area.groupby("current-energy-efficiency")[ - ["lighting-cost-current_scaled", "heating-cost-current_scaled", "hot-water-cost-current_scaled"] -].mean().reset_index() - -sample_epc_data = epc_data[pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2015-01-01"].drop_duplicates("UPRN").sample( - 1000).reset_index(drop=True) - -# TODO: In Property find_energy_sources, sort out biomass community heating - what fuel type -# TODO: We might be able to remove find_energy_sources entirely and remove estimate_electrical_consumption. It's used -# in the google solar api but is it really needed? I don't think it's super accurate. It might be better to -# just use an average energy consumption by floor area for UK households? -# Load the input properties -input_properties = [] -for row_id, config in tqdm(sample_epc_data.iterrows(), total=len(sample_epc_data)): - epc = { - k.lower().replace("_", "-"): v if not pd.isnull(v) else None for k, v in config.items() - } - # Avoid the data load inside of EPCRecord - something we should pull out - for x in ["number-habitable-rooms", "floor-height", "number-heated-rooms"]: - if pd.isnull(epc[x]): - if x == "floor-height": - epc[x] = 2.4 - if x == "number-habitable-rooms": - epc[x] = 3 - if x == "number-heated-rooms": - epc[x] = 3 - - epc_records = {'original_epc': epc, 'full_sap_epc': {}, 'old_data': []} - - prepared_epc = EPCRecord( - epc_records=epc_records, - run_mode="newdata", - cleaning_data=cleaning_data, - ) - - input_properties.append( - Property( - id=row_id, - is_new=True, - address=epc["address"], - postcode=epc["postcode"], - epc_record=prepared_epc, - already_installed={}, - property_valuation={}, - non_invasive_recommendations=[], - energy_assessment=None, - **Property.extract_kwargs(config), # TODO: Depraecate this - ) - ) - -# For each property, insert the default solar configuration -for p in tqdm(input_properties): - solar_api = GoogleSolarApi( - api_key=None, solar_materials=[m for m in materials if m["type"] == "solar_pv"], max_retries=5 - ) - panel_performance = solar_api.default_panel_performance(property_instance=p) - p.set_solar_panel_configuration( - solar_panel_configuration={ - "insights_data": None, "panel_performance": panel_performance, "unit_share_of_energy": 1 - }, - ) - -# We mock kwh preds -mocked_kwh_predictions = {"heating_kwh_predictions": [], "hotwater_kwh_predictions": []} -for p in tqdm(input_properties): - mocked_kwh_predictions["heating_kwh_predictions"].append({ - "id": p.uprn, "predictions": random.sample(range(100, 3000), 1)[0] - }) - mocked_kwh_predictions["hotwater_kwh_predictions"].append({ - "id": p.uprn, "predictions": random.sample(range(100, 3000), 1)[0] - }) -mocked_kwh_predictions["heating_kwh_predictions"] = pd.DataFrame(mocked_kwh_predictions["heating_kwh_predictions"]) -mocked_kwh_predictions["hotwater_kwh_predictions"] = pd.DataFrame(mocked_kwh_predictions["hotwater_kwh_predictions"]) - -# TODO: We might want to implement this generally, via an ETL process -for p in input_properties: - for col in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]: - if pd.isnull(p.data[col]): - min_diff = abs( - (costs_by_floor_area["current-energy-efficiency"] - p.data["current-energy-efficiency"]) - ).min() - df = costs_by_floor_area[ - abs((costs_by_floor_area["current-energy-efficiency"] - p.data[ - "current-energy-efficiency"])) == min_diff - ] - if df.shape[0] > 1: - df = df.head(1) - p.data[col] = (df[col + "_scaled"] * p.data["total-floor-area"]).values[0] - -[ - p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=mocked_kwh_predictions) for p in - input_properties -] +# import ast +# import json +# from copy import deepcopy +# from dataclasses import replace +# from datetime import datetime +# +# import random +# from tqdm import tqdm +# import pandas as pd +# import numpy as np +# from etl.epc.Record import EPCRecord +# from backend.SearchEpc import SearchEpc +# from sqlalchemy.exc import IntegrityError, OperationalError +# from sqlalchemy.orm import sessionmaker +# from starlette.responses import Response +# +# from backend.app.config import get_settings, get_prediction_buckets +# from backend.app.db.connection import db_engine +# from backend.app.db.functions.materials_functions import get_materials +# from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations +# from backend.app.db.functions.property_functions import ( +# create_property, create_property_details_epc, create_property_targets, update_property_data, +# update_or_create_property_spatial_details +# ) +# from backend.app.db.functions.recommendations_functions import ( +# create_plan, upload_recommendations, create_scenario +# ) +# from backend.app.db.functions.funding_functions import upload_funding +# from backend.app.db.functions.energy_assessment_functions import get_latest_assessment_by_uprn +# from backend.app.db.models.portfolio import rating_lookup +# from backend.app.plan.schemas import PlanTriggerRequest, WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES +# from backend.app.plan.utils import get_cleaned +# from backend.app.utils import sap_to_epc +# import backend.app.assumptions as assumptions +# +# from backend.ml_models.api import ModelApi +# from backend.Property import Property +# from backend.apis.GoogleSolarApi import GoogleSolarApi +# +# from recommendations.optimiser.CostOptimiser import CostOptimiser +# from recommendations.optimiser.GainOptimiser import GainOptimiser +# import recommendations.optimiser.optimiser_functions as optimiser_functions +# from recommendations.Recommendations import Recommendations +# from utils.logger import setup_logger +# from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3, read_excel_from_s3 +# from backend.ml_models.Valuation import PropertyValuation +# +# from etl.bill_savings.KwhData import KwhData +# from etl.spatial.OpenUprnClient import OpenUprnClient +# from etl.find_my_epc.RetrieveFindMyEpc import RetrieveFindMyEpc +# +# from backend.Funding import Funding +# from recommendations.optimiser.funding_optimiser import optimise_with_funding_paths +# from recommendations.recommendation_utils import convert_thickness_to_numeric, get_wall_u_value +# +# # Input data (temp) +# import pickle +# +# import pandas as pd +# +# with open("local_data_for_deletion.pkl", 'rb') as f: +# local_data = pickle.load(f) +# +# cleaning_data = local_data["cleaning_data"] +# materials = local_data["materials"] +# cleaned = local_data["cleaned"] +# project_scores_matrix = local_data["project_scores_matrix"] +# partial_project_scores_matrix = local_data["partial_project_scores_matrix"] +# whlg_eligible_postcodes = local_data["whlg_eligible_postcodes"] +# +# with open("kwh_client_for_deletion.pkl", "rb") as f: +# kwh_client = pickle.load(f) +# +# epc_data = pd.read_csv( +# "/Users/khalimconn-kowlessar/Downloads/all-domestic-certificates/domestic-E06000002-Middlesbrough/certificates +# .csv", +# low_memory=False +# ) +# +# # TODO: Store this for cleaning +# costs_by_floor_area = epc_data[ +# pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2024-01-01" +# ][["TOTAL_FLOOR_AREA", "CURRENT_ENERGY_EFFICIENCY", "LIGHTING_COST_CURRENT", "HEATING_COST_CURRENT", +# "HOT_WATER_COST_CURRENT"]].copy() +# +# costs_by_floor_area.columns = [c.lower().replace("_", "-") for c in costs_by_floor_area.columns] +# for c in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]: +# costs_by_floor_area[c + "_scaled"] = costs_by_floor_area[c] / costs_by_floor_area["total-floor-area"] +# +# costs_by_floor_area = costs_by_floor_area.groupby("current-energy-efficiency")[ +# ["lighting-cost-current_scaled", "heating-cost-current_scaled", "hot-water-cost-current_scaled"] +# ].mean().reset_index() +# +# sample_epc_data = epc_data[pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2015-01-01"].drop_duplicates("UPRN").sample( +# 1000).reset_index(drop=True) +# +# # TODO: In Property find_energy_sources, sort out biomass community heating - what fuel type +# # TODO: We might be able to remove find_energy_sources entirely and remove estimate_electrical_consumption. It's used +# # in the google solar api but is it really needed? I don't think it's super accurate. It might be better to +# # just use an average energy consumption by floor area for UK households? +# # Load the input properties +# input_properties = [] +# for row_id, config in tqdm(sample_epc_data.iterrows(), total=len(sample_epc_data)): +# epc = { +# k.lower().replace("_", "-"): v if not pd.isnull(v) else None for k, v in config.items() +# } +# # Avoid the data load inside of EPCRecord - something we should pull out +# for x in ["number-habitable-rooms", "floor-height", "number-heated-rooms"]: +# if pd.isnull(epc[x]): +# if x == "floor-height": +# epc[x] = 2.4 +# if x == "number-habitable-rooms": +# epc[x] = 3 +# if x == "number-heated-rooms": +# epc[x] = 3 +# +# epc_records = {'original_epc': epc, 'full_sap_epc': {}, 'old_data': []} +# +# prepared_epc = EPCRecord( +# epc_records=epc_records, +# run_mode="newdata", +# cleaning_data=cleaning_data, +# ) +# +# input_properties.append( +# Property( +# id=row_id, +# is_new=True, +# address=epc["address"], +# postcode=epc["postcode"], +# epc_record=prepared_epc, +# already_installed={}, +# property_valuation={}, +# non_invasive_recommendations=[], +# energy_assessment=None, +# **Property.extract_kwargs(config), # TODO: Depraecate this +# ) +# ) +# +# # For each property, insert the default solar configuration +# for p in tqdm(input_properties): +# solar_api = GoogleSolarApi( +# api_key=None, solar_materials=[m for m in materials if m["type"] == "solar_pv"], max_retries=5 +# ) +# panel_performance = solar_api.default_panel_performance(property_instance=p) +# p.set_solar_panel_configuration( +# solar_panel_configuration={ +# "insights_data": None, "panel_performance": panel_performance, "unit_share_of_energy": 1 +# }, +# ) +# +# # We mock kwh preds +# mocked_kwh_predictions = {"heating_kwh_predictions": [], "hotwater_kwh_predictions": []} +# for p in tqdm(input_properties): +# mocked_kwh_predictions["heating_kwh_predictions"].append({ +# "id": p.uprn, "predictions": random.sample(range(100, 3000), 1)[0] +# }) +# mocked_kwh_predictions["hotwater_kwh_predictions"].append({ +# "id": p.uprn, "predictions": random.sample(range(100, 3000), 1)[0] +# }) +# mocked_kwh_predictions["heating_kwh_predictions"] = pd.DataFrame(mocked_kwh_predictions["heating_kwh_predictions"]) +# mocked_kwh_predictions["hotwater_kwh_predictions"] = pd.DataFrame(mocked_kwh_predictions["hotwater_kwh_predictions"]) +# +# # TODO: We might want to implement this generally, via an ETL process # for p in input_properties: -# p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=mocked_kwh_predictions) - -# Run the recommendations -recommendations = {} -recommendations_scoring_data = [] -representative_recommendations = {} -for p in tqdm(input_properties): - if p.data["property-type"] == "House" and pd.isnull(p.data["built-form"]): - p.data["built-form"] = "Semi-Detached" - recommender = Recommendations( - property_instance=p, - materials=materials, - exclusions=[], - inclusions=[], - default_u_values=True - ) - property_recommendations, property_representative_recommendations = recommender.recommend() - - if not property_recommendations: - continue - - recommendations[p.id] = property_recommendations - representative_recommendations[p.id] = property_representative_recommendations - - p.create_base_difference_epc_record(cleaned_lookup=cleaned) - p.adjust_difference_record_with_recommendations( - property_recommendations, property_representative_recommendations - ) - - recommendations_scoring_data.extend(p.recommendations_scoring_data) - -recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) -recommendations_scoring_data = recommendations_scoring_data.drop( - columns=[ - "rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending" - ] -) - -model_predictions_mocked = { - "sap_change_predictions": None, - "heat_demand_predictions": None, - "carbon_change_predictions": None, - "heating_kwh_predictions": None, - "hotwater_kwh_predictions": None, -} - -for k in model_predictions_mocked.keys(): - model_predictions_mocked[k] = recommendations_scoring_data[["id"]].copy() - model_predictions_mocked[k][['property_id', 'recommendation_id']] = ( - model_predictions_mocked[k]['id'].str.split('+', expand=True) - ) - model_predictions_mocked[k]['phase'] = model_predictions_mocked[k]['recommendation_id'].apply( - ModelApi.extract_phase) - - if k in ["heating_kwh_predictions", "hotwater_kwh_predictions"]: - model_predictions_mocked[k]["predictions"] = random.choices(range(100, 3000), - k=len(recommendations_scoring_data)) - continue - - model_predictions_mocked[k] = model_predictions_mocked[k].sort_values(["property_id", "phase"], ascending=True) - preds = [] - for p_id in model_predictions_mocked[k]["property_id"].unique(): - # We add some amount each time - p = [p for p in input_properties if str(p.id) == p_id][0] - if k == "sap_change_predictions": - start = p.data["current-energy-efficiency"] - elif k == "heat_demand_predictions": - start = p.data["energy-consumption-current"] - else: - start = p.data["co2-emissions-current"] - df = model_predictions_mocked[k][model_predictions_mocked[k]["property_id"] == p_id].copy() - # Add some amount each time - to_add = random.choices(range(0, 15), k=len(df)) - to_add = np.cumsum(to_add) - df["predictions"] = start + to_add - preds.append(df) - preds = pd.concat(preds) - model_predictions_mocked[k] = preds - -for property_id in tqdm(recommendations.keys(), total=len(recommendations)): - property_instance = [p for p in input_properties if p.id == property_id][0] - - recommendations_with_impact, impact_summary = ( - Recommendations.calculate_recommendation_impact( - property_instance=property_instance, - all_predictions=model_predictions_mocked, - recommendations=recommendations, - representative_recommendations=representative_recommendations - ) - ) - - # We use the impact_summary to update the simulation_epcs with the new SAP, heat demand, carbon, cost etc - # at each phase - property_instance.update_simulation_epcs(impact_summary) - recommendations[property_id] = recommendations_with_impact - -for property_id in tqdm([p.id for p in input_properties]): - property_recommendations = recommendations.get(property_id, []) - property_instance = [p for p in input_properties if p.id == property_id][0] - - property_current_energy_bill = ( - Recommendations.calculate_recommendation_tenant_savings( - property_instance=property_instance, - kwh_simulation_predictions=model_predictions_mocked, - property_recommendations=property_recommendations, - ashp_cop=2.8 - ) - ) - property_instance.current_energy_bill = property_current_energy_bill - -body = PlanTriggerRequest( - **{'budget': None, 'goal': 'Increasing EPC', 'housing_type': 'Social', 'goal_value': 'B', 'portfolio_id': 0, - 'trigger_file_path': '', 'already_installed_file_path': '', - 'patches_file_path': None, 'non_invasive_recommendations_file_path': None, - 'valuation_file_path': '', - 'required_measures': [], 'scenario_name': 'EPC B', 'scenario_id': None, - 'multi_plan': True, 'optimise': True, 'default_u_values': True, 'ashp_cop': 2.8, - 'event_type': 'remote_assessment', 'simulate_sap_10': False, 'file_type': None, 'file_format': None, - 'sheet_name': None, 'sheet_count': None, 'index_start': None, 'index_end': None} -) - -for p in tqdm(input_properties): - if not recommendations.get(p.id): - continue - - # we need to double unlist because we have a list of lists - property_measure_types = {rec["type"] for recs in recommendations[p.id] for rec in recs} - property_required_measures = [m for m in recommendations[p.id] if m[0]["type"] in body.required_measures] - measures_to_optimise = [m for m in recommendations[p.id] if m[0]["type"] not in body.required_measures] - - # If a measure requiring ventilation is selected, and the property does not have ventilation, we enfore - # its inclusion - needs_ventilation = any( - x in property_measure_types for x in assumptions.measures_needing_ventilation - ) and not p.has_ventilation - - if not measures_to_optimise: - # Nothing to do, we just reshape the recommendations - recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( - p.id, recommendations, set() - ) - continue - - fixed_gain = optimiser_functions.calculate_fixed_gain( - property_required_measures, recommendations, p, needs_ventilation - ) - gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain) - - funding = Funding( - tenure="Social", - project_scores_matrix=project_scores_matrix, - partial_project_scores_matrix=partial_project_scores_matrix, - whlg_eligible_postcodes=whlg_eligible_postcodes, - eco4_social_cavity_abs_rate=12.5, - eco4_social_solid_abs_rate=17, - eco4_private_cavity_abs_rate=12.5, - eco4_private_solid_abs_rate=17, - gbis_social_cavity_abs_rate=21, - gbis_social_solid_abs_rate=25, - gbis_private_cavity_abs_rate=21, - gbis_private_solid_abs_rate=28, - ) - - li_thickness = convert_thickness_to_numeric( - p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"] - ) - current_wall_u_value = p.walls["thermal_transmittance"] - if current_wall_u_value is None: - current_wall_u_value = get_wall_u_value( - clean_description=p.walls["clean_description"], - age_band=p.age_band, - is_granite_or_whinstone=p.walls["is_granite_or_whinstone"], - is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"], - ) - - # We insert the innovation uplift - measures_to_optimise_with_uplift = deepcopy(measures_to_optimise) - - # TODO: Turn this into a function and store the innovaiton uplift - for group in measures_to_optimise_with_uplift: - for r in group: - - if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating", - "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]: - ( - r["partial_project_score"], - r["partial_project_funding"], - r["innovation_uplift"], - r["uplift_project_score"], - ) = ( - 0, 0, 0, 0 - ) - continue - - ( - r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], - r["uplift_project_score"] - ) = funding.get_innovation_uplift( - measure=r, - starting_sap=p.data["current-energy-efficiency"], - floor_area=p.floor_area, - is_cavity=p.walls["is_cavity_wall"], - current_wall_uvalue=current_wall_u_value, - is_partial="partial" in p.walls["clean_description"].lower(), - existing_li_thickness=li_thickness, - mainheating=p.main_heating, - main_fuel=p.main_fuel, - mainheat_energy_eff=p.data["mainheat-energy-eff"], - ) - - input_measures = optimiser_functions.prepare_input_measures( - measures_to_optimise_with_uplift, body.goal, needs_ventilation, funding=True - ) - - # When the goal is Increasing EPC, we can run the funding optimiser - if body.goal == "Increasing EPC": - - solutions = optimise_with_funding_paths( - p=p, - input_measures=input_measures, - housing_type=body.housing_type, - budget=body.budget, - target_gain=gain, - funding=funding - ) - - # Given the solutions we select the optimal one - solutions["cost_less_full_project_funding"] = np.where( - solutions["scheme"] == "eco4", - solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"], - solutions["total_cost"] - solutions["partial_project_funding"] - solutions["total_uplift"] - ) - - solutions["cost_less_full_project_funding"] = ( - solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"] - ) - solutions = solutions.sort_values("cost_less_full_project_funding", ascending=True) - - if solutions["meets_upgrade_target"].any(): - # If we have a solution that meets the upgrade target, we select that one - optimal_solution = solutions[solutions["meets_upgrade_target"]].iloc[0] - else: - # Pick the cheapest - optimal_solution = solutions.iloc[0] - - # This is the list of measures that we will recommend - scheme = optimal_solution["scheme"] - funded_measures = optimal_solution["items"] if scheme != "none" else [] - solution = optimal_solution["items"] + optimal_solution["unfunded_items"] - # This is the total amount of funding that the project will produce (including uplifts) (£) - project_funding = optimal_solution["full_project_funding"] if scheme == "eco4" else \ - optimal_solution["partial_project_funding"] - # This is the total amount of funding associated to the uplift (£) - total_uplift = optimal_solution["total_uplift"] - # This is the funding scheme selected - # This is the full project ABS - full_project_score = optimal_solution["project_score"] - # This is the partial project ABS - partial_project_score = optimal_solution["partial_project_score"] - # This is the uplift score ABS - uplift_project_score = optimal_solution["total_uplift_score"] - else: - # We optimise and then we determine eligibility for funding, based on the measures selected - optimiser = ( - GainOptimiser( - input_measures, max_cost=body.budget, max_gain=gain, allow_slack=False - ) if body.budget else CostOptimiser(input_measures, min_gain=gain) - ) - optimiser.setup() - optimiser.solve() - solution = optimiser.solution - - recommendation_types = [] - for measures in input_measures: - for measure in measures: - recommendation_types.append(measure["type"]) - recommendation_types = set(recommendation_types) - - has_wall_insulation_recommendation = any( - (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in - WALL_INSULATION_MEASURES - ) - has_roof_insulation_recommendation = any( - (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in - ROOF_INSULATION_MEASURES - ) - - funding.check_funding( - measures=solution, - starting_sap=p.data["current-energy-efficiency"], - ending_sap=p.data["current-energy-efficiency"] + sum([x["gain"] for x in solution]), - floor_area=p.floor_area, - mainheat_description=p.main_heating["clean_description"], - heating_control_description=p.main_heating_controls["clean_description"], - is_cavity=p.walls["is_cavity_wall"], - current_wall_uvalue=current_wall_u_value, - is_partial="partial" in p.walls["clean_description"].lower(), - existing_li_thickness=li_thickness, - mainheating=p.main_heating, - main_fuel=p.main_fuel, - mainheat_energy_eff=p.data["mainheat-energy-eff"], - has_wall_insulation_recommendation=has_wall_insulation_recommendation, - has_roof_insulation_recommendation=has_roof_insulation_recommendation, - ) - - # Determine the scheme - scheme = "none" - if funding.eco4_eligible: - scheme = "eco4" - if scheme == "none" and funding.gbis_eligible: - scheme = "gbis" - - funded_measures = solution if scheme in ["gbis", "eco4"] else [] - project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs - total_uplift = funding.eco4_uplift - full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs - partial_project_score = funding.partial_project_abs - uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift - - selected = {r["id"] for r in solution} - - if property_required_measures: - solution = optimiser_functions.add_required_measures( - property_id=p.id, property_required_measures=property_required_measures, - recommendations=recommendations, selected=selected, - ) - - # Add best practice measures (ventilation/trickle vents) - selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected) - # Final flattening - Don't do this! - # recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( - # p.id, recommendations, selected - # ) - - # TODO: functionise - for measure in funded_measures: - if "+mechanical_ventilation" in measure["type"]: - measure["type"] = measure["type"].split("+mechanical_ventilation")[0] - - p.insert_funding( - scheme=scheme, - funded_measures=funded_measures, - project_funding=project_funding, - total_uplift=total_uplift, - full_project_score=full_project_score, - partial_project_score=partial_project_score, - uplift_project_score=uplift_project_score - ) +# for col in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]: +# if pd.isnull(p.data[col]): +# min_diff = abs( +# (costs_by_floor_area["current-energy-efficiency"] - p.data["current-energy-efficiency"]) +# ).min() +# df = costs_by_floor_area[ +# abs((costs_by_floor_area["current-energy-efficiency"] - p.data[ +# "current-energy-efficiency"])) == min_diff +# ] +# if df.shape[0] > 1: +# df = df.head(1) +# p.data[col] = (df[col + "_scaled"] * p.data["total-floor-area"]).values[0] +# +# [ +# p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=mocked_kwh_predictions) for p in +# input_properties +# ] +# # for p in input_properties: +# # p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=mocked_kwh_predictions) +# +# # Run the recommendations +# recommendations = {} +# recommendations_scoring_data = [] +# representative_recommendations = {} +# for p in tqdm(input_properties): +# if p.data["property-type"] == "House" and pd.isnull(p.data["built-form"]): +# p.data["built-form"] = "Semi-Detached" +# recommender = Recommendations( +# property_instance=p, +# materials=materials, +# exclusions=[], +# inclusions=[], +# default_u_values=True +# ) +# property_recommendations, property_representative_recommendations = recommender.recommend() +# +# if not property_recommendations: +# continue +# +# recommendations[p.id] = property_recommendations +# representative_recommendations[p.id] = property_representative_recommendations +# +# p.create_base_difference_epc_record(cleaned_lookup=cleaned) +# p.adjust_difference_record_with_recommendations( +# property_recommendations, property_representative_recommendations +# ) +# +# recommendations_scoring_data.extend(p.recommendations_scoring_data) +# +# recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) +# recommendations_scoring_data = recommendations_scoring_data.drop( +# columns=[ +# "rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", +# "carbon_ending" +# ] +# ) +# +# model_predictions_mocked = { +# "sap_change_predictions": None, +# "heat_demand_predictions": None, +# "carbon_change_predictions": None, +# "heating_kwh_predictions": None, +# "hotwater_kwh_predictions": None, +# } +# +# for k in model_predictions_mocked.keys(): +# model_predictions_mocked[k] = recommendations_scoring_data[["id"]].copy() +# model_predictions_mocked[k][['property_id', 'recommendation_id']] = ( +# model_predictions_mocked[k]['id'].str.split('+', expand=True) +# ) +# model_predictions_mocked[k]['phase'] = model_predictions_mocked[k]['recommendation_id'].apply( +# ModelApi.extract_phase) +# +# if k in ["heating_kwh_predictions", "hotwater_kwh_predictions"]: +# model_predictions_mocked[k]["predictions"] = random.choices(range(100, 3000), +# k=len(recommendations_scoring_data)) +# continue +# +# model_predictions_mocked[k] = model_predictions_mocked[k].sort_values(["property_id", "phase"], ascending=True) +# preds = [] +# for p_id in model_predictions_mocked[k]["property_id"].unique(): +# # We add some amount each time +# p = [p for p in input_properties if str(p.id) == p_id][0] +# if k == "sap_change_predictions": +# start = p.data["current-energy-efficiency"] +# elif k == "heat_demand_predictions": +# start = p.data["energy-consumption-current"] +# else: +# start = p.data["co2-emissions-current"] +# df = model_predictions_mocked[k][model_predictions_mocked[k]["property_id"] == p_id].copy() +# # Add some amount each time +# to_add = random.choices(range(0, 15), k=len(df)) +# to_add = np.cumsum(to_add) +# df["predictions"] = start + to_add +# preds.append(df) +# preds = pd.concat(preds) +# model_predictions_mocked[k] = preds +# +# for property_id in tqdm(recommendations.keys(), total=len(recommendations)): +# property_instance = [p for p in input_properties if p.id == property_id][0] +# +# recommendations_with_impact, impact_summary = ( +# Recommendations.calculate_recommendation_impact( +# property_instance=property_instance, +# all_predictions=model_predictions_mocked, +# recommendations=recommendations, +# representative_recommendations=representative_recommendations +# ) +# ) +# +# # We use the impact_summary to update the simulation_epcs with the new SAP, heat demand, carbon, cost etc +# # at each phase +# property_instance.update_simulation_epcs(impact_summary) +# recommendations[property_id] = recommendations_with_impact +# +# for property_id in tqdm([p.id for p in input_properties]): +# property_recommendations = recommendations.get(property_id, []) +# property_instance = [p for p in input_properties if p.id == property_id][0] +# +# property_current_energy_bill = ( +# Recommendations.calculate_recommendation_tenant_savings( +# property_instance=property_instance, +# kwh_simulation_predictions=model_predictions_mocked, +# property_recommendations=property_recommendations, +# ashp_cop=2.8 +# ) +# ) +# property_instance.current_energy_bill = property_current_energy_bill +# +# body = PlanTriggerRequest( +# **{'budget': None, 'goal': 'Increasing EPC', 'housing_type': 'Social', 'goal_value': 'B', 'portfolio_id': 0, +# 'trigger_file_path': '', 'already_installed_file_path': '', +# 'patches_file_path': None, 'non_invasive_recommendations_file_path': None, +# 'valuation_file_path': '', +# 'required_measures': [], 'scenario_name': 'EPC B', 'scenario_id': None, +# 'multi_plan': True, 'optimise': True, 'default_u_values': True, 'ashp_cop': 2.8, +# 'event_type': 'remote_assessment', 'simulate_sap_10': False, 'file_type': None, 'file_format': None, +# 'sheet_name': None, 'sheet_count': None, 'index_start': None, 'index_end': None} +# ) +# +# for p in tqdm(input_properties): +# if not recommendations.get(p.id): +# continue +# +# # we need to double unlist because we have a list of lists +# property_measure_types = {rec["type"] for recs in recommendations[p.id] for rec in recs} +# property_required_measures = [m for m in recommendations[p.id] if m[0]["type"] in body.required_measures] +# measures_to_optimise = [m for m in recommendations[p.id] if m[0]["type"] not in body.required_measures] +# +# # If a measure requiring ventilation is selected, and the property does not have ventilation, we enfore +# # its inclusion +# needs_ventilation = any( +# x in property_measure_types for x in assumptions.measures_needing_ventilation +# ) and not p.has_ventilation +# +# if not measures_to_optimise: +# # Nothing to do, we just reshape the recommendations +# recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( +# p.id, recommendations, set() +# ) +# continue +# +# fixed_gain = optimiser_functions.calculate_fixed_gain( +# property_required_measures, recommendations, p, needs_ventilation +# ) +# gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain) +# +# funding = Funding( +# tenure="Social", +# project_scores_matrix=project_scores_matrix, +# partial_project_scores_matrix=partial_project_scores_matrix, +# whlg_eligible_postcodes=whlg_eligible_postcodes, +# eco4_social_cavity_abs_rate=12.5, +# eco4_social_solid_abs_rate=17, +# eco4_private_cavity_abs_rate=12.5, +# eco4_private_solid_abs_rate=17, +# gbis_social_cavity_abs_rate=21, +# gbis_social_solid_abs_rate=25, +# gbis_private_cavity_abs_rate=21, +# gbis_private_solid_abs_rate=28, +# ) +# +# li_thickness = convert_thickness_to_numeric( +# p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"] +# ) +# current_wall_u_value = p.walls["thermal_transmittance"] +# if current_wall_u_value is None: +# current_wall_u_value = get_wall_u_value( +# clean_description=p.walls["clean_description"], +# age_band=p.age_band, +# is_granite_or_whinstone=p.walls["is_granite_or_whinstone"], +# is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"], +# ) +# +# # We insert the innovation uplift +# measures_to_optimise_with_uplift = deepcopy(measures_to_optimise) +# +# # TODO: Turn this into a function and store the innovaiton uplift +# for group in measures_to_optimise_with_uplift: +# for r in group: +# +# if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating", +# "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]: +# ( +# r["partial_project_score"], +# r["partial_project_funding"], +# r["innovation_uplift"], +# r["uplift_project_score"], +# ) = ( +# 0, 0, 0, 0 +# ) +# continue +# +# ( +# r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], +# r["uplift_project_score"] +# ) = funding.get_innovation_uplift( +# measure=r, +# starting_sap=p.data["current-energy-efficiency"], +# floor_area=p.floor_area, +# is_cavity=p.walls["is_cavity_wall"], +# current_wall_uvalue=current_wall_u_value, +# is_partial="partial" in p.walls["clean_description"].lower(), +# existing_li_thickness=li_thickness, +# mainheating=p.main_heating, +# main_fuel=p.main_fuel, +# mainheat_energy_eff=p.data["mainheat-energy-eff"], +# ) +# +# input_measures = optimiser_functions.prepare_input_measures( +# measures_to_optimise_with_uplift, body.goal, needs_ventilation, funding=True +# ) +# +# # When the goal is Increasing EPC, we can run the funding optimiser +# if body.goal == "Increasing EPC": +# +# solutions = optimise_with_funding_paths( +# p=p, +# input_measures=input_measures, +# housing_type=body.housing_type, +# budget=body.budget, +# target_gain=gain, +# funding=funding +# ) +# +# # Given the solutions we select the optimal one +# solutions["cost_less_full_project_funding"] = np.where( +# solutions["scheme"] == "eco4", +# solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"], +# solutions["total_cost"] - solutions["partial_project_funding"] - solutions["total_uplift"] +# ) +# +# solutions["cost_less_full_project_funding"] = ( +# solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"] +# ) +# solutions = solutions.sort_values("cost_less_full_project_funding", ascending=True) +# +# if solutions["meets_upgrade_target"].any(): +# # If we have a solution that meets the upgrade target, we select that one +# optimal_solution = solutions[solutions["meets_upgrade_target"]].iloc[0] +# else: +# # Pick the cheapest +# optimal_solution = solutions.iloc[0] +# +# # This is the list of measures that we will recommend +# scheme = optimal_solution["scheme"] +# funded_measures = optimal_solution["items"] if scheme != "none" else [] +# solution = optimal_solution["items"] + optimal_solution["unfunded_items"] +# # This is the total amount of funding that the project will produce (including uplifts) (£) +# project_funding = optimal_solution["full_project_funding"] if scheme == "eco4" else \ +# optimal_solution["partial_project_funding"] +# # This is the total amount of funding associated to the uplift (£) +# total_uplift = optimal_solution["total_uplift"] +# # This is the funding scheme selected +# # This is the full project ABS +# full_project_score = optimal_solution["project_score"] +# # This is the partial project ABS +# partial_project_score = optimal_solution["partial_project_score"] +# # This is the uplift score ABS +# uplift_project_score = optimal_solution["total_uplift_score"] +# else: +# # We optimise and then we determine eligibility for funding, based on the measures selected +# optimiser = ( +# GainOptimiser( +# input_measures, max_cost=body.budget, max_gain=gain, allow_slack=False +# ) if body.budget else CostOptimiser(input_measures, min_gain=gain) +# ) +# optimiser.setup() +# optimiser.solve() +# solution = optimiser.solution +# +# recommendation_types = [] +# for measures in input_measures: +# for measure in measures: +# recommendation_types.append(measure["type"]) +# recommendation_types = set(recommendation_types) +# +# has_wall_insulation_recommendation = any( +# (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in +# WALL_INSULATION_MEASURES +# ) +# has_roof_insulation_recommendation = any( +# (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in +# ROOF_INSULATION_MEASURES +# ) +# +# funding.check_funding( +# measures=solution, +# starting_sap=p.data["current-energy-efficiency"], +# ending_sap=p.data["current-energy-efficiency"] + sum([x["gain"] for x in solution]), +# floor_area=p.floor_area, +# mainheat_description=p.main_heating["clean_description"], +# heating_control_description=p.main_heating_controls["clean_description"], +# is_cavity=p.walls["is_cavity_wall"], +# current_wall_uvalue=current_wall_u_value, +# is_partial="partial" in p.walls["clean_description"].lower(), +# existing_li_thickness=li_thickness, +# mainheating=p.main_heating, +# main_fuel=p.main_fuel, +# mainheat_energy_eff=p.data["mainheat-energy-eff"], +# has_wall_insulation_recommendation=has_wall_insulation_recommendation, +# has_roof_insulation_recommendation=has_roof_insulation_recommendation, +# ) +# +# # Determine the scheme +# scheme = "none" +# if funding.eco4_eligible: +# scheme = "eco4" +# if scheme == "none" and funding.gbis_eligible: +# scheme = "gbis" +# +# funded_measures = solution if scheme in ["gbis", "eco4"] else [] +# project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs +# total_uplift = funding.eco4_uplift +# full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs +# partial_project_score = funding.partial_project_abs +# uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift +# +# selected = {r["id"] for r in solution} +# +# if property_required_measures: +# solution = optimiser_functions.add_required_measures( +# property_id=p.id, property_required_measures=property_required_measures, +# recommendations=recommendations, selected=selected, +# ) +# +# # Add best practice measures (ventilation/trickle vents) +# selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected) +# # Final flattening - Don't do this! +# # recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( +# # p.id, recommendations, selected +# # ) +# +# # TODO: functionise +# for measure in funded_measures: +# if "+mechanical_ventilation" in measure["type"]: +# measure["type"] = measure["type"].split("+mechanical_ventilation")[0] +# +# p.insert_funding( +# scheme=scheme, +# funded_measures=funded_measures, +# project_funding=project_funding, +# total_uplift=total_uplift, +# full_project_score=full_project_score, +# partial_project_score=partial_project_score, +# uplift_project_score=uplift_project_score +# ) diff --git a/backend/tests/test_search_epc.py b/backend/tests/test_search_epc.py index 9bb7c39a..a0fef7e9 100644 --- a/backend/tests/test_search_epc.py +++ b/backend/tests/test_search_epc.py @@ -26,7 +26,7 @@ class TestSearchEpcIntegration: # Test case 2: Another valid address and postcode # In this case, the newest EPC, does not have a uprn associated to it. If we did a search by # uprn, we would get an old EPC - ("Flat 8, Hainton House", "DN32 9AQ", 10090082018, True, + ("Flat 8, Hainton House", "DN32 9AQ", "", True, "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 2), # Test case 3: When we make a request to the API for this property, we get back results for # flats 1, 2 and 3. We have some logic to handle the response so that we get back flat 1 @@ -56,7 +56,6 @@ class TestSearchEpcIntegration: # We check that we have the correct epc assert epc_searcher.newest_epc["lmk-key"] == lmk_key - assert epc_searcher.newest_epc["uprn"] == uprn assert len(epc_searcher.older_epcs) == n_old_epcs def test_search_housenumber(self): diff --git a/recommendations/tests/test_optimisers.py b/recommendations/tests/test_optimisers.py index df5cc2e1..e81aac69 100644 --- a/recommendations/tests/test_optimisers.py +++ b/recommendations/tests/test_optimisers.py @@ -144,6 +144,15 @@ class DummyProp: self.has_ventilation = False self.floor_area = 70.0 self.main_heating_controls = {"clean_description": "time and temperature zone control"} + self.walls = {'original_description': 'Solid brick, as built, no insulation (assumed)', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_cavity_wall': False, 'is_filled_cavity': False, + 'is_solid_brick': True, + 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, + 'is_as_built': True, + 'is_cob': False, 'is_assumed': True, 'is_sandstone_or_limestone': False, + 'insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False} self.main_heating = { 'original_description': 'Boiler and radiators, mains gas', @@ -230,6 +239,7 @@ def property_recommendations(): 'quantity_unit': 'm2', 'total': 19090.810139104888, 'labour_hours': 0.0, 'labour_days': 0.0}], 'type': 'external_wall_insulation', 'measure_type': 'external_wall_insulation', + "innovation_rate": 0, 'description': 'Install 150mm EWI Pro EPS external wall insulation system with Brick ' 'Slip finish on external walls', 'starting_u_value': 1.7, 'new_u_value': 0.32, 'already_installed': False, @@ -258,6 +268,7 @@ def property_recommendations(): 'quantity_unit': 'm2', 'total': 5694.929118083911, 'labour_hours': 134.37473199973275, 'labour_days': 4.199210374991648}], 'type': 'internal_wall_insulation', 'measure_type': 'internal_wall_insulation', + "innovation_rate": 0, 'description': 'Install 95mm ' 'SWIP EcoBatt & ' 'Plastered ' @@ -314,6 +325,7 @@ def property_recommendations(): 'quantity_unit': 'm2', 'total': 645.0, 'labour_hours': 8, 'labour_days': 1}], 'type': 'loft_insulation', 'measure_type': 'loft_insulation', + "innovation_rate": 0, 'description': 'Install 300mm of Knauf Loft Roll 44 glass fibre roll in your loft', 'starting_u_value': 2.3, 'new_u_value': 2.3, 'sap_points': np.float64(2.4), 'already_installed': False, @@ -338,6 +350,7 @@ def property_recommendations(): 'plant_cost': 0.0, 'total_cost': 350.0, 'notes': None, 'is_installer_quote': True, 'total': 700.0, 'quantity': 2, 'quantity_unit': 'part'}], 'type': 'mechanical_ventilation', 'measure_type': 'mechanical_ventilation', + "innovation_rate": 0, 'description': 'Install 2 ' 'Mechanical ' 'Extract ' @@ -387,6 +400,7 @@ def property_recommendations(): 'labour_hours': 70.08999999999999, 'labour_days': 2.920416666666666}], 'type': 'suspended_floor_insulation', 'measure_type': 'suspended_floor_insulation', + "innovation_rate": 0, 'description': 'Install 75mm Q-bot underfloor insulation insulation in suspended ' 'floor', 'starting_u_value': 0.83, 'new_u_value': 0.22, 'sap_points': 2, 'survey': True, @@ -401,6 +415,7 @@ def property_recommendations(): 'energy_cost_savings': np.float64(76.04936470588231)}], [ {'phase': 4, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting', + "innovation_rate": 0, 'description': 'Install low energy lighting in -886 outlets', 'starting_u_value': None, 'new_u_value': None, 'already_installed': False, 'sap_points': 2, 'kwh_savings': -48508.5, 'energy_cost_savings': -12481.237049999998, @@ -413,6 +428,7 @@ def property_recommendations(): 'recommendation_id': '5_phase=4', 'efficiency': -1705.5500000000002, 'heat_demand': np.float64(5.099999999999994)}], [ {'type': 'heating', 'phase': 5, 'measure_type': 'time_temperature_zone_control', + "innovation_rate": 0, 'parts': [], 'description': 'Upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control)', @@ -431,6 +447,7 @@ def property_recommendations(): 'energy_cost_savings': np.float64(65.29581176470589)}], [ {'phase': 6, 'parts': [], 'type': 'secondary_heating', 'measure_type': 'secondary_heating', + "innovation_rate": 0, 'description': 'Remove the secondary heating system', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(3.6), 'already_installed': False, 'total': 30.0, 'subtotal': 25.0, 'vat': 5.0, 'labour_hours': 3.0, @@ -443,6 +460,7 @@ def property_recommendations(): 'kwh_savings': np.float64(196.29999999999927), 'energy_cost_savings': np.float64(14.61857647058821)}], [ {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(13.0), 'already_installed': False, 'total': 6013.139999999999, 'subtotal': 5010.95, 'vat': 0, @@ -455,6 +473,7 @@ def property_recommendations(): 'kwh_savings': np.float64(2040.8566307499998), 'energy_cost_savings': np.float64(525.1124110919749)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(13.0), 'already_installed': False, 'total': 10537.008, 'subtotal': 8780.84, 'vat': 0, @@ -467,6 +486,7 @@ def property_recommendations(): 'kwh_savings': np.float64(2857.1992830499994), 'energy_cost_savings': np.float64(735.1573755287648)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 3.6 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(12.0), 'already_installed': False, 'total': 5826.491999999999, 'subtotal': 4855.41, 'vat': 0, @@ -478,6 +498,7 @@ def property_recommendations(): 'heat_demand': np.float64(83.69999999999999), 'kwh_savings': np.float64(1846.33397), 'energy_cost_savings': np.float64(475.0617304809999)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 3.6 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(12.0), 'already_installed': False, 'total': 10350.359999999999, 'subtotal': 8625.3, 'vat': 0, @@ -489,6 +510,7 @@ def property_recommendations(): 'heat_demand': np.float64(83.69999999999999), 'kwh_savings': np.float64(2584.867558), 'energy_cost_savings': np.float64(665.0864226734)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 3.2 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(11.0), 'already_installed': False, 'total': 5642.604, 'subtotal': 4702.17, 'vat': 0, @@ -500,6 +522,7 @@ def property_recommendations(): 'kwh_savings': np.float64(1650.2708274), 'energy_cost_savings': np.float64(424.61468389001993)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 3.2 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(11.0), 'already_installed': False, 'total': 10166.472, 'subtotal': 8472.06, 'vat': 0, @@ -511,6 +534,7 @@ def property_recommendations(): 'heat_demand': np.float64(78.3), 'kwh_savings': np.float64(2310.3791583599996), 'energy_cost_savings': np.float64(594.4605574460278)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.8 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(9.0), 'already_installed': False, 'total': 5458.727999999999, 'subtotal': 4548.94, 'vat': 0, @@ -522,6 +546,7 @@ def property_recommendations(): 'kwh_savings': np.float64(1453.5933906), 'energy_cost_savings': np.float64(374.00957940138)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.8 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(9.0), 'already_installed': False, 'total': 9982.596, 'subtotal': 8318.83, 'vat': 0, @@ -533,6 +558,7 @@ def property_recommendations(): 'heat_demand': np.float64(64.0), 'kwh_savings': np.float64(2035.03074684), 'energy_cost_savings': np.float64(523.6134111619319)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.4 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(8.0), 'already_installed': False, 'total': 5274.852, 'subtotal': 4395.71, 'vat': 0, @@ -544,6 +570,7 @@ def property_recommendations(): 'kwh_savings': np.float64(1255.12594), 'energy_cost_savings': np.float64(322.94390436199996)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.4 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(8.0), 'already_installed': False, 'total': 9798.72, 'subtotal': 8165.6, 'vat': 0, @@ -555,6 +582,7 @@ def property_recommendations(): 'heat_demand': np.float64(54.3), 'kwh_savings': np.float64(1757.1763159999998), 'energy_cost_savings': np.float64(452.1214661067999)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.0 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(7.0), 'already_installed': False, 'total': 5090.976, 'subtotal': 4242.48, 'vat': 0, @@ -566,6 +594,7 @@ def property_recommendations(): 'kwh_savings': np.float64(1048.341318), 'energy_cost_savings': np.float64(269.7382211214)}, {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + "innovation_rate": 0, 'description': 'Install a 2.0 kilowatt-peak (kWp) solar panel system, with a battery.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(7.0), 'already_installed': False, 'total': 9614.844, 'subtotal': 8012.369999999999, 'vat': 0, @@ -586,10 +615,20 @@ def _attach_costs_and_uplifts(recs, funding, p): for group in out: for r in group: if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating"]: - r["innovation_uplift"] = 0 + ( + r["partial_project_score"], + r["partial_project_funding"], + r["innovation_uplift"], + r["uplift_project_score"], + ) = ( + 0, 0, 0, 0 + ) continue - r["uplift"] = 0.0 # fixed for determinism in test - r["innovation_uplift"] = funding.get_innovation_uplift( + + ( + r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], + r["uplift_project_score"] + ) = funding.get_innovation_uplift( measure=r, starting_sap=55, floor_area=70.0, @@ -663,3 +702,100 @@ def test_social_fabric_only_returns_only_fabric_types(p, funding, property_recom unfunded_rows = solutions[ solutions["path"].apply(lambda x: isinstance(x, dict) and x.get("reference") == "unfunded:all")] assert not unfunded_rows.empty + + +def test_private_solid_wall_no_innovation_epc_d(p, funding, mock_project_scores_matrix, mock_partial_scores_matrix): + """ + We have a specific test for this case which was implemented incorrectly originally. + This is an EPC D property and so shouldn't be eligible for ECO4. Instead, only GBIS should be considered. + """ + + # Overwrite the data - copied from real example + p2 = deepcopy(p) + p2.data = { + "current-energy-rating": "D", + "current-energy-efficiency": 68, + "mainheat-energy-eff": "Good", + } + p2.walls = {'original_description': 'Sandstone or limestone, as built, no insulation (assumed)', + 'clean_description': 'Sandstone or limestone, as built, no insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_cavity_wall': False, 'is_filled_cavity': False, + 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_assumed': True, + 'is_sandstone_or_limestone': True, 'is_park_home': False, 'insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False} + + funding2 = Funding( + tenure="Private", + project_scores_matrix=mock_project_scores_matrix, + partial_project_scores_matrix=mock_partial_scores_matrix, + whlg_eligible_postcodes=pd.DataFrame([{"Postcode": "ab12cd"}]), + eco4_social_cavity_abs_rate=12.5, + eco4_social_solid_abs_rate=17, + eco4_private_cavity_abs_rate=12.5, + eco4_private_solid_abs_rate=17, + gbis_social_cavity_abs_rate=21, + gbis_social_solid_abs_rate=25, + gbis_private_cavity_abs_rate=21, + gbis_private_solid_abs_rate=28, + ) + + input_measures = [ + [{'id': '0_phase=0', 'cost': np.float64(4441.202499013676), 'gain': np.float64(3.4000000000000057), + 'type': 'internal_wall_insulation+mechanical_ventilation', 'innovation_uplift': np.float64(0.0), + 'cost_minus_uplift': np.float64(4441.202499013676), 'raw_cost': 3881.2024990136756, + 'partial_project_funding': np.float64(2300.1000000000004), 'partial_project_score': np.float64(135.3), + 'uplift_project_score': np.float64(0.0)}], [ + {'id': '2_phase=2', 'cost': np.float64(2280.0), 'gain': np.float64(0.4), 'type': 'secondary_glazing', + 'innovation_uplift': np.float64(0.0), 'cost_minus_uplift': np.float64(2280.0), + 'raw_cost': np.float64(2280.0), 'partial_project_funding': np.float64(1421.1999999999998), + 'partial_project_score': np.float64(83.6), 'uplift_project_score': np.float64(0.0)}], [ + {'id': '3_phase=3', 'cost': np.float64(604.5840000000001), 'gain': np.float64(1.2), + 'type': 'time_temperature_zone_control', 'innovation_uplift': np.float64(0.0), + 'cost_minus_uplift': np.float64(604.5840000000001), 'raw_cost': 604.5840000000001, + 'partial_project_funding': np.float64(702.0999999999999), 'partial_project_score': np.float64(41.3), + 'uplift_project_score': np.float64(0.0)}], [ + {'id': '4_phase=4', 'cost': 60.0, 'gain': np.float64(0.0), 'type': 'secondary_heating', + 'innovation_uplift': 0, 'cost_minus_uplift': 60.0, 'raw_cost': 60.0, 'partial_project_funding': 0, + 'partial_project_score': 0, 'uplift_project_score': 0}] + ] + + solutions = optimise_with_funding_paths( + p=p2, + input_measures=input_measures, + housing_type="Private", + budget=None, + target_gain=1.5, + funding=funding2 + ) + + # 3) basic shape assertions + assert isinstance(solutions, pd.DataFrame) + assert not solutions.empty + + # We should have 2 rows + assert solutions.shape[0] == 2 + + # We should only have None or GBIS + assert set(solutions["scheme"].unique()) == {"none", "gbis"} + + meets_upgrade_gbis = solutions[solutions["meets_upgrade_target"] & solutions["is_eligible"]] + assert meets_upgrade_gbis.shape[0] == 1 + + # Check exact result + assert meets_upgrade_gbis.squeeze().to_dict() == { + 'fixed_ids': ['0_phase=0'], 'items': [ + {'id': '0_phase=0', 'cost': 3881.2024990136756, 'gain': np.float64(3.4000000000000057), + 'type': 'internal_wall_insulation+mechanical_ventilation', 'innovation_uplift': np.float64(0.0), + 'cost_minus_uplift': np.float64(4441.202499013676), 'raw_cost': 3881.2024990136756, + 'partial_project_funding': np.float64(2300.1000000000004), 'partial_project_score': np.float64(135.3), + 'uplift_project_score': np.float64(0.0)}], 'total_cost': 3881.2024990136756, + 'total_gain': 3.4000000000000057, 'path': [{'AND': ['internal_wall_insulation+mechanical_ventilation'], + 'reference': + 'internal_wall_insulation+mechanical_ventilation:gbis'}], + 'scheme': 'gbis', 'is_eligible': True, 'unfunded_items': [], 'meets_upgrade_target': True, 'starting_sap': 68, + 'floor_area': 70.0, 'ending_sap': 71.4, 'starting_band': 'High_D', 'ending_band': 'Low_C', + 'floor_area_band': '0-72', 'project_score': 540.0, 'full_project_funding': 0.0, + 'partial_project_funding': 2300.1000000000004, 'partial_project_score': 135.3, 'total_uplift': 0.0, + 'total_uplift_score': 0.0 + }