From b632823ca2035788ac5d4a96a05f55ecae4a2711 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 29 Aug 2025 01:34:06 +0100 Subject: [PATCH] preparing integration test --- backend/app/assumptions.py | 5 + backend/tests/test_integration.py | 320 ++++++++++++++++++++++++++ recommendations/HeatingRecommender.py | 8 + recommendations/Recommendations.py | 1 + 4 files changed, 334 insertions(+) diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index d813e1a9..a0234f75 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -73,6 +73,11 @@ DESCRIPTIONS_TO_FUEL_TYPES = { "Electric storage heaters, Room heaters, electric": {"fuel": "Electricity", "cop": 1}, 'Boiler and underfloor heating, oil': {"fuel": "Oil", "cop": 0.85}, "Boiler and radiators, smokeless fuel": {"fuel": "Smokeless Fuel", "cop": 0.85}, + "Boiler and radiators, mains gas, Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.85}, + "Electric ceiling heating, electric": {"fuel": "Electricity", "cop": 1}, + "Air source heat pump, warm air, electric": { + "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 + } } # These are the measure types where if there is a ventilation recommendation, we force the inclusion of it diff --git a/backend/tests/test_integration.py b/backend/tests/test_integration.py index 9071ac78..e6bcfce8 100644 --- a/backend/tests/test_integration.py +++ b/backend/tests/test_integration.py @@ -1,6 +1,7 @@ import ast import json from copy import deepcopy +from dataclasses import replace from datetime import datetime import random @@ -209,3 +210,322 @@ for p in tqdm(input_properties): ) recommendations_scoring_data.extend(p.recommendations_scoring_data) + +recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) +recommendations_scoring_data = recommendations_scoring_data.drop( + columns=[ + "rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", + "carbon_ending" + ] +) + +model_predictions_mocked = { + "sap_change_predictions": None, + "heat_demand_predictions": None, + "carbon_change_predictions": None, + "heating_kwh_predictions": None, + "hotwater_kwh_predictions": None, +} + +for k in model_predictions_mocked.keys(): + model_predictions_mocked[k] = recommendations_scoring_data[["id"]].copy() + model_predictions_mocked[k][['property_id', 'recommendation_id']] = ( + model_predictions_mocked[k]['id'].str.split('+', expand=True) + ) + model_predictions_mocked[k]['phase'] = model_predictions_mocked[k]['recommendation_id'].apply( + ModelApi.extract_phase) + + if k in ["heating_kwh_predictions", "hotwater_kwh_predictions"]: + model_predictions_mocked[k]["predictions"] = random.choices(range(100, 3000), + k=len(recommendations_scoring_data)) + continue + + model_predictions_mocked[k] = model_predictions_mocked[k].sort_values(["property_id", "phase"], ascending=True) + preds = [] + for p_id in model_predictions_mocked[k]["property_id"].unique(): + # We add some amount each time + p = [p for p in input_properties if str(p.id) == p_id][0] + if k == "sap_change_predictions": + start = p.data["current-energy-efficiency"] + elif k == "heat_demand_predictions": + start = p.data["energy-consumption-current"] + else: + start = p.data["co2-emissions-current"] + df = model_predictions_mocked[k][model_predictions_mocked[k]["property_id"] == p_id].copy() + # Add some amount each time + to_add = random.choices(range(0, 15), k=len(df)) + to_add = np.cumsum(to_add) + df["predictions"] = start + to_add + preds.append(df) + preds = pd.concat(preds) + model_predictions_mocked[k] = preds + +for property_id in tqdm(recommendations.keys(), total=len(recommendations)): + property_instance = [p for p in input_properties if p.id == property_id][0] + + recommendations_with_impact, impact_summary = ( + Recommendations.calculate_recommendation_impact( + property_instance=property_instance, + all_predictions=model_predictions_mocked, + recommendations=recommendations, + representative_recommendations=representative_recommendations + ) + ) + + # We use the impact_summary to update the simulation_epcs with the new SAP, heat demand, carbon, cost etc + # at each phase + property_instance.update_simulation_epcs(impact_summary) + recommendations[property_id] = recommendations_with_impact + +for property_id in tqdm([p.id for p in input_properties]): + property_recommendations = recommendations.get(property_id, []) + property_instance = [p for p in input_properties if p.id == property_id][0] + + property_current_energy_bill = ( + Recommendations.calculate_recommendation_tenant_savings( + property_instance=property_instance, + kwh_simulation_predictions=model_predictions_mocked, + property_recommendations=property_recommendations, + ashp_cop=2.8 + ) + ) + property_instance.current_energy_bill = property_current_energy_bill + +body = PlanTriggerRequest( + **{'budget': None, 'goal': 'Increasing EPC', 'housing_type': 'Social', 'goal_value': 'B', 'portfolio_id': 0, + 'trigger_file_path': '', 'already_installed_file_path': '', + 'patches_file_path': None, 'non_invasive_recommendations_file_path': None, + 'valuation_file_path': '', + 'required_measures': [], 'scenario_name': 'EPC B', 'scenario_id': None, + 'multi_plan': True, 'optimise': True, 'default_u_values': True, 'ashp_cop': 2.8, + 'event_type': 'remote_assessment', 'simulate_sap_10': False, 'file_type': None, 'file_format': None, + 'sheet_name': None, 'sheet_count': None, 'index_start': None, 'index_end': None} +) + +for p in tqdm(input_properties): + if not recommendations.get(p.id): + continue + + # we need to double unlist because we have a list of lists + property_measure_types = {rec["type"] for recs in recommendations[p.id] for rec in recs} + property_required_measures = [m for m in recommendations[p.id] if m[0]["type"] in body.required_measures] + measures_to_optimise = [m for m in recommendations[p.id] if m[0]["type"] not in body.required_measures] + + # If a measure requiring ventilation is selected, and the property does not have ventilation, we enfore + # its inclusion + needs_ventilation = any( + x in property_measure_types for x in assumptions.measures_needing_ventilation + ) and not p.has_ventilation + + if not measures_to_optimise: + # Nothing to do, we just reshape the recommendations + recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( + p.id, recommendations, set() + ) + continue + + fixed_gain = optimiser_functions.calculate_fixed_gain( + property_required_measures, recommendations, p, needs_ventilation + ) + gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain) + + funding = Funding( + tenure="Social", + project_scores_matrix=project_scores_matrix, + partial_project_scores_matrix=partial_project_scores_matrix, + whlg_eligible_postcodes=whlg_eligible_postcodes, + eco4_social_cavity_abs_rate=12.5, + eco4_social_solid_abs_rate=17, + eco4_private_cavity_abs_rate=12.5, + eco4_private_solid_abs_rate=17, + gbis_social_cavity_abs_rate=21, + gbis_social_solid_abs_rate=25, + gbis_private_cavity_abs_rate=21, + gbis_private_solid_abs_rate=28, + ) + + li_thickness = convert_thickness_to_numeric( + p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"] + ) + current_wall_u_value = p.walls["thermal_transmittance"] + if current_wall_u_value is None: + current_wall_u_value = get_wall_u_value( + clean_description=p.walls["clean_description"], + age_band=p.age_band, + is_granite_or_whinstone=p.walls["is_granite_or_whinstone"], + is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"], + ) + + # We insert the innovation uplift + measures_to_optimise_with_uplift = deepcopy(measures_to_optimise) + + # TODO: Turn this into a function and store the innovaiton uplift + for group in measures_to_optimise_with_uplift: + for r in group: + + if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating", + "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]: + ( + r["partial_project_score"], + r["partial_project_funding"], + r["innovation_uplift"], + r["uplift_project_score"], + ) = ( + 0, 0, 0, 0 + ) + continue + + ( + r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], + r["uplift_project_score"] + ) = funding.get_innovation_uplift( + measure=r, + starting_sap=p.data["current-energy-efficiency"], + floor_area=p.floor_area, + is_cavity=p.walls["is_cavity_wall"], + current_wall_uvalue=current_wall_u_value, + is_partial="partial" in p.walls["clean_description"].lower(), + existing_li_thickness=li_thickness, + mainheating=p.main_heating, + main_fuel=p.main_fuel, + mainheat_energy_eff=p.data["mainheat-energy-eff"], + ) + + input_measures = optimiser_functions.prepare_input_measures( + measures_to_optimise_with_uplift, body.goal, needs_ventilation, funding=True + ) + + # When the goal is Increasing EPC, we can run the funding optimiser + if body.goal == "Increasing EPC": + + solutions = optimise_with_funding_paths( + p=p, + input_measures=input_measures, + housing_type=body.housing_type, + budget=body.budget, + target_gain=gain, + funding=funding + ) + + # Given the solutions we select the optimal one + solutions["cost_less_full_project_funding"] = np.where( + solutions["scheme"] == "eco4", + solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"], + solutions["total_cost"] - solutions["partial_project_funding"] - solutions["total_uplift"] + ) + + solutions["cost_less_full_project_funding"] = ( + solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"] + ) + solutions = solutions.sort_values("cost_less_full_project_funding", ascending=True) + + if solutions["meets_upgrade_target"].any(): + # If we have a solution that meets the upgrade target, we select that one + optimal_solution = solutions[solutions["meets_upgrade_target"]].iloc[0] + else: + # Pick the cheapest + optimal_solution = solutions.iloc[0] + + # This is the list of measures that we will recommend + scheme = optimal_solution["scheme"] + funded_measures = optimal_solution["items"] if scheme != "none" else [] + solution = optimal_solution["items"] + optimal_solution["unfunded_items"] + # This is the total amount of funding that the project will produce (including uplifts) (£) + project_funding = optimal_solution["full_project_funding"] if scheme == "eco4" else \ + optimal_solution["partial_project_funding"] + # This is the total amount of funding associated to the uplift (£) + total_uplift = optimal_solution["total_uplift"] + # This is the funding scheme selected + # This is the full project ABS + full_project_score = optimal_solution["project_score"] + # This is the partial project ABS + partial_project_score = optimal_solution["partial_project_score"] + # This is the uplift score ABS + uplift_project_score = optimal_solution["total_uplift_score"] + else: + # We optimise and then we determine eligibility for funding, based on the measures selected + optimiser = ( + GainOptimiser( + input_measures, max_cost=body.budget, max_gain=gain, allow_slack=False + ) if body.budget else CostOptimiser(input_measures, min_gain=gain) + ) + optimiser.setup() + optimiser.solve() + solution = optimiser.solution + + recommendation_types = [] + for measures in input_measures: + for measure in measures: + recommendation_types.append(measure["type"]) + recommendation_types = set(recommendation_types) + + has_wall_insulation_recommendation = any( + (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in + WALL_INSULATION_MEASURES + ) + has_roof_insulation_recommendation = any( + (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in + ROOF_INSULATION_MEASURES + ) + + funding.check_funding( + measures=solution, + starting_sap=p.data["current-energy-efficiency"], + ending_sap=p.data["current-energy-efficiency"] + sum([x["gain"] for x in solution]), + floor_area=p.floor_area, + mainheat_description=p.main_heating["clean_description"], + heating_control_description=p.main_heating_controls["clean_description"], + is_cavity=p.walls["is_cavity_wall"], + current_wall_uvalue=current_wall_u_value, + is_partial="partial" in p.walls["clean_description"].lower(), + existing_li_thickness=li_thickness, + mainheating=p.main_heating, + main_fuel=p.main_fuel, + mainheat_energy_eff=p.data["mainheat-energy-eff"], + has_wall_insulation_recommendation=has_wall_insulation_recommendation, + has_roof_insulation_recommendation=has_roof_insulation_recommendation, + ) + + # Determine the scheme + scheme = "none" + if funding.eco4_eligible: + scheme = "eco4" + if scheme == "none" and funding.gbis_eligible: + scheme = "gbis" + + funded_measures = solution if scheme in ["gbis", "eco4"] else [] + project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs + total_uplift = funding.eco4_uplift + full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs + partial_project_score = funding.partial_project_abs + uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift + + selected = {r["id"] for r in solution} + + if property_required_measures: + solution = optimiser_functions.add_required_measures( + property_id=p.id, property_required_measures=property_required_measures, + recommendations=recommendations, selected=selected, + ) + + # Add best practice measures (ventilation/trickle vents) + selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected) + # Final flattening - Don't do this! + # recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults( + # p.id, recommendations, selected + # ) + + # TODO: functionise + for measure in funded_measures: + if "+mechanical_ventilation" in measure["type"]: + measure["type"] = measure["type"].split("+mechanical_ventilation")[0] + + p.insert_funding( + scheme=scheme, + funded_measures=funded_measures, + project_funding=project_funding, + total_uplift=total_uplift, + full_project_score=full_project_score, + partial_project_score=partial_project_score, + uplift_project_score=uplift_project_score + ) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index d2bccbcc..73edff53 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -82,6 +82,14 @@ class HeatingRecommender: "controls_prefix": "" }, "dual": None + }, + 'Electric storage heaters, room heaters, electric': { + "hhr": { + "mainheating_description": "Electric storage heaters, radiators", + "recommendation_description": "Install high heat retention electric storage heaters.", + "controls_prefix": "" + }, + "dual": None } } diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 614e4a4a..fa8fe256 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -693,6 +693,7 @@ class Recommendations: if hotwater_description in [ "From main system", "From main system, no cylinder thermostat", + 'From main system, waste water heat recovery' ]: return { "heating_fuel_type": heating_fuel, "hotwater_fuel_type": heating_fuel,