completed funding tests

This commit is contained in:
Khalim Conn-Kowlessar 2025-08-11 16:19:58 +01:00
parent f355f12ed3
commit 97a6a27a15
13 changed files with 710 additions and 269 deletions

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="epc_clean" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="epc_clean" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>

View file

@ -7,6 +7,7 @@ from backend.app.plan.schemas import HousingType, WALL_INSULATION_MEASURES, ROOF
class EligibilityCaveats(Enum):
EPC_RATING = "epc_rating" # EPC requirements not met
TENANT_ON_BENEFITS_OR_LOW_INCOME = "tenant_on_benefits_or_low_income"
INNOVATION_REQUIRED = "innovation_required"
SOLAR_NEEDS_HEATING = "solar_needs_heating"
@ -26,19 +27,27 @@ class Funding:
def __init__(
self,
tenure: str, # 'Private' or 'Social'
social_cavity_abs_rate: float,
social_solid_abs_rate: float,
private_cavity_abs_rate: float,
private_solid_abs_rate: float,
eco4_social_cavity_abs_rate: float,
eco4_social_solid_abs_rate: float,
eco4_private_cavity_abs_rate: float,
eco4_private_solid_abs_rate: float,
gbis_social_cavity_abs_rate: float,
gbis_social_solid_abs_rate: float,
gbis_private_cavity_abs_rate: float,
gbis_private_solid_abs_rate: float,
project_scores_matrix,
partial_project_scores_matrix,
whlg_eligible_postcodes
):
self.tenure = tenure
self.social_cavity_abs_rate = social_cavity_abs_rate
self.social_solid_abs_rate = social_solid_abs_rate
self.private_cavity_abs_rate = private_cavity_abs_rate
self.private_solid_abs_rate = private_solid_abs_rate
self.eco4_social_cavity_abs_rate = eco4_social_cavity_abs_rate
self.eco4_social_solid_abs_rate = eco4_social_solid_abs_rate
self.eco4_private_cavity_abs_rate = eco4_private_cavity_abs_rate
self.eco4_private_solid_abs_rate = eco4_private_solid_abs_rate
self.gbis_social_cavity_abs_rate = gbis_social_cavity_abs_rate
self.gbis_social_solid_abs_rate = gbis_social_solid_abs_rate
self.gbis_private_cavity_abs_rate = gbis_private_cavity_abs_rate
self.gbis_private_solid_abs_rate = gbis_private_solid_abs_rate
self.starting_sap_band = None
self.ending_sap_band = None
@ -55,6 +64,7 @@ class Funding:
# Funding calculation variables
self.full_project_abs = None
self.gbis_funding = None
self.eco4_funding = None
self.eco4_uplift = 0
@ -141,7 +151,7 @@ class Funding:
if not meets_epc or not meets_upgrade_target:
self.eco4_eligible = False
self.eco4_eligibility_caveats = []
self.eco4_eligibility_caveats.append(EligibilityCaveats.EPC_RATING)
return
if has_solar and not solar_eligible:
@ -587,6 +597,22 @@ class Funding:
raise ValueError("something went wrong, more than one pps for ashp")
return pps.squeeze()["Cost Savings"]
if measure_type == "high_heat_retention_storage_heater":
pps_data = filtered_pps_matrix[
filtered_pps_matrix["Post_Main_Heating_Source"] == "High Heat Retention Storage Heaters"
]
# Not every heating upgrade, that ends at HHRSH, will have a PPS. E.g. a gas boiler to HHRSH upgrade
# doesn't have a PPS
if pre_heating_system in pps_data["Pre_Main_Heating_Source"].values:
pps = pps_data[
pps_data["Pre_Main_Heating_Source"] == pre_heating_system
]
if pps.shape[0] != 1:
raise ValueError("something went wrong, more than one pps for HHRSH")
return pps.squeeze()["Cost Savings"]
return 0
raise ValueError(f"Invalid measure type for partial project ABS calculation: {measure_type}")
# -----------------------
@ -752,6 +778,40 @@ class Funding:
return True
def calc_innovation_uplift(
self,
measure_types,
innovation_flags,
uplifts,
filtered_pps_matrix,
pre_heating_system,
mainheating,
main_fuel,
mainheat_energy_eff,
current_wall_uvalue,
is_partial,
existing_li_thickness,
):
"""Wrapper fundgion to calculate the innovation uplift for a project."""
project_uplifts = []
for i, measure in enumerate(measure_types):
if not innovation_flags[i]:
project_uplifts.append(0)
continue
pps = self.calculate_partial_project_abs(
measure_type=measure,
mainheating=mainheating,
main_fuel=main_fuel,
mainheat_energy_eff=mainheat_energy_eff,
current_wall_uvalue=current_wall_uvalue,
is_partial=is_partial,
existing_li_thickness=existing_li_thickness,
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system
)
project_uplifts.append(pps * uplifts[i])
return sum(project_uplifts)
def check_funding(
self,
measures: List[dict],
@ -835,12 +895,43 @@ class Funding:
self.gbis_prs_eligibility(starting_sap, council_tax_band or "", measure_types)
if self.eco4_eligible:
# Calculate the full project ABS for ECO4
self.full_project_abs = self.calculate_full_project_abs()
self.eco4_uplift = self.calc_innovation_uplift(
measure_types=measure_types,
innovation_flags=innovation_flags,
uplifts=uplifts,
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system,
mainheating=mainheating,
main_fuel=main_fuel,
mainheat_energy_eff=mainheat_energy_eff,
current_wall_uvalue=current_wall_uvalue,
is_partial=is_partial,
existing_li_thickness=existing_li_thickness,
)
self.full_project_abs += self.eco4_uplift
self.eco4_funding = self.full_project_abs * (
self.private_cavity_abs_rate if is_cavity else self.private_solid_abs_rate)
self.eco4_social_cavity_abs_rate if is_cavity else self.eco4_social_solid_abs_rate
)
if self.gbis_eligible:
raise NotImplementedError("FIX ME")
self.partial_project_abs = self.calculate_partial_project_abs(
measure_type=measure_types[0],
mainheating=mainheating,
main_fuel=main_fuel,
mainheat_energy_eff=mainheat_energy_eff,
current_wall_uvalue=current_wall_uvalue,
is_partial=is_partial,
existing_li_thickness=existing_li_thickness,
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system
)
self.gbis_funding = self.partial_project_abs * (
self.gbis_private_cavity_abs_rate if is_cavity else self.gbis_private_solid_abs_rate
)
elif self.tenure == "Social":
# ECO4 Social
@ -855,30 +946,23 @@ class Funding:
# Calculate the full project ABS for ECO4
self.full_project_abs = self.calculate_full_project_abs()
# We calculate uplift innovation, where required
project_uplifts = []
for i, measure in enumerate(measure_types):
if not innovation_flags[i]:
# Capture 0 innovation uplift for debugging
project_uplifts.append(0)
continue
self.eco4_uplift = self.calc_innovation_uplift(
measure_types=measure_types,
innovation_flags=innovation_flags,
uplifts=uplifts,
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system,
mainheating=mainheating,
main_fuel=main_fuel,
mainheat_energy_eff=mainheat_energy_eff,
current_wall_uvalue=current_wall_uvalue,
is_partial=is_partial,
existing_li_thickness=existing_li_thickness,
)
pps = self.calculate_partial_project_abs(
measure_type=measure,
mainheating=mainheating,
main_fuel=main_fuel,
mainheat_energy_eff=mainheat_energy_eff,
current_wall_uvalue=current_wall_uvalue,
is_partial=is_partial,
existing_li_thickness=existing_li_thickness,
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system
)
project_uplifts.append(pps * uplifts[i])
self.eco4_uplift = sum(project_uplifts)
self.full_project_abs += self.eco4_uplift
self.eco4_funding = self.full_project_abs * (
self.social_cavity_abs_rate if is_cavity else self.social_solid_abs_rate
self.eco4_social_cavity_abs_rate if is_cavity else self.eco4_social_solid_abs_rate
)
if self.gbis_eligible:
@ -894,6 +978,9 @@ class Funding:
filtered_pps_matrix=filtered_pps_matrix,
pre_heating_system=pre_heating_system
)
self.gbis_funding = self.partial_project_abs * (
self.gbis_social_cavity_abs_rate if is_cavity else self.gbis_social_solid_abs_rate
)
else:

View file

@ -1,10 +1,10 @@
from backend.Funding import Funding, EligibilityCaveats
from backend.Funding import EligibilityCaveats
innovation_scenarios = [
# 1) Innovation PV, non-eligible heating system in place, EPC D - not eligible
{
"description": "Innovation PV, non-eligible heating system in place, EPC D",
"measures": [{"type": "solar_pv", "is_innovation": True}],
"measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
"starting_sap": 60,
"mainheat_description": "Electric storage heaters",
"heating_control_description": "Manual charge control",
@ -16,7 +16,7 @@ innovation_scenarios = [
# 2) Innovation PV, eligible heating system in place, EPC D - eligible
{
"description": "Innovation PV, eligible heating system in place, EPC D",
"measures": [{"type": "solar_pv", "is_innovation": True}],
"measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
"heating_control_description": "Programmer, room thermostat and TRVs",
@ -29,8 +29,8 @@ innovation_scenarios = [
{
"description": "Innovation PV + HHRSH upgrade, EPC E",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "high_heat_retention_storage_heater", "is_innovation": True}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1}
],
"starting_sap": 50,
"mainheat_description": "Electric storage heaters",
@ -44,8 +44,8 @@ innovation_scenarios = [
{
"description": "Innovation PV + HHRSH upgrade, EPC E",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "high_heat_retention_storage_heater", "is_innovation": True}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1}
],
"starting_sap": 50,
"mainheat_description": "Electric storage heaters",
@ -58,7 +58,7 @@ innovation_scenarios = [
# 5) Innovation PV, needs wall insulation, no wall insulation measure - not eligible
{
"description": "Innovation PV, wall insulation recommended, but not installed",
"measures": [{"type": "solar_pv", "is_innovation": True}],
"measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
"heating_control_description": "Programmer, room thermostat and TRVs",
@ -71,8 +71,8 @@ innovation_scenarios = [
{
"description": "Innovation PV, wall insulation recommended and installed",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "internal_wall_insulation", "is_innovation": False}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25}
],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
@ -85,7 +85,7 @@ innovation_scenarios = [
# 7) Innovation PV, needs roof insulation, no roof insulation measure - not eligible
{
"description": "Innovation PV, roof insulation recommended, not installed",
"measures": [{"type": "solar_pv", "is_innovation": True}],
"measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
"heating_control_description": "Programmer, room thermostat and TRVs",
@ -98,8 +98,8 @@ innovation_scenarios = [
{
"description": "Innovation PV, roof insulation recommended and installed",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "loft_insulation", "is_innovation": False}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "loft_insulation", "is_innovation": False, "uplift": 0}
],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
@ -112,7 +112,7 @@ innovation_scenarios = [
# 9) Innovation PV, needs both roof + wall insulation, no insulation - not eligible
{
"description": "Innovation PV, both insulations recommended, none installed",
"measures": [{"type": "solar_pv", "is_innovation": True}],
"measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
"heating_control_description": "Programmer, room thermostat and TRVs",
@ -125,8 +125,8 @@ innovation_scenarios = [
{
"description": "Innovation PV, both insulations recommended, only wall done",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "internal_wall_insulation", "is_innovation": False}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25}
],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
@ -140,8 +140,8 @@ innovation_scenarios = [
{
"description": "Innovation PV, both insulations recommended, only roof done",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "loft_insulation", "is_innovation": False}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "loft_insulation", "is_innovation": False, "uplift": 0}
],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",
@ -155,9 +155,9 @@ innovation_scenarios = [
{
"description": "Innovation PV, both insulations recommended and installed",
"measures": [
{"type": "solar_pv", "is_innovation": True},
{"type": "internal_wall_insulation", "is_innovation": False},
{"type": "loft_insulation", "is_innovation": False}
{"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
{"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25},
{"type": "loft_insulation", "is_innovation": False, "uplift": 0}
],
"starting_sap": 60,
"mainheat_description": "Air source heat pump, radiators",

File diff suppressed because it is too large Load diff

View file

@ -40,28 +40,35 @@ def app():
cleaned_data = {}
epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
errors = []
for directory in tqdm(epc_directories):
data = pd.read_csv(directory / "certificates.csv", low_memory=False)
# Rename the columns to the same format as the api returns
data.columns = [c.replace("_", "-").lower() for c in data.columns]
# Take just date before the date threshold
data = data[data["lodgement-date"] >= "2011-01-01"]
try:
data = pd.read_csv(directory / "certificates.csv", low_memory=False)
# Rename the columns to the same format as the api returns
data.columns = [c.replace("_", "-").lower() for c in data.columns]
# Take just date before the date threshold
data = data[data["lodgement-date"] >= "2011-01-01"]
# Convert to list of dictioaries as returned by the api
data = data.to_dict("records")
# Convert to list of dictioaries as returned by the api
data = data.to_dict("records")
# Incorporate input data into cleaning
cleaner = EpcClean(data)
# Incorporate input data into cleaning
cleaner = EpcClean(data)
cleaner.clean()
# Extended cleaned_data
for k, data in cleaner.cleaned.items():
if k not in cleaned_data:
cleaned_data[k] = data
else:
existing_descriptions = [x["original_description"] for x in cleaned_data[k]]
new_data = [x for x in data if x["original_description"] not in existing_descriptions]
cleaned_data[k].extend(new_data)
cleaner.clean()
# Extended cleaned_data
for k, data in cleaner.cleaned.items():
if k not in cleaned_data:
cleaned_data[k] = data
else:
existing_descriptions = [x["original_description"] for x in cleaned_data[k]]
new_data = [x for x in data if x["original_description"] not in existing_descriptions]
cleaned_data[k].extend(new_data)
except Exception as e:
errors.append(directory)
if errors:
raise ValueError("We have errors")
# Basic check to make sure all descriptions are unique
for _, cleaned in cleaned_data.items():
@ -75,7 +82,6 @@ def app():
# data being read in will be extremely small, meaning quicker load times. We'll begin by storing as a single
# file and monitor usage patterns to see if it makes sense to split the data up
# TODO: Copy the existing cleaned to an archive location, in case we wish to roll back easily
cleaned_historic = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name=f"retrofit-data-{ENVIRONMENT}"

View file

@ -34,6 +34,8 @@ class FloorAttributes(Definitions):
"i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
"i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
"i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation",
"igçör awyr y tu allan, wedigçöi inswleiddio (rhagdybiaeth)": "to external air, insulated (assumed)",
"crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)"
}
def __init__(self, description: str):

View file

@ -130,6 +130,7 @@ class HotWaterAttributes(Definitions):
"o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder "
"thermostat",
"o r brif system, gydag ynni r haul": "from main system, plus solar",
"pwmp gwres": "heat pump"
}
NODATA_DESCRIPTIONS = [

View file

@ -12,6 +12,7 @@ class LightingAttributes(Definitions):
"goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets',
"effeithlonrwydd goleuo da": 'good lighting efficiency',
"effeithlonrwydd goleuo is na'r cyfartaledd": 'below average lighting efficiency',
"effeithlonrwydd goleuo rhagorol": "excellent lighting efficiency"
}
OBSERVED_ERRORS = []

View file

@ -92,7 +92,9 @@ class MainHeatAttributes(Definitions):
"gas-fired heat pumps, electric": "air source heat pump, electric",
"radiator heating, heat from boilers - gas": "boiler and radiators, mains gas",
"heat pump, warm air, mains gas": "air source heat pump, warm air, mains gas",
"air sourceheat pump, radiators, electric": "air source heat pump, radiators, electric"
"air sourceheat pump, radiators, electric": "air source heat pump, radiators, electric",
"bwyler gyda rheiddiaduron a gwres dan y llawr, nwy prif gyflenwad": "Boiler and radiators, mains gas, "
"Boiler and underfloor heating, mains gas",
}
edge_case_result = {}

View file

@ -75,6 +75,7 @@ class MainheatControlAttributes(Definitions):
TO_REMAP = {
"celect control": 'celect-type control',
"celect controls": 'celect-type control',
"celect type controls": 'celect-type control',
"trv's, program & flow switch": 'trvs, programmer & flow switch',
'appliance thermostat': 'appliance thermostats',
}

View file

@ -864,7 +864,7 @@ mainheat_cases = [
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': True,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, "has_electric_heat_pumps": False,
"has_micro-cogeneration": False},
"has_micro-cogeneration": False, 'has_mineral_and_wood': True},
{'original_description': 'Room heaters, electric', 'has_radiators': False, 'has_fan_coil_units': False,
'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
'has_pipes_in_concrete_slab': False, 'has_boiler': False, 'has_air_source_heat_pump': False,
@ -1455,8 +1455,7 @@ mainheat_cases = [
'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
'has_dual_fuel_mineral_and_wood': True, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False,
'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
"has_electric_heat_pumps": False,
"has_micro-cogeneration": False},
"has_electric_heat_pumps": False, "has_micro-cogeneration": False, "has_mineral_and_wood": True},
{'original_description': 'Bwyler a rheiddiaduron, dau danwydd (mwynau a choed)', 'has_radiators': True,
'has_fan_coil_units': False,
'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
@ -1468,8 +1467,8 @@ mainheat_cases = [
'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
'has_dual_fuel_mineral_and_wood': True, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False,
'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
"has_electric_heat_pumps": False,
"has_micro-cogeneration": False},
"has_electric_heat_pumps": False, "has_micro-cogeneration": False, "has_mineral_and_wood": True
},
{'original_description': 'Pwmp gwres syGÇÖn tarddu yn y ddaear, dan y llawr, trydan', 'has_radiators': False,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
@ -1541,7 +1540,7 @@ mainheat_cases = [
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': True,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, "has_electric_heat_pumps": False,
"has_micro-cogeneration": False},
"has_micro-cogeneration": False, "has_mineral_and_wood": True},
{'original_description': 'Room heaters, wood pellets', 'has_radiators': False, 'has_fan_coil_units': False,
'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
'has_pipes_in_concrete_slab': False, 'has_boiler': False, 'has_air_source_heat_pump': False,

View file

@ -51,6 +51,12 @@ class TestHeatingRecommendations:
:return:
"""
# We patch an old version of cleaned which is missing some attributes for 'mainheat-description'
for x in cleaned['mainheat-description']:
x["has_hot-water-only"] = False
x["has_mineral_and_wood"] = False
x["has_dual_fuel_appliance"] = False
epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []}
epc_record = EPCRecord(