fixing funding edge cases and adding tests wip

backend/Funding.py

@@ -1,6 +1,5 @@
 from enum import Enum
 from typing import List
-
 import pandas as pd
 
 from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
@@ -57,6 +56,7 @@ class Funding:
         # Funding calculation variables
         self.full_project_abs = None
         self.eco4_funding = None
+        self.eco4_uplift = 0
 
         self.partial_project_abs = None
 
@@ -875,8 +875,8 @@ class Funding:
                         pre_heating_system=pre_heating_system
                     )
                     project_uplifts.append(pps * uplifts[i])
-                total_uplift = sum(project_uplifts)
+                self.eco4_uplift = sum(project_uplifts)
-                self.full_project_abs += total_uplift
+                self.full_project_abs += self.eco4_uplift
                 self.eco4_funding = self.full_project_abs * (
                     self.social_cavity_abs_rate if is_cavity else self.social_solid_abs_rate
                 )

backend/tests/test_data/pre_heating_scenarios.py

@@ -0,0 +1,144 @@
+# Each scenario: super explicit about inputs and expected mapping
+pre_main_heating_scenarios = [
+    # --- Mains gas boilers (radiators) ---
+    {
+        "description": "Boiler and radiators, mains gas (condensing expected)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+    {
+        "description": "Boiler and radiators, mains gas (non-condensing expected)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "mains gas - this is for backwards compatibility only and should not be used",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Non Condensing Gas Boiler",
+    },
+    {
+        "description": "Boiler and radiators, mains gas (very poor => back boiler to rads)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "Gas: mains gas",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Gas Back Boiler to Radiators",
+    },
+
+    # --- Warm air (treated like gas boiler family in your mapper) ---
+    {
+        "description": "Warm air, mains gas (good => condensing)",
+        "MAINHEAT_DESCRIPTION": "Warm air, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+
+    # --- Community scheme (CHP vs non-CHP depends on energy eff) ---
+    {
+        "description": "Community scheme (gas, good => CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "mains gas (community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "DHS CHP",
+    },
+    {
+        "description": "Community scheme (gas, average => non-CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "mains gas (community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "DHS non-CHP",
+    },
+    {
+        "description": "Community scheme (no fuel data, good => CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "NO DATA!",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "DHS CHP",
+    },
+
+    # --- Electric storage heaters (ESH responsiveness split) ---
+    {
+        "description": "Electric storage heaters (average => responsiveness > 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Electric Storage Heaters Responsiveness >0.2",
+    },
+    {
+        "description": "Electric storage heaters (poor => responsiveness > 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity - this is for backwards compatibility only and should not be used",
+        "MAINHEAT_ENERGY_EFF": "Poor",
+        "expected": "Electric Storage Heaters Responsiveness >0.2",
+    },
+    {
+        "description": "Electric storage heaters (very poor => responsiveness <= 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Storage Heaters Responsiveness <=0.2",
+    },
+
+    # --- Electric direct-acting / room heaters ---
+    {
+        "description": "Room heaters, electric (very poor)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, electric",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Room heaters, electric (poor, unspecified tariff)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, electric",
+        "MAIN_FUEL": "Electricity: electricity, unspecified tariff",
+        "MAINHEAT_ENERGY_EFF": "Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Portable electric heaters assumed for most rooms (maps to electric room heaters)",
+        "MAINHEAT_DESCRIPTION": "Portable electric heaters assumed for most rooms",
+        "MAIN_FUEL": "mains gas (not community)",  # weird in EPCs, but your mapper forces electric room heaters here
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "No system present: electric heaters assumed",
+        "MAINHEAT_DESCRIPTION": "No system present: electric heaters assumed",
+        "MAIN_FUEL": "To be used only when there is no heating/hot-water system",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Electric underfloor heating => direct-acting electric",
+        "MAINHEAT_DESCRIPTION": "Electric underfloor heating",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Electric Room Heaters",
+    },
+
+    # --- Gas room heaters ---
+    {
+        "description": "Room heaters, mains gas (average)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Gas Room Heaters",
+    },
+
+    # --- Electric boiler ---
+    {
+        "description": "Boiler and radiators, electric (very poor => electric boiler)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, electric",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Boiler",
+    },
+
+    # --- Gas boiler + UFH (still ‘boiler’ logic) ---
+    {
+        "description": "Boiler and underfloor heating, mains gas (good => condensing)",
+        "MAINHEAT_DESCRIPTION": "Boiler and underfloor heating, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+]

backend/tests/test_funding.py

@@ -2,6 +2,7 @@ import pytest
 import pandas as pd
 from backend.Funding import Funding, EligibilityCaveats
 from backend.tests.test_data.innovation_measure_fixtures import innovation_scenarios
+from backend.tests.test_data.pre_heating_scenarios import pre_main_heating_scenarios
 
 
 @pytest.fixture
@@ -43,6 +44,49 @@ def mock_whlg_postcodes():
     return pd.DataFrame([{"Postcode": "ab12cd"}])
 
 
+@pytest.fixture
+def mock_mainheating():
+    return {
+        'original_description': 'Electric storage heaters', 'has_radiators': False,
+        'has_fan_coil_units': False,
+        'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False,
+        'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+        'has_air_source_heat_pump': False,
+        'has_room_heaters': False, 'has_electric_storage_heaters': True, 'has_warm_air': False,
+        'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False,
+        'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+        'has_portable_electric_heaters': False,
+        'has_water_source_heat_pump': False, 'has_electric': True, 'has_mains_gas': False,
+        'has_wood_logs': False,
+        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite':
+            False,
+        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False,
+        'has_assumed': False,
+        'has_electricaire': False, 'has_assumed_for_most_rooms': False,
+        'has_underfloor_heating': False,
+        "has_electric_heat_pumps": False,
+        "has_micro-cogeneration": False
+    }
+
+
+@pytest.fixture
+def mock_main_fuel():
+    return {
+        'original_description': 'Electricity: electricity, unspecified tariff', 'fuel_type':
+            'electricity',
+        'tariff_type': 'unspecified tariff', 'is_community': False,
+        'no_individual_heating_or_community_network': False,
+        'complex_fuel_type': None
+    }
+
+
+@pytest.fixture
+def mock_mainheat_energy_eff():
+    return "Average"
+
+
 ### -------------------------
 ### PRIVATE RENTED SECTOR (PRS)
 ### -------------------------
@@ -916,7 +960,10 @@ def test_custom_eco4_scenarios(
 def test_uplift(
     mock_project_scores_matrix,
     mock_partial_scores_matrix,
-    mock_whlg_postcodes
+    mock_whlg_postcodes,
+    mock_mainheating,
+    mock_main_fuel,
+    mock_mainheat_energy_eff
 ):
     funding = Funding(
         project_scores_matrix=mock_project_scores_matrix,
@@ -939,38 +986,6 @@ def test_uplift(
         {"type": "cavity_wall_insulation", "is_innovation": False, "uplift": 0.25},
     ]
 
-    mainheating = {
-        'original_description': 'Electric storage heaters', 'has_radiators': False,
-        'has_fan_coil_units': False,
-        'has_pipes_in_screed_above_insulation': False,
-        'has_pipes_in_insulated_timber_floor': False,
-        'has_pipes_in_concrete_slab': False, 'has_boiler': False,
-        'has_air_source_heat_pump': False,
-        'has_room_heaters': False, 'has_electric_storage_heaters': True, 'has_warm_air': False,
-        'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
-        'has_community_scheme': False,
-        'has_ground_source_heat_pump': False, 'has_no_system_present': False,
-        'has_portable_electric_heaters': False,
-        'has_water_source_heat_pump': False, 'has_electric': True, 'has_mains_gas': False,
-        'has_wood_logs': False,
-        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite':
-            False,
-        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False,
-        'has_assumed': False,
-        'has_electricaire': False, 'has_assumed_for_most_rooms': False,
-        'has_underfloor_heating': False,
-        "has_electric_heat_pumps": False,
-        "has_micro-cogeneration": False
-    }
-    main_fuel = {
-        'original_description': 'Electricity: electricity, unspecified tariff', 'fuel_type':
-            'electricity',
-        'tariff_type': 'unspecified tariff', 'is_community': False,
-        'no_individual_heating_or_community_network': False,
-        'complex_fuel_type': None
-    }
-    mainheat_energy_eff = "Good"
-
     funding.check_funding(
         measures=measures,
         starting_sap=33,
@@ -984,11 +999,46 @@ def test_uplift(
         existing_li_thickness=0,
         has_wall_insulation_recommendation=True,
         has_roof_insulation_recommendation=True,
-        mainheating=mainheating,
+        mainheating=mock_mainheating,
-        main_fuel=main_fuel,
+        main_fuel=mock_main_fuel,
-        mainheat_energy_eff=mainheat_energy_eff,
+        mainheat_energy_eff=mock_mainheat_energy_eff,
     )
 
+    assert funding.eco4_funding == 123
+    assert funding.eco4_uplift == 456
+
+
+def _dummy_funding():
+    # Matrices/whlg are unused by _map_to_pre_main_heating; pass harmless placeholders
+    return Funding(
+        tenure="Social",
+        social_cavity_abs_rate=0.0,
+        social_solid_abs_rate=0.0,
+        private_cavity_abs_rate=0.0,
+        private_solid_abs_rate=0.0,
+        project_scores_matrix=None,
+        partial_project_scores_matrix=None,
+        whlg_eligible_postcodes=set(),
+    )
+
+
+@pytest.mark.parametrize("scenario", pre_main_heating_scenarios)
+def test_map_to_pre_main_heating(scenario):
+    funding = _dummy_funding()
+
+    # Build normalized mainheating / main_fuel using your attribute processors
+    h = MainHeatAttributes(description=scenario["MAINHEAT_DESCRIPTION"]).process()
+    f = MainFuelAttributes(description=scenario["MAIN_FUEL"]).process()
+
+    result = funding._map_to_pre_main_heating(
+        mainheating=h,
+        main_fuel=f,
+        mainheat_energy_eff=scenario["MAINHEAT_ENERGY_EFF"],
+    )
+
+    assert result == scenario[
+        "expected"], f"Failed: {scenario['description']} -> {result} (expected {scenario['expected']})"
+
 
 # Large scale testing for various measures
 measures = [
@@ -1003,7 +1053,7 @@ measures = [
     {"type": "high_heat_retention_storage_heaters", "is_innovation": False, "uplift": 0},
 ]
 epc_df = pd.read_csv(
-    "/Users/khalimconn-kowlessar/Downloads/domestic-E08000025-Birmingham/certificates.csv"
+    "/Users/khalimconn-kowlessar/Downloads/domestic-E08000003-Manchester/certificates.csv"
 )
 from tqdm import tqdm
 from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
@@ -1082,3 +1132,7 @@ x = errored_epcs[
     (errored_epcs["MAINHEAT_DESCRIPTION"] == unique_combs["MAINHEAT_DESCRIPTION"].values[i]) &
     (errored_epcs["MAIN_FUEL"] == unique_combs["MAIN_FUEL"].values[i])
     ].head(1).squeeze()
+
+most_prominent_combinations = epc_df.groupby(
+    ["MAINHEAT_ENERGY_EFF", "MAINHEAT_DESCRIPTION", "MAIN_FUEL"]
+)["LMK_KEY"].nunique().reset_index().sort_values("LMK_KEY", ascending=False).head(30).to_dict("records")

etl/epc_clean/tests/test_attribute_utils.py

@@ -53,12 +53,11 @@ def test_process_part_value_errors():
     with pytest.raises(ValueError):
         attribute_utils.process_part(result, part, attr_list, prefix)
 
-
+# Test for no attribute matches found - we don't raise this error any more
-# Test for no attribute matches found
+# def test_process_part_no_matches():
-def test_process_part_no_matches():
+#     result = {'has_glazing': False, 'has_glazed': False, 'has_glaze': False}
-    result = {'has_glazing': False, 'has_glazed': False, 'has_glaze': False}
+#     part = 'high performance coating'
-    part = 'high performance coating'
+#     attr_list = ['glazing', 'glazed', 'glaze']
-    attr_list = ['glazing', 'glazed', 'glaze']
+#     prefix = 'has_'
-    prefix = 'has_'
+#     with pytest.raises(ValueError):
-    with pytest.raises(ValueError):
+#         attribute_utils.process_part(result, part, attr_list, prefix)
-        attribute_utils.process_part(result, part, attr_list, prefix)