estimating eco/gbis eligibility for birmingham

etl/customers/bcc_tender/app.py

@@ -0,0 +1,159 @@
+"""
+This script prepares some data for the Birmingham City Council tender
+"""
+import pandas as pd
+import numpy as np
+
+epc_data = pd.read_csv("local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv")
+
+# Broad assumptions
+# Around 67% of homes in the Uk have an EPC, to be conservative with our estimates, we round up to 70%:
+# https://www.ons.gov.uk/peoplepopulationandcommunity/housing/articles/energyefficiencyofhousinginenglandandwales/2023
+# However, we have 322128 homes in Birmingham with an EPC, which is 76% of the total number of homes in Birmingham
+# based on the 2021 census, which put this figure at 423,500 homes
+PROPORTION_OF_HOMES_WITH_AN_EPC = 0.761
+N_HOUSEHOLDS_IN_BIRMINGHAM = 423_500
+N_HOMES_WITHOUT_AN_EPC = 423_500 - 322128
+
+# 55% of households are recipients of benefits in the West Midlands
+# (2021/2022 - https://www.statista.com/statistics/382858/uk-state-benefits-by-region/)
+PROPORTION_OF_HOMES_ON_BENEFITS = 0.55
+
+# https://www.justgroupplc.co.uk/~/media/Files/J/Just-Retirement-Corp/news-doc/2023/six-in-10-homeowners-eligible-for
+# -benefits-failing-to-claim-just-group-annual-insight-report.pdf
+PROPORTION_OF_HOMEOWNERS_CLAIMING_FOR_BENEFITS = 0.106
+
+# Breakdown of properties in council tax bands in the UK, to give us an estimate of the number of properties in A-D
+band_a_proportion = 0.239
+band_b_proportion = 0.195
+band_c_proportion = 0.219
+band_d_proportion = 0.156
+COUNCIL_TAX_BAND_A_TO_D_PROPORTION = band_a_proportion + band_b_proportion + band_c_proportion + band_d_proportion
+
+# Get the newest record, based on lodgment datetime, by uprn
+epc_data["LODGEMENT_DATETIME"] = pd.to_datetime(epc_data["LODGEMENT_DATETIME"], errors="coerce")
+epc_data = epc_data.sort_values(["LODGEMENT_DATETIME"], ascending=False).drop_duplicates("UPRN")
+
+# We want to figure out the number of properties that are eligible for ECO/GBIS funding
+
+social_tenures = ["Rented (social)", "rental (social)"]
+owner_occupied_tenures = ["Owner-occupied", "owner-occupied"]
+prs_tenures = ["Rented (private)", "rental (private)"]
+
+# If social tenure, then as long as the property is EPC D-G, it's eligible
+epc_data["eligibility_type"] = None
+
+# Eligibiltiy 1: ECO4 help to heat group OO - tenure is owner occupied and EPC rating D-G
+epc_data["eligibility_type"] = np.where(
+    (
+        epc_data["TENURE"].isin(owner_occupied_tenures) &
+        epc_data["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"]) &
+        pd.isnull(epc_data["eligibility_type"])
+    ),
+    "eco4_oo_hthg_needs_scaling_on_benefits",
+    epc_data["eligibility_type"]
+)
+
+# Eligibility 2: ECO4 help to heat group PRS - tenure is private rental and EPC rating E-G
+epc_data["eligibility_type"] = np.where(
+    (
+        epc_data["TENURE"].isin(prs_tenures) &
+        epc_data["CURRENT_ENERGY_RATING"].isin(["E", "F", "G"]) &
+        pd.isnull(epc_data["eligibility_type"])
+    ),
+    "eco4_prs_hthg_needs_scaling_on_benefits",
+    epc_data["eligibility_type"]
+)
+
+# Eligibiltiy 3: ECO4 Social housing - tenure is social rented and EPC rating D-G
+epc_data["eligibility_type"] = np.where(
+    (
+        epc_data["TENURE"].isin(social_tenures) &
+        epc_data["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"]) &
+        pd.isnull(epc_data["eligibility_type"])
+    ),
+    "eco4_social_housing",
+    epc_data["eligibility_type"]
+)
+
+# Eligibility 4: GBIS General Eligibility, OO - tenure is owner occupied and EPC rating D-G
+# This is a subset of Eligiblity 1. We scale eco4_oo_hthg_needs_scaling based on thhe % of properties on benefits
+# For any properties left over that are deemed as not eligibile, a % of these will be eligible for GBIS via Eligibility
+# 4, and therefore any properties that fall out of Eligibility 1, a % will fall into eligibility 4 based a % of units
+# being in council tax bands A-D
+
+# Eligibility 5: GBIS General Eligibility, PRS - tenure is private rental and EPC rating D-G
+# Additionally, some units that fall our of Eligibility 2 will be eligible for GBIS via Eligibility 5, via the same
+# mechanism as Eligibility 4. We handle this later
+epc_data["eligibility_type"] = np.where(
+    (
+        epc_data["TENURE"].isin(prs_tenures) &
+        epc_data["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"]) &
+        pd.isnull(epc_data["eligibility_type"])
+    ),
+    "gbis_prs_ge_needs_scaling_on_council_tax_band",
+    epc_data["eligibility_type"]
+)
+
+# Eligibiilty 6: GBIS General Eligibility, Social - tenure is social rented and EPC rating D-G, but also the property
+# should be rented out below market rate
+# This is a subset of Eligibility 3 - we likely don't need to do any scaling
+
+n_eco4_oo_hthg_needs_scaling_on_benefits = epc_data[
+    epc_data["eligibility_type"] == "eco4_oo_hthg_needs_scaling_on_benefits"
+    ].shape[0]
+
+n_eco4_prs_hthg_needs_scaling_on_benefits = epc_data[
+    epc_data["eligibility_type"] == "eco4_prs_hthg_needs_scaling_on_benefits"
+    ].shape[0]
+
+n_eco4_social = epc_data[
+    epc_data["eligibility_type"] == "eco4_social_housing"
+    ].shape[0]
+
+n_gbis_prs_ge_needs_scaling_on_council_tax_band = epc_data[
+    epc_data["eligibility_type"] == "gbis_prs_ge_needs_scaling_on_council_tax_band"
+    ].shape[0]
+
+n_eligibility_1 = np.floor(n_eco4_oo_hthg_needs_scaling_on_benefits * PROPORTION_OF_HOMEOWNERS_CLAIMING_FOR_BENEFITS)
+
+n_eligibility_2 = np.floor(n_eco4_prs_hthg_needs_scaling_on_benefits * PROPORTION_OF_HOMES_ON_BENEFITS)
+
+n_eligiblity_3 = n_eco4_social
+
+# We subtract the number of homes in eligiblity 1, from the number of homes under ECO4 OO, HTHG, before scaling on
+# benefits. This gives us the number of homes that were not on benefits. We then scale this number based on the % of
+# homes in council tax bands A-D
+n_eligiblity_4 = np.floor(
+    (n_eco4_oo_hthg_needs_scaling_on_benefits - n_eligibility_1) * COUNCIL_TAX_BAND_A_TO_D_PROPORTION
+)
+
+# We also need to add on homes that fall out of eligibility 2
+n_eligibiltiy_5 = np.floor(
+    np.floor(n_gbis_prs_ge_needs_scaling_on_council_tax_band * COUNCIL_TAX_BAND_A_TO_D_PROPORTION) +
+    np.floor((n_eco4_prs_hthg_needs_scaling_on_benefits - n_eligibility_2) * COUNCIL_TAX_BAND_A_TO_D_PROPORTION)
+)
+
+total_eligible = n_eligibility_1 + n_eligibility_2 + n_eligiblity_3 + n_eligiblity_4 + n_eligibiltiy_5
+
+# We don't scale up the # of homes based on % of homes with an EPC, because
+n_owner_occupied = epc_data[epc_data["TENURE"].isin(owner_occupied_tenures)].shape[0]
+oo_eligibility = (n_eligibility_1 + n_eligiblity_4)
+
+# 68% of owner occupied are eligibiltiy
+proportion_of_oo_eligible = oo_eligibility / n_owner_occupied
+# We then use this % on the rest of the homes in Birmingham that do not have an EPC
+oo_eligible_without_an_epc = np.floor(N_HOMES_WITHOUT_AN_EPC * proportion_of_oo_eligible)
+oo_eligibility = oo_eligibility + oo_eligible_without_an_epc
+
+# All private rentals require an EPC
+prs_eligibility = (n_eligibility_2 + n_eligibiltiy_5)
+# Most social housing properties will have an EPC so we don't scale this up
+social_eligibility = n_eligiblity_3
+
+# We scale this up since this number is based on the number of homes in Birmingham with an EPC, and we want to
+# estimate the total number of homes in Birmingham
+total_eligible = oo_eligibility + prs_eligibility + social_eligibility
+
+proportion_of_homes_eligibile = total_eligible / N_HOUSEHOLDS_IN_BIRMINGHAM
+# Approx 58% of homes in Birmingham are eligible for ECO/GBIS funding

recommendations/HeatingRecommender.py

@@ -28,7 +28,7 @@ class HeatingRecommender:
             self.property.main_heating["clean_description"] in self.ELECTRIC_HEATING_DESCRIPTIONS
         )
 
-    def is_high_heat_retention_valid(self):
+    def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, exclusions):
         """
         Check conditions if high heat retention storage is valid
         :return:
@@ -40,11 +40,59 @@ class HeatingRecommender:
             self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"]
         )
 
-        return self.has_electric_heating_description or electric_heating_assumed
+        has_electric = self.has_electric_heating_description or electric_heating_assumed
+
+        return (
+            has_electric and (not ashp_only_heating_recommendation) and ("boiler_upgrade" not in exclusions)
+        )
+
+    def is_boiler_upgrade_suitable(self, exclusions, ashp_only_heating_recommendation):
+        """
+        These are the conditions we apply to recommend a boiler installation
+        :return:
+        """
+
+        # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls
+        has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"]
+
+        # 2) If the property doesn't have a heating system, but it has access to the mains gas
+        no_heating_has_mains = self.property.main_heating["clean_description"] in [
+            'No system present, electric heaters assumed'
+        ] and self.property.data["mains-gas-flag"]
+
+        # The property is using portable heaters and has access to gas mains
+        has_room_heaters = (
+            self.property.main_heating["clean_description"] in ["Room heaters, mains gas", "Room heaters, electric"] and
+            self.property.data["mains-gas-flag"]
+        )
+
+        # We also check if the property has electric heating, but it has access to the mains gas
+        electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"]
+
+        portable_heaters_has_mains = (
+            self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"]
+            and
+            self.property.data["mains-gas-flag"]
+        )
+
+        is_valid = (
+            (
+                has_boiler or
+                no_heating_has_mains or
+                electic_heating_has_mains or
+                has_room_heaters or
+                portable_heaters_has_mains
+            ) and
+            (not ashp_only_heating_recommendation) and
+            ("boiler_upgrade" not in exclusions)
+        )
+
+        return is_valid, has_boiler
 
     def recommend(self, has_cavity_or_loft_recommendations, phase=0, exclusions=None):
         """
         Produces heating recommendations
+
         :param has_cavity_or_loft_recommendations: boolean indicating if we have produced a cavity or loft insulation
         recommendation. If there are cavity or loft recommendations, the property would need to complete those measures
         before being able to get the boiler upgrade scheme benefits. The messaging in the front end would be to
@@ -56,6 +104,8 @@ class HeatingRecommender:
         #       the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this
         #       in the Costs class, stored as SYSTEM_FLUSH_COST
 
+        # TODO: Right now, we don't have recommendations for electric boilers - we should probably have one
+
         exclusions = [] if exclusions is None else exclusions
         non_invasive_ashp_recommendation = next(
             (r for r in self.property.non_invasive_recommendations if r["type"] == "air_source_heat_pump"),
@@ -72,47 +122,19 @@ class HeatingRecommender:
         # This first iteration of the recommender will provide very basic recommendation
         # We recommend heating controls based on the main heating system
 
-        if (self.is_high_heat_retention_valid() and
+        hhr_valid = self.is_high_heat_retention_valid(ashp_only_heating_recommendation, exclusions)
-            (not ashp_only_heating_recommendation) and
+
-            ("boiler_upgrade" not in exclusions)
+        if hhr_valid:
-        ):
             # Recommend high heat retention storage heaters
             # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just
             #       needs the controls
             self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False)
 
-        # if the property has mains heating with boiler and radiators, we recommend optimal heating controls
+        gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable(
-        has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"]
+            exclusions=exclusions, ashp_only_heating_recommendation=ashp_only_heating_recommendation
-
-        # We also check that the property doesn't have a heating system, but it has access to the mains gas
-        no_heating_has_mains = self.property.main_heating["clean_description"] in [
-            'No system present, electric heaters assumed'
-        ] and self.property.data["mains-gas-flag"]
-
-        has_gas_heaters = (
-            self.property.main_heating["clean_description"] in ["Room heaters, mains gas"] and
-            self.property.data["mains-gas-flag"]
         )
 
-        # We also check if the property has electric heating, but it has access to the mains gas
+        if gas_boiler_suitable:
-        electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"]
-
-        portable_heaters_has_mains = (
-            self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"]
-            and
-            self.property.data["mains-gas-flag"]
-        )
-
-        if ((
-            has_boiler or
-            no_heating_has_mains or
-            electic_heating_has_mains or
-            has_gas_heaters or
-            portable_heaters_has_mains
-        ) and
-            (not ashp_only_heating_recommendation) and
-            ("boiler_upgrade" not in exclusions)
-        ):
             # This indicates that the home previously did not have a boiler in place and so would require
             # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler
             system_change = not has_boiler

recommendations/tests/test_data/heating_recommendations_data.py

@@ -279,7 +279,7 @@ testing_examples = [
             'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 86% of fixed outlets',
             'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good',
             'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0.0, 'heat-loss-corridor': 'no corridor',
-            'unheated-corridor-length': None, 'floor-height': None, 'photo-supply': 0.0,
+            'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': 0.0,
             'solar-water-heating-flag': None,
             'mechanical-ventilation': 'natural', 'address': '13, Starbuck Street, Rudry',
             'local-authority-label': 'Caerphilly', 'constituency-label': 'Caerphilly', 'posttown': 'CAERPHILLY',
@@ -287,9 +287,67 @@ testing_examples = [
             'tenure': 'rental (private)', 'fixed-lighting-outlets-count': 7.0, 'low-energy-fixed-light-count': 6.0,
             'uprn': 43088770.0, 'uprn-source': 'Address Matched',
         },
-        "heating_recommendation_descriptions": [],
+        "heating_recommendation_descriptions": [
+            'Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention '
+            'Storage Heater Controls'
+        ],
         "heating_controls_recommendation_descriptions": [],
-        "notes": ""
+        "notes": "This property is a flat so we don't have an ASHP recommendation. It also doesn't have access to the "
+                 "mains and so it can't have a gas boiler. We don't expect any controls recommendations"
+    },
+    {
+        "epc": {
+            'lmk-key': '492646189022010060208143796198410', 'address1': '67, Ridgeway Road', 'address2': None,
+            'address3': None, 'postcode': 'HP5 2EW', 'building-reference-number': 1976846768,
+            'current-energy-rating': 'D', 'potential-energy-rating': 'D', 'current-energy-efficiency': 64,
+            'potential-energy-efficiency': 68, 'property-type': 'Bungalow', 'built-form': 'Detached',
+            'inspection-date': '2010-06-01', 'local-authority': 'E07000005', 'constituency': 'E14000631',
+            'county': 'Buckinghamshire', 'lodgement-date': '2010-06-02', 'transaction-type': 'marketed sale',
+            'environment-impact-current': 67, 'environment-impact-potential': 70, 'energy-consumption-current': 249,
+            'energy-consumption-potential': 231.0, 'co2-emissions-current': 3.5, 'co2-emiss-curr-per-floor-area': 35,
+            'co2-emissions-potential': 3.2, 'lighting-cost-current': 89.0, 'lighting-cost-potential': 51.0,
+            'heating-cost-current': 627.0, 'heating-cost-potential': 603.0, 'hot-water-cost-current': 105.0,
+            'hot-water-cost-potential': 105.0, 'total-floor-area': 76.0, 'energy-tariff': 'Single',
+            'mains-gas-flag': 'Y', 'floor-level': 'NO DATA!', 'flat-top-storey': None, 'flat-storey-count': None,
+            'main-heating-controls': 2104.0, 'multi-glaze-proportion': 100.0,
+            'glazed-type': 'double glazing installed during or after 2002', 'glazed-area': 'Normal',
+            'extension-count': 0.0, 'number-habitable-rooms': 7.0, 'number-heated-rooms': 7.0,
+            'low-energy-lighting': 25.0, 'number-open-fireplaces': 1.0, 'hotwater-description': 'From main system',
+            'hot-water-energy-eff': 'Very Good', 'hot-water-env-eff': 'Very Good',
+            'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': None, 'floor-env-eff': None,
+            'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', 'windows-env-eff': 'Good',
+            'walls-description': 'Cavity wall, filled cavity', 'walls-energy-eff': 'Good', 'walls-env-eff': 'Good',
+            'secondheat-description': 'Room heaters, wood logs', 'sheating-energy-eff': None, 'sheating-env-eff': None,
+            'roof-description': 'Pitched, 150 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good',
+            'mainheat-description': 'Boiler and radiators, mains gas', 'mainheat-energy-eff': 'Very Good',
+            'mainheat-env-eff': 'Very Good', 'mainheatcont-description': 'Programmer and room thermostat',
+            'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
+            'lighting-description': 'Low energy lighting in 25% of fixed outlets', 'lighting-energy-eff': 'Average',
+            'lighting-env-eff': 'Average',
+            'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used',
+            'wind-turbine-count': 0.0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None,
+            'floor-height': 2.4, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N',
+            'mechanical-ventilation': 'natural', 'address': '67, Ridgeway Road', 'local-authority-label': 'Chiltern',
+            'constituency-label': 'Chesham and Amersham', 'posttown': 'CHESHAM',
+            'construction-age-band': 'England and Wales: 1930-1949', 'lodgement-datetime': '2010-06-02 08:14:37',
+            'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None,
+            'uprn': 100080513604.0, 'uprn-source': 'Address Matched'
+        },
+        "heating_recommendation_descriptions": [
+            'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and '
+            'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade '
+            'scheme grant'
+        ],
+        "heating_controls_recommendation_descriptions": [
+            'upgrade heating controls to Room thermostat, programmer and TRVs',
+            'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & '
+            'temperature zone control)'
+
+        ],
+        "notes": "This has a very efficient boiler and is a detached bungalow, but only has "
+                 "Programmer and room thermostat for heating controls so we'd expect an ASHP heating recommendation"
+                 "as the only option, and heating controls recommendations for programmer, room thermostats and trvs"
+                 "as well as ttzc"
     }
 ]
 
@@ -306,6 +364,7 @@ directory = random.sample(epc_directories, 1)[0]
 data = pd.read_csv(directory / "certificates.csv", low_memory=False)
 # Rename the columns to the same format as the api returns
 data.columns = [c.replace("_", "-").lower() for c in data.columns]
+data["floor-height"] = data["floor-height"].fillna(2.45)
 
 used_examples = pd.DataFrame(
     [
@@ -327,3 +386,6 @@ data = data[pd.isnull(data["used"])].drop(columns=["used"])
 eg = data.sample(1).to_dict("records")[0]
 print(eg["mainheat-description"])
 print(eg["mainheat-energy-eff"])
+print(eg["property-type"])
+print(eg["built-form"])
+print(eg["mainheatcont-description"])