adding heating unit tests

2026-06-08 11:17:27 +00:00 · 2024-08-31 15:18:05 +01:00 · 2024-08-31 15:18:05 +01:00 · 8004d2f126
commit 8004d2f126
parent e2e9721605
7 changed files with 336 additions and 69 deletions
--- a/backend/apis/GoogleSolarApi.py
+++ b/backend/apis/GoogleSolarApi.py
@ -148,7 +148,7 @@ class GoogleSolarApi:
        # Extract key data from the insights response
        self.roof_segments = self.insights_data["solarPotential"].get('roofSegmentStats', [])
        # Automatically exclude north-facing segments
-        self.exclude_north_facing_segments()
+        self.exclude_north_facing_segments(property_instance=property_instance)
        # If a property is semi-detached, it's possible for us to include segments from an attached unit
        if (property_instance.data["built-form"] == "Semi-Detached") and (
            property_instance.data["extension-count"] == 0
@ -291,6 +291,8 @@ class GoogleSolarApi:
                )

            roi_summary = pd.DataFrame(roi_summary)
+            if roi_summary.empty:
+                continue

            weighted_ratio = np.average(
                roi_summary["ratio"].values, weights=roi_summary["generated_dc_energy"].values
@ -309,7 +311,7 @@ class GoogleSolarApi:
                }
            )

-        panel_performance = pd.DataFrame([panel_performance])
+        panel_performance = pd.DataFrame(panel_performance)

        if panel_performance.empty:
            self.panel_performance = pd.DataFrame(
@ -487,7 +489,7 @@ class GoogleSolarApi:

        self.panel_performance = panel_performance

-    def exclude_north_facing_segments(self):
+    def exclude_north_facing_segments(self, property_instance):
        """
        Filter out any north-facing roof segments from the roof_segments attribute.

@ -498,7 +500,9 @@ class GoogleSolarApi:
        for segment_index, segment in enumerate(self.roof_segments):
            segment["segmentIndex"] = segment_index
            # Check if the segment is north-facing
-            if self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]:
+            if (
+                self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]
+            ) and not property_instance.roof["is_flat"]:
                continue

            filtered_segments.append(segment)
--- a/backend/app/plan/schemas.py
+++ b/backend/app/plan/schemas.py
@ -35,7 +35,9 @@ class PlanTriggerRequest(BaseModel):
        "air_source_heat_pump",
        "internal_wall_insulation",
        "external_wall_insulation",
-        "secondary_heating"
+        "secondary_heating",
+        "boiler_upgrade",
+        "high_heat_retention_storage_heater",
    }

    _allowed_goals = {"Increasing EPC"}
--- a/etl/customers/orbit/archetypes.py
+++ b/etl/customers/orbit/archetypes.py
@ -77,47 +77,48 @@ def lesney_farms():
        29291,  # No EPC for 225 Slade Green Road, Erith, Kent, DA8 2JW
    ]
    # Get the EPC data
-    epc_data = []
-    for _, home in tqdm(all_assets.iterrows(), total=len(all_assets)):
-        if home["Asset Reference"] in known_no_epc:
-            continue
-
-        address = home["Address"]
-        # Spelling error
-        if "Frinstead" in address:
-            address = address.replace("Frinstead", "Frinsted")
-
-        address1 = address.split(",")[0]
-
-        asset_type_map = {
-            "HOUSE": "House",
-            "BUNGALOWS": "Bungalow",
-            "FLATS": "Flat",
-            "MAISONETTES": "Maisonette",
-        }
-
-        searcher = SearchEpc(
-            address1=address1,
-            postcode=home["Address - Postcode"],
-            auth_token=EPC_AUTH_TOKEN,
-            os_api_key="",
-            full_address=address,
-        )
-        searcher.ordnance_survey_client.property_type = asset_type_map[home["Asset Type"]]
-        searcher.ordnance_survey_client.built_form = None
-
-        searcher.find_property(skip_os=True)
-        if searcher.newest_epc is None:
-            raise Exception("Couldn't find")
-
-        epc_data.append(
-            {
-                "Asset Reference": home["Asset Reference"],
-                **searcher.newest_epc.copy()
-            }
-        )
-
-    epc_data = pd.DataFrame(epc_data)
+    # epc_data = []
+    # for _, home in tqdm(all_assets.iterrows(), total=len(all_assets)):
+    #     if home["Asset Reference"] in known_no_epc:
+    #         continue
+    #
+    #     address = home["Address"]
+    #     # Spelling error
+    #     if "Frinstead" in address:
+    #         address = address.replace("Frinstead", "Frinsted")
+    #
+    #     address1 = address.split(",")[0]
+    #
+    #     asset_type_map = {
+    #         "HOUSE": "House",
+    #         "BUNGALOWS": "Bungalow",
+    #         "FLATS": "Flat",
+    #         "MAISONETTES": "Maisonette",
+    #     }
+    #
+    #     searcher = SearchEpc(
+    #         address1=address1,
+    #         postcode=home["Address - Postcode"],
+    #         auth_token=EPC_AUTH_TOKEN,
+    #         os_api_key="",
+    #         full_address=address,
+    #     )
+    #     searcher.ordnance_survey_client.property_type = asset_type_map[home["Asset Type"]]
+    #     searcher.ordnance_survey_client.built_form = None
+    #
+    #     searcher.find_property(skip_os=True)
+    #     if searcher.newest_epc is None:
+    #         raise Exception("Couldn't find")
+    #
+    #     epc_data.append(
+    #         {
+    #             "Asset Reference": home["Asset Reference"],
+    #             **searcher.newest_epc.copy()
+    #         }
+    #     )
+    #
+    # epc_data = pd.DataFrame(epc_data)
+    epc_data = pd.read_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Orbit - Wates/Bexley EPC data.csv", )
    # epc_data.to_csv(
    #     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Orbit - Wates/Bexley EPC data.csv", index=False
    # )
@ -316,7 +317,7 @@ def lesney_farms():
    lesney_4[["Address", "Address - Postcode", "lodgement-date", "roof-description"]]

    assigned_archetypes = archetyped_data[
-        ["Asset Reference", "archetype ID", "Address"] + chosen_combination +
+        ["Asset Reference", "archetype ID", "Address", "Address - Postcode"] + chosen_combination +
        ["lodgement-date", "current-energy-rating", "current-energy-efficiency", "walls-description"]
        ].copy()
    # Map the archetype ID to their string representation
--- a/etl/customers/orbit/funding_example_portfolio.py
+++ b/etl/customers/orbit/funding_example_portfolio.py
@ -0,0 +1,141 @@
+import pandas as pd
+
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 100
+
+
+def app():
+    """
+    This function sets up an asset list with just a few properties to model the impact of the following scenarios:
+    1) EWI
+    2) EWI + Solar
+    3) EWI + Solar + ASHP
+    :return:
+    """
+
+    asset_list = [
+        # This is an example of a low D - SAP score is 60
+        {
+            "address": "37, Birling Road",
+            "postcode": "DA8 3JQ",
+            "uprn": 100020225444
+        },
+        {
+            "address": "16, Brasted Road",
+            "postcode": "DA8 3HU",
+            "uprn": 100020225805
+        },
+        {
+            "address": "25, Birling Road",
+            "postcode": "DA8 3JQ",
+            "uprn": 100020225432,
+        },
+        {
+            "address": "4, Halstead Road",
+            "postcode": "DA8 3HX",
+            "uprn": 100020229555
+        }
+    ]
+    asset_list = pd.DataFrame(asset_list)
+
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    non_invasive_recs = []
+    for _, al in asset_list.iterrows():
+        solar_rec = {
+            "type": "solar_pv",
+            "suitable": True,
+            "array_wattage": 4000,
+            "initial_ac_kwh_per_year": 3800,
+            "cost": 4009,
+            "panneled_roof_area": 20  # Rough estimate for 10 panels, around 1m x 1.8m (accomodate gaps and 30cm edge)
+        }
+
+        non_invasive_recs.append({
+            "uprn": al["uprn"],
+            "recommendations": [solar_rec],
+        })
+
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.csv"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recs),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    body1 = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increasing EPC",
+        "goal_value": "A",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": "",
+        "scenario_name": "ECO4 funding - EWI",
+        "multi_plan": True,
+        "exclusions": [
+            "internal_wall_insulation",
+            "roof_insulation", "ventilation", "floor_insulation", "windows", "fireplace", "heating", "hot_water",
+            "lighting", "secondary_heating", "solar_pv"
+        ],
+        "budget": None,
+    }
+    print(body1)
+
+    body2 = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increasing EPC",
+        "goal_value": "A",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "scenario_name": "ECO4 funding - EWI + Solar",
+        "multi_plan": True,
+        "exclusions": [
+            "internal_wall_insulation",
+            "roof_insulation",
+            "ventilation",
+            "floor_insulation",
+            "windows",
+            "fireplace",
+            "heating",
+            "hot_water",
+            "lighting",
+            "secondary_heating",
+            "boiler_upgrade",
+            "high_heat_retention_storage_heater",
+        ],
+        "budget": None,
+    }
+    print(body2)
+
+    body3 = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increasing EPC",
+        "goal_value": "A",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "scenario_name": "ECO4 funding - EWI + Solar + ASHP",
+        "multi_plan": True,
+        "exclusions": [
+            "internal_wall_insulation",
+            "roof_insulation", "ventilation", "floor_insulation", "windows", "fireplace", "hot_water",
+            "lighting", "secondary_heating",
+        ],
+        "budget": None,
+    }
+    print(body3)
--- a/recommendations/HeatingRecommender.py
+++ b/recommendations/HeatingRecommender.py
@ -72,7 +72,10 @@ class HeatingRecommender:
        # This first iteration of the recommender will provide very basic recommendation
        # We recommend heating controls based on the main heating system

-        if self.is_high_heat_retention_valid() and not ashp_only_heating_recommendation:
+        if (self.is_high_heat_retention_valid() and
+            (not ashp_only_heating_recommendation) and
+            ("boiler_upgrade" not in exclusions)
+        ):
            # Recommend high heat retention storage heaters
            # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just
            #       needs the controls
@ -106,7 +109,10 @@ class HeatingRecommender:
            electic_heating_has_mains or
            has_gas_heaters or
            portable_heaters_has_mains
-        ) and not ashp_only_heating_recommendation):
+        ) and
+            (not ashp_only_heating_recommendation) and
+            ("boiler_upgrade" not in exclusions)
+        ):
            # This indicates that the home previously did not have a boiler in place and so would require
            # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler
            system_change = not has_boiler
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@ -99,7 +99,11 @@ class SolarPvRecommendations:
        best_configurations = panel_performance.head(1).reset_index(drop=True)

        for rank, recommendation_config in best_configurations.iterrows():
-            roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100)
+            # If we dont have the panneled_roof_area in the recommendation_config we calculate it
+            if recommendation_config.get("panneled_roof_area", None):
+                roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100)
+            else:
+                raise Exception("IMPLEMENT ME")
            # Spread the cost to the individual units - adding a 20% contingency
            total_cost = recommendation_config["total_cost"] / n_units
            kw = np.floor(recommendation_config["array_wattage"] / 100) / 10
@ -162,9 +166,12 @@ class SolarPvRecommendations:

        if non_invasive_recommendation.get("array_wattage") is not None:

-            roof_area = esimtate_pitched_roof_area(
-                floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"]
-            )
+            if self.property.roof["is_flat"]:
+                roof_area = self.property.insulation_floor_area
+            else:
+                roof_area = esimtate_pitched_roof_area(
+                    floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"]
+                )
            solar_configurations = pd.DataFrame(
                [
                    {
@ -175,6 +182,7 @@ class SolarPvRecommendations:
                ]
            )
        else:
+            # TODO: There may be some instances where we don't want to use the solar API so we should cover for them
            panel_performance = self.property.solar_panel_configuration["panel_performance"]
            roof_area = self.property.roof_area
            solar_configurations = panel_performance.head(3).reset_index(drop=True)
@ -182,6 +190,8 @@ class SolarPvRecommendations:
        # We combine each of these configurations with estimates with and without a battery
        for rank, recommendation_config in solar_configurations.iterrows():
            roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / roof_area * 100)
+            # We round up to the nearest 10
+            roof_coverage_percent = np.ceil(roof_coverage_percent / 10) * 10
            for has_battery in [False, True]:
                cost_result = self.costs.solar_pv(
                    wattage=recommendation_config["array_wattage"],
--- a/recommendations/tests/test_data/heating_recommendations_data.py
+++ b/recommendations/tests/test_data/heating_recommendations_data.py
@ -1,19 +1,3 @@
-# import random
-# from pathlib import Path
-# import inspect
-# import pandas as pd
-#
-# # this can be used to get example data to build the test cases
-# src_file_path = inspect.getfile(lambda: None)
-# EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates"
-# epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
-# directory = random.sample(epc_directories, 1)[0]
-# data = pd.read_csv(directory / "certificates.csv", low_memory=False)
-# # Rename the columns to the same format as the api returns
-# data.columns = [c.replace("_", "-").lower() for c in data.columns]
-#
-# eg = data.sample(1).to_dict("records")[0]
-
 testing_examples = [
    {
        "epc": {
@ -67,5 +51,124 @@ testing_examples = [
        "notes": "This property has a boiler, radiators & mains gas with good efficiency so the only recommendation"
                 "we expect here is for an air source heat pump. The heating controls are a programmer, room thermostat"
                 "and TRVs and so we should expect a TTZC recommendation"
+    },
+    {
+        "epc": {
+            'lmk-key': '153995620832008100717310934068296', 'address1': 'Apartment 13 The Quays',
+            'address2': 'Burscough', 'address3': None, 'postcode': 'L40 5TW',
+            'building-reference-number': 2604281568, 'current-energy-rating': 'C', 'potential-energy-rating': 'B',
+            'current-energy-efficiency': 69, 'potential-energy-efficiency': 84, 'property-type': 'Flat',
+            'built-form': 'Detached', 'inspection-date': '2008-10-06', 'local-authority': 'E07000127',
+            'constituency': 'E14001033', 'county': 'Lancashire', 'lodgement-date': '2008-10-07',
+            'transaction-type': 'marketed sale', 'environment-impact-current': 78,
+            'environment-impact-potential': 78, 'energy-consumption-current': 195,
+            'energy-consumption-potential': 192.0, 'co2-emissions-current': 1.7,
+            'co2-emiss-curr-per-floor-area': 29, 'co2-emissions-potential': 1.7, 'lighting-cost-current': 35,
+            'lighting-cost-potential': 38, 'heating-cost-current': 108, 'heating-cost-potential': 89,
+            'hot-water-cost-current': 256, 'hot-water-cost-potential': 104, 'total-floor-area': 57.2,
+            'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': '1st', 'flat-top-storey': 'Y',
+            'flat-storey-count': 2.0, 'main-heating-controls': 2603.0, 'multi-glaze-proportion': 100.0,
+            'glazed-type': 'double glazing installed during or after 2002', 'glazed-area': 'Normal',
+            'extension-count': 0.0, 'number-habitable-rooms': 3.0, 'number-heated-rooms': 3.0,
+            'low-energy-lighting': 77.0, 'number-open-fireplaces': 0.0,
+            'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor',
+            'hot-water-env-eff': 'Poor', 'floor-description': '(other premises below)', 'floor-energy-eff': None,
+            'floor-env-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good',
+            'windows-env-eff': 'Good', 'walls-description': 'Cavity wall, as built, insulated (assumed)',
+            'walls-energy-eff': 'Good', 'walls-env-eff': 'Good',
+            'secondheat-description': 'Portable electric heaters', 'sheating-energy-eff': None,
+            'sheating-env-eff': None, 'roof-description': '(another dwelling above)', 'roof-energy-eff': None,
+            'roof-env-eff': None, 'mainheat-description': 'Room heaters, electric',
+            'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Poor',
+            'mainheatcont-description': 'Programmer and appliance thermostats', 'mainheatc-energy-eff': 'Good',
+            'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 77% of fixed outlets',
+            'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good',
+            'main-fuel': 'electricity - this is for backwards compatibility only and should not be used',
+            'wind-turbine-count': 0.0, 'heat-loss-corridor': 'heated corridor', 'unheated-corridor-length': None,
+            'floor-height': 2.3, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N',
+            'mechanical-ventilation': 'natural', 'address': 'Apartment 13 The Quays, Burscough',
+            'local-authority-label': 'West Lancashire', 'constituency-label': 'West Lancashire',
+            'posttown': 'ORMSKIRK', 'construction-age-band': 'England and Wales: 2003-2006',
+            'lodgement-datetime': '2008-10-07 17:31:09', 'tenure': 'owner-occupied',
+            'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10012342725.0,
+            'uprn-source': 'Address Matched', 'used': None
+        },
+        "heating_recommendation_descriptions": [
+            "Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention "
+            "Storage Heater Controls"
+        ],
+        "heating_controls_recommendation_descriptions": [],
+        "notes": "This property has electric room heaters and is off gas so a boiler recommendation is not appropriate."
+                 "We would expect a high heat retention storage recommendation. The property is a flat and therefore"
+                 "we don't expect an air source heat pump recommendation. We also wouldn't expect a specific heating"
+                 "control recommendation here"
+    },
+    {
+        'lmk-key': '751851300152012022010205497220090', 'address1': '21, Fullers Close', 'address2': 'Kelvedon',
+        'address3': None, 'postcode': 'CO5 9JX', 'building-reference-number': 8075968, 'current-energy-rating': 'D',
+        'potential-energy-rating': 'D', 'current-energy-efficiency': 55, 'potential-energy-efficiency': 56,
+        'property-type_x': 'Bungalow', 'built-form_x': 'Detached', 'inspection-date': '2012-02-20',
+        'local-authority': 'E07000067', 'constituency': 'E14001045', 'county': 'Essex', 'lodgement-date': '2012-02-20',
+        'transaction-type': 'non marketed sale', 'environment-impact-current': 39, 'environment-impact-potential': 39,
+        'energy-consumption-current': 475, 'energy-consumption-potential': 472.0, 'co2-emissions-current': 5.4,
+        'co2-emiss-curr-per-floor-area': 84, 'co2-emissions-potential': 5.4, 'lighting-cost-current': 53.0,
+        'lighting-cost-potential': 40.0, 'heating-cost-current': 674.0, 'heating-cost-potential': 678.0,
+        'hot-water-cost-current': 110.0, 'hot-water-cost-potential': 110.0, 'total-floor-area': 64.45,
+        'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', 'flat-top-storey': None,
+        'flat-storey-count': None, 'main-heating-controls': '2402', 'multi-glaze-proportion': 100.0,
+        'glazed-type': 'double glazing installed before 2002', 'glazed-area': 'Normal', 'extension-count': 0.0,
+        'number-habitable-rooms': 3.0, 'number-heated-rooms': 3.0, 'low-energy-lighting': 67.0,
+        'number-open-fireplaces': 0.0, 'hotwater-description': 'Electric immersion, off-peak',
+        'hot-water-energy-eff': 'Average', 'hot-water-env-eff': 'Very Poor',
+        'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': None, 'floor-env-eff': None,
+        'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average',
+        'walls-description': 'Cavity wall, as built, insulated (assumed)', 'walls-energy-eff': 'Good',
+        'walls-env-eff': 'Good', 'secondheat-description': 'Room heaters, electric', 'sheating-energy-eff': None,
+        'sheating-env-eff': None, 'roof-description': 'Pitched, 300+ mm loft insulation',
+        'roof-energy-eff': 'Very Good',
+        'roof-env-eff': 'Very Good', 'mainheat-description': 'Electric storage heaters', 'mainheat-energy-eff': 'Poor',
+        'mainheat-env-eff': 'Very Poor', 'mainheatcont-description': 'Automatic charge control',
+        'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
+        'lighting-description': 'Low energy lighting in 67% of fixed outlets', 'lighting-energy-eff': 'Good',
+        'lighting-env-eff': 'Good', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0.0,
+        'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.38, 'photo-supply': 0.0,
+        'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', 'address': '21, Fullers Close, Kelvedon',
+        'local-authority-label': 'Braintree', 'constituency-label': 'Witham', 'posttown': 'COLCHESTER',
+        'construction-age-band': 'England and Wales: 1983-1990', 'lodgement-datetime': '2012-02-20 10:20:54',
+        'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 6.0, 'low-energy-fixed-light-count': 4.0,
+        'uprn': 100090311351.0, 'uprn-source': 'Address Matched', 'property-type_y': None, 'built-form_y': None,
+        'used': None
    }
+
 ]
+
+import random
+from pathlib import Path
+import inspect
+import pandas as pd
+
+# this can be used to get example data to build the test cases
+src_file_path = inspect.getfile(lambda: None)
+EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates"
+epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
+directory = random.sample(epc_directories, 1)[0]
+data = pd.read_csv(directory / "certificates.csv", low_memory=False)
+# Rename the columns to the same format as the api returns
+data.columns = [c.replace("_", "-").lower() for c in data.columns]
+
+used_examples = pd.DataFrame(
+    [
+        {
+            "mainheat-description": x["epc"]["mainheat-description"],
+            "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"],
+            "property-type": x["epc"]["property-type"],
+            "built-form": x["epc"]["built-form"],
+            "used": True
+        } for x in testing_examples
+    ]
+)
+
+data = data.merge(used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff"])
+data = data[pd.isnull(data["used"])]
+
+eg = data.sample(1).to_dict("records")[0]