Merge pull request #222 from Hestia-Homes/rdsap-data

Rdsap data - major changes
2026-06-08 11:17:27 +00:00 · 2023-09-12 17:54:48 +01:00 · 2023-09-12 17:54:48 +01:00 · a6d1ba6f85
commit a6d1ba6f85
parent e05e8ff636 dd49798dad
43 changed files with 2119 additions and 649 deletions
--- a/.coveragerc
+++ b/.coveragerc
@ -7,4 +7,6 @@ omit =
    model_data/__init__.py
    model_data/app.py
    model_data/plotting/*
-    recommendations/rdsap_tables.py
+    recommendations/rdsap_tables.py
+    model_data/simulation_system/*
+    model_data/cleaner_app.py
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@ -0,0 +1,37 @@
+name: Run unit tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    # For the moment, we just run with python 3.10
+    #    strategy:
+    #      matrix:
+    #        python-version: [ 3.8, 3.9, 3.10 ]
+
+    steps:
+      - uses: actions/checkout@v2
+      #      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          #          python-version: ${{ matrix.python-version }}
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r model_data/requirements/requirements.txt
+      - name: Run tests with pytest
+        run: |
+          pip install -r model_data/requirements/dev.txt
+          pytest
+#      - name: Upload coverage to Codecov
+#        uses: codecov/codecov-action@v2
+#        with:
+#          token: ${{ secrets.CODECOV_TOKEN }}
+#          fail_ci_if_error: true
--- a/backend/app/db/functions/recommendations_functions.py
+++ b/backend/app/db/functions/recommendations_functions.py
@ -91,6 +91,8 @@ def upload_recommendations(session, recommendations_to_upload, property_id):
    )]

    # Prepare data for bulk insert for RecommendationMaterials
+    # We can have multiple materials per recommendation. The aggregation of the materials will total the
+    # recommendation figures
    recommendation_materials_data = [
        {
            "recommendation_id": recommendation_id,
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@ -17,7 +17,9 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import IntegrityError, OperationalError
 from datetime import datetime
 import pandas as pd
-import requests
+
+# model apis
+from backend.ml_models.sap_change_model.api import SAPChangeModelAPI

 # database interaction functions
 from backend.app.db.functions.property_functions import (
@ -136,6 +138,13 @@ def insert_temp_recommendation_id(property_recommendations):
    return property_recommendations


+def score_measures():
+    """
+    This wrapper function prepares data to be passed to the sap model api
+    :return:
+    """
+
+
@router.post("/trigger")
 async def trigger_plan(body: PlanTriggerRequest):
    logger.info("Connecting to db")
@ -289,47 +298,12 @@ async def trigger_plan(body: PlanTriggerRequest):
            if wall_recomender.recommendations:
                property_recommendations.append(wall_recomender.recommendations)

-            # Use the optimiser to pick the default recommendations and decide if we need certain
-            # recommendations to get to the goal
+            # We insert temporary ids into the recommendations which is important for the optimiser later
            property_recommendations = insert_temp_recommendation_id(property_recommendations)

            if not property_recommendations:
                continue

-            input_measures = prepare_input_measures(property_recommendations, body.goal)
-
-            if body.budget:
-                optimiser = GainOptimiser(input_measures, max_cost=body.budget)
-            else:
-                # The minimum gain is the minimum number of SAP points required to get to the target SAP band
-                current_sap_points = int(p.data["current-energy-efficiency"])
-                target_sap_points = epc_to_sap_lower_bound(body.goal_value)
-
-                # If the gain is negative, the optimiser will return an empty solution
-                optimiser = CostOptimiser(
-                    input_measures, min_gain=target_sap_points - current_sap_points
-                )
-
-            optimiser.setup()
-            optimiser.solve()
-            solution = optimiser.solution
-
-            selected_recommendations = {r["id"] for r in solution}
-            # We'll use the set of selected recommendations to filter the recommendations to upload
-
-            property_recommendations = [
-                [
-                    {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
-                    for rec in recommendations_by_type
-                ]
-                for recommendations_by_type in property_recommendations
-            ]
-
-            # We'll also unlist the recommendations so they're a bit easier to handle from here onwards
-            property_recommendations = [
-                rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type
-            ]
-
            recommendations[p.id] = property_recommendations

            # Finally, we'll prepare data for predicting the impact on SAP
@ -351,25 +325,26 @@ async def trigger_plan(body: PlanTriggerRequest):
                'Suspended, no insulation (assumed)': 'Suspended, insulated (assumed)',
                'Solid, no insulation (assumed)': 'Solid, insulated (assumed)',
            }
-            for rec in property_recommendations:
-                scoring_dict = {
-                    "UPRN": p.data["uprn"],
-                    "id": "+".join([str(p.id), str(rec["recommendation_id"])]),
-                    "LOCAL_AUTHORITY": p.data["local-authority"],
-                    **starting_epc_data.to_dict("records")[0],
-                    **ending_epc_data.to_dict("records")[0],
-                    **fixed_data.to_dict("records")[0]
-                }
+            for recommendations_by_type in property_recommendations:
+                for rec in recommendations_by_type:
+                    scoring_dict = {
+                        "UPRN": p.data["uprn"],
+                        "id": "+".join([str(p.id), str(rec["recommendation_id"])]),
+                        "LOCAL_AUTHORITY": p.data["local-authority"],
+                        **starting_epc_data.to_dict("records")[0],
+                        **ending_epc_data.to_dict("records")[0],
+                        **fixed_data.to_dict("records")[0]
+                    }

-                # We update the description to indicate it's insulated
-                if rec["type"] == "wall_insulation":
-                    scoring_dict["WALLS_DESCRIPTION_ENDING"] = scoring_map[p.walls["clean_description"]]
-                elif rec["type"] == "floor_insulation":
-                    scoring_dict["FLOOR_DESCRIPTION_ENDING"] = scoring_map[p.floor["clean_description"]]
-                else:
-                    raise NotImplementedError("Implement me")
+                    # We update the description to indicate it's insulated
+                    if rec["type"] == "wall_insulation":
+                        scoring_dict["WALLS_DESCRIPTION_ENDING"] = scoring_map[p.walls["clean_description"]]
+                    elif rec["type"] == "floor_insulation":
+                        scoring_dict["FLOOR_DESCRIPTION_ENDING"] = scoring_map[p.floor["clean_description"]]
+                    else:
+                        raise NotImplementedError("Implement me")

-                recommendations_scoring_data.append(scoring_dict)
+                    recommendations_scoring_data.append(scoring_dict)

        recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)

@ -410,33 +385,79 @@ async def trigger_plan(body: PlanTriggerRequest):

        save_dataframe_to_s3_parquet(
            df=recommendations_scoring_data,
-            bucket_name="retrofit-data-dev",
+            bucket_name="retrofit-data-{environment}".format(environment=get_settings().ENVIRONMENT),
            file_key=file_location
        )

-        # Call the sap change model
-        response = requests.post(
-            url="https://api.dev.hestia.homes/sapmodel/predict",
-            json={
-                "file_location": "s3://retrofit-data-dev/" + file_location,
-                "property_id": 999,
-                "portfolio_id": 4,
-                "created_at": created_at
-            }
+        sap_change_model_api = SAPChangeModelAPI()
+        response = sap_change_model_api.predict(
+            file_location="s3://retrofit-data-dev/" + file_location,
+            created_at=created_at,
+            portfolio_id=body.portfolio_id
        )
-        # TODO: Handle the response depending on response code

        # Retrieve the predictions
-        predictions = read_csv_from_s3(
-            bucket_name="retrofit-sap-predictions-dev",
-            filepath=f"{body.portfolio_id}/999/{created_at}.csv"
-        )
-        predictions = pd.DataFrame(predictions)
+        predictions = pd.DataFrame(read_csv_from_s3(
+            bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT),
+            filepath=response["storage_filepath"]
+        ))
        # We round the predictions
        predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(0)
        # Extract property_id and recommendation_id
        predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True)

+        # Insert the predictions into the recommendations and run the optimiser
+        for property_id in recommendations.keys():
+
+            property = [p for p in input_properties if p.id == property_id][0]
+            property_predictions = predictions[predictions["property_id"] == str(property_id)]
+
+            for recommendations_by_type in recommendations[property_id]:
+                for rec in recommendations_by_type:
+                    rec["sap_points"] = property_predictions[property_predictions["recommendation_id"] == str(
+                        rec["recommendation_id"]
+                    )]["RDSAP_CHANGE"].values[0]
+
+                    if not rec["sap_points"]:
+                        raise ValueError("Sap points missing")
+
+            input_measures = prepare_input_measures(recommendations[property_id], body.goal)
+
+            if body.budget:
+                optimiser = GainOptimiser(input_measures, max_cost=body.budget)
+            else:
+                # The minimum gain is the minimum number of SAP points required to get to the target SAP band
+                current_sap_points = int(property.data["current-energy-efficiency"])
+                target_sap_points = epc_to_sap_lower_bound(body.goal_value)
+
+                # If the gain is negative, the optimiser will return an empty solution
+                optimiser = CostOptimiser(
+                    input_measures, min_gain=target_sap_points - current_sap_points
+                )
+
+            optimiser.setup()
+            optimiser.solve()
+            solution = optimiser.solution
+
+            selected_recommendations = {r["id"] for r in solution}
+
+            # We'll use the set of selected recommendations to filter the recommendations to upload
+            final_recommendations = [
+                [
+                    {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
+                    for rec in recommendations_by_type
+                ]
+                for recommendations_by_type in recommendations[property_id]
+            ]
+
+            # We'll also unlist the recommendations so they're a bit easier to handle from here onwards
+            final_recommendations = [
+                rec for recommendations_by_type in final_recommendations for rec in recommendations_by_type
+            ]
+            # We update recommendations[property_id]
+
+            recommendations[property_id] = final_recommendations
+
        # 1) the property data
        # 2) the property details (epc)
        # 3) the recommendations
@ -457,16 +478,6 @@ async def trigger_plan(body: PlanTriggerRequest):
            if not recommendations_to_upload:
                continue

-            property_predictions = predictions[predictions["property_id"] == str(p.id)]
-            for rec in recommendations_to_upload:
-                # Insert the prediction for sap points
-                rec["sap_points"] = property_predictions[property_predictions["recommendation_id"] == str(
-                    rec["recommendation_id"]
-                )]["RDSAP_CHANGE"].values[0]
-
-                if not rec["sap_points"]:
-                    raise ValueError("Sap points missing")
-
            # Create a plan
            new_plan_id = create_plan(
                session,
--- a/backend/ml_models/sap_change_model/init.py
+++ b/backend/ml_models/sap_change_model/init.py
--- a/backend/ml_models/sap_change_model/api.py
+++ b/backend/ml_models/sap_change_model/api.py
@ -0,0 +1,44 @@
+import requests
+from requests.exceptions import RequestException
+from utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class SAPChangeModelAPI:
+    def __init__(self, base_url="https://api.dev.hestia.homes"):
+        self.base_url = base_url
+
+    def predict(self, file_location, property_id="", portfolio_id=4, created_at=None):
+        """Makes a POST request to the SAP Change Model API with the provided parameters.
+
+        Args:
+            file_location (str): The file location to be passed in the request payload.
+            property_id (int, optional): The property ID to be passed in the request payload. Defaults to 999.
+            portfolio_id (int, optional): The portfolio ID to be passed in the request payload. Defaults to 4.
+            created_at (str, optional): The creation timestamp to be passed in the request payload. Defaults to None.
+
+        Returns:
+            dict: The API response as a dictionary if the request was successful, None otherwise.
+        """
+        url = f"{self.base_url}/sapmodel/predict"
+        payload = {
+            "file_location": f"s3://retrofit-data-dev/{file_location}",
+            "property_id": property_id,
+            "portfolio_id": portfolio_id,
+            "created_at": created_at
+        }
+
+        try:
+            response = requests.post(url, json=payload)
+
+            # Check if the response status code is 2xx (success)
+            response.raise_for_status()
+
+            # Return the JSON response as a Python dictionary
+            return response.json()
+        except RequestException as e:
+            logger.error(f"An error occurred: {e}")
+            # In case of an error, you might want to return None or raise the exception
+            # depending on how you want to handle errors in your application
+            return None
--- a/model_data/EpcClean.py
+++ b/model_data/EpcClean.py
@ -2,6 +2,8 @@ from typing import List, Dict, Any
 from collections import Counter
 from collections import defaultdict

+import pandas as pd
+
 from model_data.utils import correct_spelling
 from model_data.epc_attributes.FloorAttributes import FloorAttributes
 from model_data.epc_attributes.HotWaterAttributes import HotWaterAttributes
@ -47,6 +49,9 @@ class EpcClean:
        else:
            self.lighting_averages = lighting_averages

+    def insert_extra_data(self):
+        pass
+
    def _calculate_lighting_averages(self):

        """
@ -97,7 +102,7 @@ class EpcClean:
        self._init_empty_cleaned_obj()

        for field in self.CLEANING_FIELDS:
-            self.unique_vals[field] = Counter([v[field] for v in self.data])
+            self.unique_vals[field] = Counter([v[field] for v in self.data if not pd.isnull(v[field])])

        self.clean_wrapper(field="floor-description", cleaning_cls=FloorAttributes)
        self.clean_wrapper(field="hotwater-description", cleaning_cls=HotWaterAttributes)
--- a/model_data/app.py
+++ b/model_data/app.py
@ -1,94 +0,0 @@
-from tqdm import tqdm
-import os
-
-from model_data.config import EPC_AUTH_TOKEN
-from epc_api.client import EpcClient
-from model_data.downloader import pagenated_epc_download
-from model_data.EpcClean import EpcClean
-from model_data.analysis.UvalueEstimations import UvalueEstimations
-from model_data.analysis.SapModel import SapModel
-
-LAND_REGISTRY_PATHS = [
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-monthly-update-new-version.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2022 (1).csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2021.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2020.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2019.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2018.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part1.csv",
-    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part2.csv",
-]
-
-
-def app():
-    """
-    For a pre-defined list of constituencies and property data_types, we'll download EPC data from the API
-    and produce a dataset of cleaned fields so that when we get new properties, we can quickly
-    sanitise any description data
-    :return:
-    """
-
-    epc_client = EpcClient(auth_token=EPC_AUTH_TOKEN)
-
-    constituencies = {'E14000555', 'E14000726', 'E14000720', 'E14000721', 'E14000553', 'E14000752'}
-    property_types = ["bungalow", "flat", "house", "maisonette", "park home"]
-    floor_areas = ["unknown", "s", "m", "l", "xl", "xxl", "xxxl"]
-
-    # We pull properties from local authorities, by property type. This will allow us to build
-    # a dataset of up to 10k properties per local authority/property type combination
-    # For particularly old EPC data, we have inconsistent records so we'll only include EPCS that were
-    # conducted after 2010, since SAP09 was introduced in 2009 an later SAP12 was introduced in England
-    # and Wales from 31 July 2014
-    # Download data from August 2014 onwards
-    data = []
-    for c in tqdm(constituencies):
-        for pt in property_types:
-            for fa in floor_areas:
-                data.extend(
-                    pagenated_epc_download(
-                        client=epc_client,
-                        params={
-                            "constituency": c,
-                            "property-type": pt,
-                            "from-month": 8,
-                            "from-year": 2014,
-                            "floor-area": fa,
-                        },
-                        page_size=5000,
-                        n_pages=10,
-                    )
-                )
-
-    # Production of sample data for land registry
-    # address_meta = [
-    #     {
-    #         "postcode": x["postcode"].upper(),
-    #         "address1": x["address1"].upper(),
-    #         "address2": x["address2"].upper(),
-    #         "address3": x["address3"].upper(),
-    #         "address": x["address"],
-    #         "uprn": x["uprn"]
-    #     } for x in data
-    # ]
-    #
-    # import pickle
-    # with open("sample_addresses.pkl", "wb") as f:
-    #     pickle.dump(address_meta, f)
-
-    # Incorporate input data into cleaning
-    cleaner = EpcClean(data)
-    lighting_averages = cleaner.lighting_averages
-    # TODO: WE need to store lighting_averages to a db
-    #       We should also extend these averages so they're by more variables (property type, age band, constituency,
-    #       etc)
-    cleaner.clean()
-    # TODO: cleaner.cleaned datasets to a db
-
-    # TODO: Add property age band into this
-    uvalue_estimates = UvalueEstimations(data=data)
-    uvalue_estimates.get_estimates(cleaner=cleaner)
-    # TODO: Store these to a db
-
-    sap_model = SapModel(data=data, cleaner=cleaner)
-    sap_model.run()
-    # TODO: Store outputs to db
--- a/model_data/cleaner_app.py
+++ b/model_data/cleaner_app.py
@ -0,0 +1,95 @@
+from tqdm import tqdm
+import os
+import pandas as pd
+import msgpack
+
+from model_data.EpcClean import EpcClean
+from model_data.analysis.UvalueEstimations import UvalueEstimations
+from model_data.simulation_system.core.Settings import EARLIEST_EPC_DATE
+from pathlib import Path
+from model_data.utils import save_data_to_s3
+
+LAND_REGISTRY_PATHS = [
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-monthly-update-new-version.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2022 (1).csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2021.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2020.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2019.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2018.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part1.csv",
+    os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part2.csv",
+]
+
+EPC_DIRECTORY = Path(__file__).parent / "model_data" / "simulation_system" / "data" / "all-domestic-certificates"
+
+ENVIRONMENT = os.getenv("ENVIRONMENT", "dev")
+
+
+def app():
+    """
+    For a pre-defined list of constituencies and property data_types, we'll download EPC data from the API
+    and produce a dataset of cleaned fields so that when we get new properties, we can quickly
+    sanitise any description data
+
+    Currently, this application is just run on a local machine
+    """
+
+    cleaned_data = {}
+    epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
+    for directory in tqdm(epc_directories):
+        directory_destructured = str(directory).split("/")[-1].split("-")
+        gss_code = directory_destructured[1]
+        local_authority = directory_destructured[2]
+
+        data = pd.read_csv(directory / "certificates.csv", low_memory=False)
+        # Rename the columns to the same format as the api returns
+        data.columns = [c.replace("_", "-").lower() for c in data.columns]
+        # Take just date before the date threshold
+        data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE]
+
+        # Convert to list of dictioaries as returned by the api
+        data = data.to_dict("records")
+
+        # Incorporate input data into cleaning
+        cleaner = EpcClean(data)
+
+        cleaner.clean()
+        # Extended cleaned_data
+        for k, data in cleaner.cleaned.items():
+            if k not in cleaned_data:
+                cleaned_data[k] = data
+            else:
+                existing_descriptions = [x["original_description"] for x in cleaned_data[k]]
+                new_data = [x for x in data if x["original_description"] not in existing_descriptions]
+                cleaned_data[k].extend(new_data)
+
+        # TODO: Add property age band into this
+        # uvalue_estimates = UvalueEstimations(data=data)
+        # uvalue_estimates.get_estimates(cleaner=cleaner)
+        # # TODO: Store these to a s3
+        # uvalue_estimates.walls
+        # uvalue_estimates.floors
+        # uvalue_estimates.roofs
+
+    # Basic check to make sure all descriptions are unique
+    for _, cleaned in cleaned_data.items():
+        descriptions = [x["original_description"] for x in cleaned]
+        if len(descriptions) != len(set(descriptions)):
+            raise ValueError("Duplicated descriptions found, check me")
+
+    # We store a singular file however we could store the data under the following file path:
+    # cleaned_epc_data/{component}/{original_description}/cleaned.bson
+    # where component is one of the keys of cleaned_data. If we store it against the original data, this
+    # data being read in will be extremely small, meaning quicker load times. We'll begin by storing as a single
+    # file and monitor usage patterns to see if it makes sense to split the data up
+    save_data_to_s3(
+        data=msgpack.packb(cleaned_data, use_bin_type=True),
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name=f"retrofit-data-{ENVIRONMENT}"
+    )
+
+    save_data_to_s3(
+        data=msgpack.packb(cleaned_data, use_bin_type=True),
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name=f"retrofit-data-{ENVIRONMENT}"
+    )
--- a/model_data/epc_attributes/FloorAttributes.py
+++ b/model_data/epc_attributes/FloorAttributes.py
@ -1,3 +1,4 @@
+import re
 from typing import Dict, Union
 from model_data.BaseUtility import Definitions
 from model_data.epc_attributes.attribute_utils import extract_thermal_transmittance, extract_component_types
@ -14,6 +15,23 @@ class FloorAttributes(Definitions):

    WELSH_TEXT = {
        "(anheddiad arall islaw)": "(another dwelling below)",
+        "solet, dim inswleiddio (rhagdybiaeth)": "solid, no insulation (assumed)",
+        "solet, dim inswleiddio": "solid, no insulation)",
+        "crog, dim inswleiddio (rhagdybiaeth)": "suspended, no insulation (assumed)",
+        "crog, dim inswleiddio": "suspended, no insulation",
+        "(eiddo arall islaw)": "(other premises below)",
+        "solet, inswleiddio cyfyngedig (rhagdybiaeth)": "solid, limited insulation (assumed)",
+        "solet, inswleiddio cyfyngedig": "solid, limited insulation",
+        "crog, wedigçöi inswleiddio (rhagdybiaeth)": "suspended, insulated (assumed)",
+        "crog, wedigçöi inswleiddio": "suspended, insulated",
+        "igçör awyr y tu allan, dim inswleiddio (rhagdybiaeth)": "to external air, no insulation (assumed)",
+        "igçör awyr y tu allan, dim inswleiddio": "to external air, no insulation",
+        "i ofod heb ei wresogi, wedigçöi inswleiddio (rhagdybiaeth)": "to unheated space, insulated (assumed)",
+        "i ofod heb ei wresogi, wedigçöi inswleiddio": "to unheated space, insulated",
+        "solet, wedigçöi inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)",
+        "solet, wedigçöi inswleiddio": "solid, insulated",
+        "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
+        "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation"
    }

    def __init__(self, description: str):
@ -23,11 +41,7 @@ class FloorAttributes(Definitions):
            description in self.OBSERVED_ERRORS)

        # Try and perform a translation, incase it's in welsh
-        translation = self.WELSH_TEXT.get(self.description)
-
-        if translation:
-            self.nodata = False
-            self.description = translation
+        self.translate_welsh_text()

        if not self.nodata and not any(
            rt in self.description for rt in
@ -35,6 +49,30 @@ class FloorAttributes(Definitions):
        ):
            raise ValueError('Invalid description')

+    def translate_welsh_text(self):
+
+        uvalue_match = re.search(
+            r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k', self.description
+        )
+
+        uvalue_match2 = re.search(
+            r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description
+        )
+
+        # Step 2: Generalized translation with placeholder
+        if uvalue_match is not None or uvalue_match2 is not None:
+            if uvalue_match is not None:
+                uvalue = uvalue_match.group(1)
+            else:
+                uvalue = uvalue_match2.group(1)
+            self.description = f"average thermal transmittance {uvalue} w/m-¦K"
+
+        else:
+            translation = self.WELSH_TEXT.get(self.description)
+            if translation:
+                self.nodata = False
+                self.description = translation
+
    def process(self) -> Dict[str, Union[str, bool, int, None]]:

        if self.nodata:
--- a/model_data/epc_attributes/HotWaterAttributes.py
+++ b/model_data/epc_attributes/HotWaterAttributes.py
@ -15,7 +15,10 @@ class HotWaterAttributes(Definitions):
        'oil boiler',  # A boiler that uses oil as fuel to heat water
        'electric instantaneous',  # Similar to gas instantaneous, but uses electricity as its energy source
        'gas multipoint',  # A gas water heater that can supply hot water to multiple points of use at once
-        'heat pump'  # A general category for heat pumps, regardless of the energy source
+        'heat pump',  # A general category for heat pumps, regardless of the energy source
+        'solid fuel boiler',  # burns solid materials to generate heat for water heating and/or space heating
+        'solid fuel range cooker',
+        'room heaters',  # Generic/unspecified category
    ]

    # SYSTEM_TYPES refer to the larger system within which the heater operates.
@ -83,6 +86,7 @@ class HotWaterAttributes(Definitions):
    # not common, especially in modern homes.
    APPLIANCE_SYSTEMS = [
        'gas range cooker',  # A gas-powered range cooker
+        'oil range cooker'
    ]

    # Descriptions which represent the same thing
@ -92,12 +96,33 @@ class HotWaterAttributes(Definitions):

    WELSH_TEXT = {
        "ogçör brif system": "from main system",
+        "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery",
+        "bwyler/cylchredydd nwy": "gas boiler/circulator",
+        "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat",
+        "twymwr tanddwr, an-frig": "electric immersion, off-peak",
+        "ogçör brif system, gydag ynnigçör haul": "from main system, plus solar",
+        "twymwr tanddwr, tarriff safonol": "electric immersion, standard tariff",
+        "trydan ar unwaith yn y fan lle maegçön cael ei ddefnyddio": 'electric instantaneous at point of use',
+        "o gynllun cymunedol": "community scheme",
+        "o'r brif system": "from main system",
+        "trydan ar unwaith yn y fan lle mae'n cael ei ddefnyddio": 'electric instantaneous at point of use',
+        "popty estynedig olew, dim thermostat ar y silindr": "oil range cooker, no cylinder thermostat",
+        "cynllun cymunedol": "community scheme",
+        "nwy wrth fwy nag un pwynt": "gas multipoint",
+        "popty estynedig olew": "oil range cooker",
+        "dim system ar gael rhagdybir bod twymwr tanddwr trydan": "no system present electric immersion assumed",
+        "o'r brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat",
+        "trydan ar unwaith yn y fan lle maegçön cael ei ddefnyddio, adfer gwres d+¦r gwastraff": "electric "
+                                                                                                 "instantaneous at "
+                                                                                                 "point of use, "
+                                                                                                 "waste water heat "
+                                                                                                 "recovery"
    }

    def __init__(self, description: str):
-        self.description: str = clean_description(description.lower())
+        self.description: str = clean_description(description.lower()).strip()

-        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES
+        self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES

        translation = self.WELSH_TEXT.get(self.description)

@ -105,7 +130,7 @@ class HotWaterAttributes(Definitions):
            self.nodata = False
            self.description = translation

-        if not any(
+        if not self.nodata and not any(
            self._keyword_in_description(keywords)
            for keywords in [
                self.HEATER_TYPES,
@ -118,6 +143,7 @@ class HotWaterAttributes(Definitions):
                self.CHP_SYSTEMS,
                self.NO_SYSTEM_PRESENT_KEYWORDS,
                self.APPLIANCE_SYSTEMS,
+                self.DISTRIBUTION_SYSTEM_KEYWORDS
            ]
        ):
            raise ValueError('Invalid description')
--- a/model_data/epc_attributes/LightingAttributes.py
+++ b/model_data/epc_attributes/LightingAttributes.py
@ -4,9 +4,20 @@ from model_data.utils import correct_spelling


 class LightingAttributes:
+    WELSH_TEXT = {
+        "goleuadau ynni-isel ym mhob un ogçör mannau gosod": "low energy lighting in all fixed outlets",
+        "dim goleuadau ynni-isel": "no low energy lighting",
+        "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets'
+    }

    def __init__(self, description, averages):
        self.description: str = clean_description(description.lower())
+
+        translation = self.WELSH_TEXT.get(self.description)
+        if translation:
+            self.nodata = False
+            self.description = translation
+
        self.description = correct_spelling(self.description)
        self.averages = averages

@ -20,6 +31,9 @@ class LightingAttributes:
        if "all fixed outlets" in description:
            return {"low_energy_proportion": 1}

+        if "excellent lighting efficiency" in description:
+            return {"low_energy_proportion": 1}
+
        if ('good lighting efficiency' in description) or ('excellent lighting efficiency' in description) or \
            ('below average lighting efficiency' in description):
            average = [
--- a/model_data/epc_attributes/MainFuelAttributes.py
+++ b/model_data/epc_attributes/MainFuelAttributes.py
@ -26,6 +26,15 @@ class MainFuelAttributes(Definitions):
        # Wood pellets have a higher energy density than wood chips. This is due to their manufacturing process,
        # which compresses the wood and removes most of the moisture, making them more efficient as a fuel
        'wood pellets',
+        'b30k',
+        'dual fuel appliance mineral and wood',
+        'coal',
+        'b30d',
+        'bioethanol',
+        'solid fuel',
+        'manufactured smokeless fuel',
+        "lng",  # Liquified natural gas
+        "electric heat pump"
    ]

    COMPLEX_FUEL_KEYWORDS = [
--- a/model_data/epc_attributes/MainheatAttributes.py
+++ b/model_data/epc_attributes/MainheatAttributes.py
@ -1,5 +1,5 @@
 from model_data.BaseUtility import Definitions
-from model_data.epc_attributes.attribute_utils import clean_description, process_part
+from model_data.epc_attributes.attribute_utils import clean_description, process_part, switch_chars
 from typing import Dict, Union


@ -8,24 +8,69 @@ class MainHeatAttributes(Definitions):
        "boiler", "air source heat pump", "room heaters", "electric storage heaters", "warm air",
        "electric underfloor heating", "electric ceiling heating", "community scheme",
        "ground source heat pump", "no system present", "portable electric heaters",
-        "water source heat pump", "electric heat pumps",
+        "water source heat pump", "electric heat pump",
        # "Micro-cogeneration", also known as micro combined heat and power (micro-CHP), is a technology that
        # generates heat and electricity simultaneously from the same energy source in residential or commercial
        # buildings. The main output of micro-CHP systems is heat, with electricity generation as a secondary output.
-        "micro-cogeneration"
+        "micro-cogeneration",
+        "solar assisted heat pump",
+        "exhaust source heat pump",
+        "community heat pump",
    ]
-    FUEL_TYPES = ["electric", "mains gas", "wood logs", "LPG", "coal", "oil", "wood pellets", "anthracite",
-                  "dual fuel mineral and wood", "smokeless fuel", "lpg"]
+    FUEL_TYPES = ["electric", "mains gas", "wood logs", "coal", "oil", "wood pellets", "anthracite",
+                  "dual fuel mineral and wood", "smokeless fuel", "lpg", "b30k"]
    DISTRIBUTION_SYSTEMS = ["radiators", "fan coil units", "pipes in screed above insulation",
                            "pipes in insulated timber floor", "pipes in concrete slab"]
    OTHERS = ["assumed", "electricaire", "assumed for most rooms"]

    WELSH_TEXT = {
        "bwyler a rheiddiaduron, nwy prif gyflenwad": "boiler and radiators, mains gas",
+        "st+¦r wresogyddion trydan": "electric storage heaters",
+        "bwyler a rheiddiaduron, olew": "boiler and radiators, oil",
+        "heat pumptrydan": "electric heat pump",
+        "bwyler a rheiddiaduron, trydan": "boiler and radiators, electric",
+        "bwyler a gwres dan y llawr, olew": "boiler and underfloor heating, oil",
+        'bwyler a rheiddiaduron, lpg': 'boiler and radiators, lpg',
+        "gwresogyddion ystafell, trydan": "room heaters, electric",
+        "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan": "air source heat pump, underfloor heating, "
+                                                                    "electric",
+        "cynllun cymunedol": "community scheme",
+        "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas",
+        "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs',
+        "bwyler a rheiddiaduron, tanwydd di-fwg": "boiler and radiators, smokeless fuel",
+        "bwyler a rheiddiaduron, b30k": "boiler and radiators, b30k",
+        "bwyler a rheiddiaduron, glo": "boiler and radiators, coal",
+        "dim system ar gael, rhagdybir bod gwresogyddion trydan": "no system present, electric heaters assumed",
+        "gwresogyddion ystafell, glo carreg": "room heaters, coal",
+        "pwmp gwres sygçön tarddu yn yr awyr, rheiddiaduron, trydan": "air source heat pump, radiators, electric",
+        "gwresogyddion ystafell, nwy prif gyflenwad": "room heaters, mains gas",
+        "bwyler a rheiddiaduron, dau danwydd mwynau a choed": "boiler and radiators, dual fuel mineral and wood",
+        "gwresogyddion ystafell, dau danwydd mwynau a choed": "room heaters, dual fuel mineral and wood",
+        "pwmp gwres sygçön tarddu yn y ddaear, dan y llawr, trydan": "ground source heat pump, underfloor, electric",
+        "gwresogi dan y llawr trydan": "electric underfloor heating",
+        # This descripton is slightly unclear & was repeated
+        "st+¦r wresogyddion trydan, st+¦r wresogyddion trydan": "room heaters, electric",
+        "pwmp gwres sygçön tarddu yn y ddaear, rheiddiaduron, trydan": "ground source heat pump, radiators, electric",
+        "gwresogyddion ystafell, pelenni coed": "room heaters, wood pellets",
+        "gwresogyddion ystafell, glo": "room heaters, coal",
+        "bwyler a gwres dan y llawr, lpg": "boiler and underfloor heating, lpg",
+        "bwyler a gwres dan y llawr, trydan": "boiler and underfloor heating, electric"
    }

+    REMAP = {
+        "electric ceiling": "electric ceiling heating",
+        "electric heat pumps": "electric heat pump",
+        "solar-assisted heat pump": "solar assisted heat pump"
+    }
+
+    edge_case_result = {}
+    is_edge_case = False
+
    def __init__(self, description: str):
-        self.description: str = clean_description(description.lower())
+
+        self.description = switch_chars(description.lower())
+
+        self.description: str = clean_description(self.description).strip()
        # Remove special characters
        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES

@ -34,12 +79,57 @@ class MainHeatAttributes(Definitions):
            self.nodata = False
            self.description = translation

-        if not description or not any(
+        remapped = []
+        for term in self.description.split(", "):
+            remap = self.REMAP.get(term)
+            if remap:
+                remapped.append(remap)
+            else:
+                remapped.append(term)
+        remapped = ", ".join(remapped)
+
+        self.description = remapped
+
+        self.process_edge_cases()
+
+        if (not description or not any(
            rt in self.description for rt in
            self.HEAT_SYSTEMS + self.FUEL_TYPES + self.DISTRIBUTION_SYSTEMS + self.OTHERS
-        ):
+        ) and not self.is_edge_case):
            raise ValueError('Invalid description')

+    def process_edge_cases(self) -> (dict, bool):
+        """
+        We handle some edge cases that will cause issues, for example descriptions that are missing a
+        heating system
+        :return: truple containing dictionary result, and boolean is_edge_case
+        """
+
+        self.edge_case_result = {}
+        self.is_edge_case = False
+
+        if self.description == ", underfloor, electric":
+            self.edge_case_result["has_electric"] = True
+            self.edge_case_result['has_underfloor_heating'] = True
+            self.is_edge_case = True
+            return
+
+        if self.description == ", radiators, electric":
+            self.edge_case_result["has_electric"] = True
+            self.edge_case_result['has_radiators'] = True
+            self.is_edge_case = True
+            return
+
+        if self.description == ", underfloor":
+            self.edge_case_result['has_underfloor_heating'] = True
+            self.is_edge_case = True
+            return
+
+        if self.description == ", wood pellets":
+            self.edge_case_result['has_wood_pellets'] = True
+            self.is_edge_case = True
+            return
+
    def process(self) -> Dict[str, Union[str, bool]]:

        result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS}
@ -51,11 +141,17 @@ class MainHeatAttributes(Definitions):
        if self.nodata:
            return result

+        if self.is_edge_case:
+            result.update(self.edge_case_result)
+            return result
+
        description = self.description.split(',')

        # Process each part separately
        for part in description:
            part = part.strip()  # remove leading/trailing white spaces
+            if not part:
+                continue

            # Heating Systems
            process_part(result, part, self.HEAT_SYSTEMS, 'has_')
--- a/model_data/epc_attributes/MainheatControlAttributes.py
+++ b/model_data/epc_attributes/MainheatControlAttributes.py
@ -67,9 +67,59 @@ class MainheatControlAttributes(Definitions):
        'at least two room thermostats'
    ]

+    RATE_CONTROL_KEYWORDS = [
+        'single rate heating',
+    ]
+
+    # Sufficiently similar descriptions to be remapped
+    TO_REMAP = {
+        "celect control": 'celect-type control',
+        "celect controls": 'celect-type control',
+    }
+
+    WELSH_TEXT = {
+        "rhaglennydd, dim thermostat ystafell": "programmer, no room thermostat",
+        "rhaglennydd a thermostat ystafell": "programmer and room thermostat",
+        "rheoligçör t+ól +ó llaw": "manual charge control",
+        "rheoli'r t+ól +ó llaw": "manual charge control",
+        "rheolaeth amser a rheolaeth parthau tymheredd": "time and temperature zone control",
+        "rhaglennydd a thermostatau ar y cyfarpar": "programmer, room thermostat",
+        "rheolyddion i wresogyddion storio sygçön cadw llawer o wres": "controls for high heat retention storage "
+                                                                       "heaters",
+        "t+ól un gyfradd, rhaglennydd a thermostat ystafell": "single rate heating, programmer and room thermostat",
+        "rhaglennydd ac o leiaf ddau thermostat ystafell": "programmer and at least two room thermostats",
+        "thermostat ystafell yn unig": "room thermostat only",
+        "dim rheolaeth amser na rheolaeth thermostatig ar dymheredd yr ystafell": "no time or thermostatic control of "
+                                                                                  "room temperature",
+        "rheoli gwefr drydanol yn awtomatig": "automatic charge control",
+        'system dalu wedigçöi chysylltu +ó defnyddio gwres cymunedol, thermostat ystafell yn unig': "charging system "
+                                                                                                    "linked to use of"
+                                                                                                    " community "
+                                                                                                    "heating, "
+                                                                                                    "room thermostat "
+                                                                                                    "only",
+        "dim": "none",
+        "dim rheolaeth thermostatig ar dymheredd yr ystafell": "no thermostatic control of room temperature",
+        "thermostatau ar y cyfarpar": "appliance thermostats",
+        "rhaglennydd a thermostatau ystafell": "programmer and room thermostats",
+        "system dalu wedigçöi chysylltu +ó defnyddio gwres cymunedol, rhaglennydd a thermostat ystafell": (
+            "charging system linked to use of community heating, programmer and room thermostat"
+        ),
+    }
+
    def __init__(self, description: str):
-        self.description: str = clean_description(description.lower())
-        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES
+        self.description: str = clean_description(description.lower()).strip()
+        self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES
+
+        translation = self.WELSH_TEXT.get(self.description)
+        if translation:
+            self.nodata = False
+            self.description = translation
+
+        # Remap
+        remapped = self.TO_REMAP.get(self.description)
+        if remapped:
+            self.description = remapped

        if not self.nodata:
            if not any(
@ -81,7 +131,8 @@ class MainheatControlAttributes(Definitions):
                    self.DHW_CONTROL_KEYWORDS,
                    self.COMMUNITY_HEATING_KEYWORDS,
                    self.TRVS_KEYWORDS,
-                    self.NO_CONTROL_SYSTEM_KEYWORDS
+                    self.NO_CONTROL_SYSTEM_KEYWORDS,
+                    self.RATE_CONTROL_KEYWORDS
                ]
            ):
                raise ValueError('Invalid description')
@ -89,7 +140,7 @@ class MainheatControlAttributes(Definitions):
    def _keyword_in_description(self, keywords):
        return any(keyword in self.description for keyword in keywords)

-    def process(self) -> Dict[str, Union[str, bool]]:
+    def process(self) -> Dict[str, Union[str, bool, None]]:

        if self.nodata:
            result = {
@ -101,11 +152,12 @@ class MainheatControlAttributes(Definitions):
                "community_heating": False,
                "multiple_room_thermostats": False,
                "auxiliary_systems": False,
-                "trvs": False
+                "trvs": False,
+                "rate_control": False
            }
            return result

-        result: Dict[str, Union[str, bool]] = {
+        result: Dict[str, Union[str, bool, None]] = {
            "thermostatic_control": find_keyword(self.description, self.THERMOSTATIC_CONTROL_KEYWORDS),
            "charging_system": find_keyword(self.description, self.CHARGING_SYSTEM_KEYWORDS),
            "switch_system": find_keyword(self.description, self.SWITCH_SYSTEM_KEYWORDS),
@ -116,7 +168,11 @@ class MainheatControlAttributes(Definitions):
                phrase in self.description for phrase in self.MULTIPLE_ROOM_THERMOSTATS_PHRASES
            ),
            "auxiliary_systems": find_keyword(self.description, self.AUXILIARY_SYSTEM_KEYWORDS),
-            "trvs": find_keyword(self.description, self.TRVS_KEYWORDS)
+            "trvs": find_keyword(self.description, self.TRVS_KEYWORDS),
+            "rate_control": find_keyword(self.description, self.RATE_CONTROL_KEYWORDS),
        }

+        if result["no_control"] == 'no room thermostat':
+            result["thermostatic_control"] = None
+
        return result
--- a/model_data/epc_attributes/RoofAttributes.py
+++ b/model_data/epc_attributes/RoofAttributes.py
@ -10,6 +10,27 @@ class RoofAttributes(Definitions):

    WELSH_TEXT = {
        "ar oleddf, dim inswleiddio": "pitched, no insulation",
+        "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)",
+        "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)",
+        "ar oleddf, wedigçöi inswleiddio": "pitched, insulated",
+        "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)",
+        "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation",
+        "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters',
+        "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)",
+        "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation",
+        "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)",
+        "yn wastad, dim inswleiddio": "flat, no insulation",
+        "yn wastad, wedigçöi inswleiddio (rhagdybiaeth)": "flat, insulated (assumed)",
+        "yn wastad, wedigçöi inswleiddio": "flat, insulated",
+        "(eiddo arall uwchben)": "(another dwelling above)",
+        "(annedd arall uwchben)": "(another dwelling above)",
+        "ystafell(oedd) to, wedigçöi hinswleiddio": "roof room(s), insulated",
+        "ystafell(oedd) to, wedigçöi hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)",
+        "ystafell(oedd) to, inswleiddio cyfyngedig (rhagdybiaeth)": "roof room(s), limited insulation (assumed)",
+        "ystafell(oedd) to, inswleiddio cyfyngedig": "roof room(s), limited insulation",
+        "ystafell(oedd) to, nenfwd wedigçöi inswleiddio": "roof room(s), ceiling insulated",
+        "ystafell(oedd) to, dim inswleiddio (rhagdybiaeth)": "roof room(s), no insulation (assumed)",
+        "ystafell(oedd) to, dim inswleiddio": "roof room(s), no insulation",
    }

    def __init__(self, description: str):
@ -17,19 +38,58 @@ class RoofAttributes(Definitions):
        :param description: Description of the roof.
        """

-        self.description: str = description.lower()
+        self.description: str = description.lower().strip()
        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES

-        translation = self.WELSH_TEXT.get(self.description)
-        if translation:
-            self.nodata = False
-            self.description = translation
+        self.welsh_translation_search()

        if not self.nodata and not any(
            rt in self.description for rt in self.ROOF_TYPES + self.DWELLING_ABOVE + ["average thermal transmittance"]
        ):
            raise ValueError('Invalid description')

+    def welsh_translation_search(self):
+        """
+        For some descriptions,
+        we want to translate, however they have a consistent structure, where the only change
+        is the thickness of insulation. Instead of manually adding a record for each translation, we
+        search for regular expressions and translate
+        """
+
+        loft_insulation_thickness_match = re.search(r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", self.description)
+        loft_insulation_thickness_match2 = re.search(r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", self.description)
+        loft_insulation_thickness_match3 = re.search(r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof",
+                                                     self.description)
+
+        uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description)
+        uvalue_search2 = re.search(
+            r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description, re.IGNORECASE
+        )
+
+        # Step 2: Generalized translation with placeholder
+        if (loft_insulation_thickness_match is not None) | \
+            (loft_insulation_thickness_match2 is not None) | \
+            (loft_insulation_thickness_match3 is not None):
+            if loft_insulation_thickness_match is not None:
+                insulation_thickness = loft_insulation_thickness_match.group(1)
+            elif loft_insulation_thickness_match2 is not None:
+                insulation_thickness = loft_insulation_thickness_match2.group(1)
+            else:
+                insulation_thickness = loft_insulation_thickness_match3.group(1)
+
+            self.description = f"pitched, {insulation_thickness} loft insulation"
+        elif uvalue_search is not None or uvalue_search2 is not None:
+            if uvalue_search is not None:
+                uvalue = uvalue_search.group(1)
+            else:
+                uvalue = uvalue_search2.group(1)
+            self.description = f"average thermal transmittance {uvalue} W/m-¦K"
+        else:
+            translation = self.WELSH_TEXT.get(self.description)
+            if translation:
+                self.nodata = False
+                self.description = translation
+
    def process(self) -> Dict[str, Union[float, str, bool, None]]:

        result: Dict[str, Union[float, str, bool, None]] = {}
--- a/model_data/epc_attributes/WindowAttributes.py
+++ b/model_data/epc_attributes/WindowAttributes.py
@ -19,6 +19,17 @@ class WindowAttributes(Definitions):

    WELSH_TEXT = {
        "gwydrau dwbl llawn": "full double glazing",
+        "gwydrau dwbl rhannol": "partial double glazing",
+        "gwydrau dwbl gan mwyaf": "mostly double glazing",
+        "rhai gwydrau dwbl": "some double glazing",
+        "gwydrau sengl": "single glazed",
+        "ffenestri perfformiad uchel": "high performance glazing",
+        "gwydrau triphlyg llawn": "fully triple glazed",
+        "gwydrau triphlyg rhannol": "partial triple glazed",
+        "gwydrau triphlyg mwyaf": "mostly triple glazed",
+        "gwydrau eilaidd llawn": "full secondary glazing",
+        "gwydrau eilaidd mwyaf": "mostly secondary glazing",
+        "gwydrau eilaidd rhannol": "partial secondary glazing",
    }

    def __init__(self, description: str):
--- a/model_data/epc_attributes/attribute_utils.py
+++ b/model_data/epc_attributes/attribute_utils.py
@ -65,6 +65,20 @@ def clean_description(description: str) -> str:
    return description


+def switch_chars(description: str) -> str:
+    """
+    Switches specified characters in a description with a ,
+    Useful for descriptions like "Gas: mains gas"
+    """
+
+    # Switch : to ,
+    chars = [":"]
+    for char in chars:
+        description = description.replace(char, ",")
+
+    return description
+
+
 def process_part(result: Dict[str, Union[str, bool]], part: str, attr_list: List[str], prefix: str):
    """
    Process a part of the description with a given list of epc_attributes
--- a/model_data/requirements/requirements.txt
+++ b/model_data/requirements/requirements.txt
@ -20,3 +20,4 @@ pyspellchecker
 textblob
 boto3
 pyarrow
+msgpack==1.0.5
--- a/model_data/simulation_system/core/DataProcessor.py
+++ b/model_data/simulation_system/core/DataProcessor.py
@ -1,8 +1,8 @@
 from pathlib import Path
 import numpy as np
 import pandas as pd
-from BaseUtility import Definitions
-from simulation_system.core.Settings import (
+from model_data.BaseUtility import Definitions
+from model_data.simulation_system.core.Settings import (
    DATA_PROCESSOR_SETTINGS,
    EARLIEST_EPC_DATE,
    FULLY_GLAZED_DESCRIPTIONS,
@ -33,19 +33,132 @@ class DataProcessor:
    def insert_data(self, data: pd.DataFrame) -> None:
        self.data = data

+    def standardise_construction_age_band(self):
+        """
+        This function will tidy up some of the non-standard values that are populated in the construction age
+        band, which is useful for cleaning
+        """
+        bounds_map = {
+            "England and Wales: before 1900": {"l": 0, "u": 1899},
+            "England and Wales: 1930-1949": {"l": 1930, "u": 1949},
+            "England and Wales: 1900-1929": {"l": 1900, "u": 1929},
+            "England and Wales: 1950-1966": {"l": 1950, "u": 1966},
+            "England and Wales: 1967-1975": {"l": 1967, "u": 1975},
+            "England and Wales: 1976-1982": {"l": 1976, "u": 1982},
+            "England and Wales: 1983-1990": {"l": 1983, "u": 1990},
+            "England and Wales: 1991-1995": {"l": 1991, "u": 1995},
+            "England and Wales: 1996-2002": {"l": 1996, "u": 2002},
+            "England and Wales: 2003-2006": {"l": 2003, "u": 2006},
+            "England and Wales: 2007-2011": {"l": 2007, "u": 2011},
+            "England and Wales: 2012 onwards": {"l": 2012, "u": 3000},
+        }
+
+        remap = {
+            "England and Wales: 2007 onwards": "England and Wales: 2007-2011"
+        }
+
+        expanded_map = {
+            i: [
+                label for label, bounds in bounds_map.items() if (i <= bounds["u"]) and (i >= bounds['l'])
+            ][0] for i in range(0, 3001)
+        }
+
+        def is_int(x):
+            try:
+                int(x)
+                return True
+            except:
+                return False
+
+        def clean_construction_age_band(x):
+            # Firstly, we check if it's an error value
+            if x in Definitions.DATA_ANOMALY_MATCHES or x in [None, np.nan]:
+                return x
+
+            # Next, we check if it's a value in our map
+            if bounds_map.get(x):
+                return x
+
+            # We check if it's a standard remap value
+            remap_value = remap.get(x, None)
+            if remap_value:
+                return remap_value
+
+            # We check if it's a number
+            if is_int(x):
+                x_int = int(x)
+                return expanded_map[x_int]
+
+            raise NotImplementedError("Not handled the case for value %s" % x)
+
+        self.data["CONSTRUCTION_AGE_BAND_CLEANED"] = self.data["CONSTRUCTION_AGE_BAND"].apply(
+            lambda x: clean_construction_age_band(x)
+        )
+
+    def clean_missing_rooms(self):
+        """
+        For the number of heated rooms and number of habitable rooms, we clean these values up front,
+        based on property archetype and age
+
+        TODO: We could use a model based impution approach for possibly more accurate cleaning
+        """
+
+        self.data["POSTAL_AREA"] = self.data["POSTCODE"].apply(lambda x: x.split(" ")[0])
+
+        def apply_clean(data, matching_columns):
+
+            cleaning_data = data[~pd.isnull(data[col])].groupby(
+                matching_columns
+            )[col].median().reset_index()
+
+            data = data.merge(
+                cleaning_data, how="left", on=matching_columns, suffixes=("", "_CLEANING")
+            )
+
+            data[col] = np.where(pd.isnull(data[col]), data[f"{col}_CLEANING"], data[col])
+            data = data.drop(columns=f"{col}_CLEANING")
+            return data
+
+        for col in ["NUMBER_HEATED_ROOMS", "NUMBER_HABITABLE_ROOMS"]:
+
+            to_index = 3
+            matching_columns = ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND_CLEANED", "POSTAL_AREA"]
+            has_missings = pd.isnull(self.data[col]).sum()
+            while has_missings:
+                self.data = apply_clean(
+                    data=self.data,
+                    matching_columns=matching_columns[0:to_index + 1]
+                )
+                has_missings = pd.isnull(self.data[col]).sum()
+
+                if not has_missings or to_index == 0:
+                    # Check if we've gotten to index 0 and still have missings - something has gone wrong or
+                    # we have a very unique property type
+                    if has_missings:
+                        raise NotImplementedError("Handle this edge case, we still have missings for column %s" % col)
+
+                    break
+                to_index -= 1
+
    def pre_process(self) -> pd.DataFrame:
        """
        Load data and begin initial cleaning
        """
        if not self.data:
            self.load_data(low_memory=DATA_PROCESSOR_SETTINGS["low_memory"])
+
        self.confine_data()

-        # TODO: CLean number of heated rooms and habitable rooms
+        # We have some non-standard construction age bands which we'll clean for matching
+        self.standardise_construction_age_band()
+        self.clean_missing_rooms()
+
        self.recast_df_columns(
            column_mappings=DATA_PROCESSOR_SETTINGS["column_mappings"]
        )
        self.clean_multi_glaze_proportion()
+        self.clean_photo_supply()
+
        self.retain_multiple_epc_properties(
            epc_minimum_count=DATA_PROCESSOR_SETTINGS["epc_minimum_count"]
        )
@ -235,8 +348,7 @@ class DataProcessor:

        for key, values in column_mappings.items():
            if key not in self.data.columns:
-                print("Column mapping incorrectly specified")
-                exit(1)
+                raise ValueError("Column mapping incorrectly specified")
            for value in values:
                self.data[key] = self.data[key].astype(value)

@ -272,6 +384,13 @@ class DataProcessor:
        ) & (self.data["WINDOWS_DESCRIPTION"].isin(FULLY_GLAZED_DESCRIPTIONS))
        self.data.loc[no_multi_glaze_proportion_index, "MULTI_GLAZE_PROPORTION"] = 100

+    def clean_photo_supply(self) -> None:
+        """
+        We fill photo supply with zeros where it's missing
+        """
+
+        self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0)
+
    @staticmethod
    def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on):
        """
--- a/model_data/simulation_system/core/Settings.py
+++ b/model_data/simulation_system/core/Settings.py
@ -53,6 +53,11 @@ DEPLOYMENT_FOLDER = "deployment"
 TOTAL_FLOOR_AREA_NATIONAL_AVERAGE = 70
 FLOOR_HEIGHT_NATIONAL_AVERAGE = 2.45

+AVERAGE_FIXED_FEATURES = [
+    "TOTAL_FLOOR_AREA",
+    "FLOOR_HEIGHT"
+]
+
 COLUMNS_TO_MERGE_ON = [
    "PROPERTY_TYPE",
    "BUILT_FORM",
@ -103,12 +108,11 @@ COMPONENT_FEATURES = [
    "NUMBER_OPEN_FIREPLACES",
    "MAINHEATCONT_DESCRIPTION",
    "EXTENSION_COUNT",
+    "TOTAL_FLOOR_AREA",
+    "FLOOR_HEIGHT",
    # 'GLAZED_AREA',  # May not need this since we have MULTI_GLAZE_PROPORTION
 ]

-# For these fields, we take an average if we have multiple values
-AVERAGE_FIXED_FEATURES = ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT"]
-
 # For these fields, we take the latest value if we have multiple values
 # Since more recent EPCs have been conducted with more rigour, we assume that the latest value is
 # the most accurate
--- a/model_data/simulation_system/generate_rdsap_change.py
+++ b/model_data/simulation_system/generate_rdsap_change.py
@ -1,16 +1,15 @@
-import numpy as np
 import pandas as pd
 from tqdm import tqdm

 from pathlib import Path
 from simulation_system.core.Settings import (
    MANDATORY_FIXED_FEATURES,
-    AVERAGE_FIXED_FEATURES,
    LATEST_FIELD,
    COMPONENT_FEATURES,
    RDSAP_RESPONSE,
    HEAT_DEMAND_RESPONSE,
    COLUMNS_TO_MERGE_ON,
+    EARLIEST_EPC_DATE
 )
 from simulation_system.core.DataProcessor import DataProcessor
 from utils import save_dataframe_to_s3_parquet
@ -18,9 +17,6 @@ from utils import save_dataframe_to_s3_parquet
 DATA_DIRECTORY = Path(__file__).parent / "simulation_system" / "data" / "all-domestic-certificates"


-# TODO: Have a look at temporal features
-
-
 def app():
    # Get all the files in the directory

@ -32,10 +28,23 @@ def app():

    dataset = []
    cleaning_dataset = []
-    # 116
-    # 128048706
-    # PosixPath('/home/ubuntu/Documents/python/hestia/Model/model_data/simulation_system/data/all-domestic
-    # -certificates/domestic-E09000021-Kingston-upon-Thames')
+
+    # TODO [x] : Does energy tariff make a difference
+    #           - leave for now but it may not
+    # TODO: [x] : Add starting SAP and head demand as a feature
+    # TODO [x] : If SAP hasn't changed, we don't include the record
+    # TODO [x]: If SAP gets worse, it genuinely looks like in the vast majority of cases that the building looks
+    #           worse in the newer epc, so we can switch the orders
+    # TODO [x] : Have a look at temporal features
+    # TODO [x] : Floor area will impact the EPC so instead of averaging, we should have a starting and ending value.
+    # TODO [x]: Same as floor area for floor height
+    # TODO []: If fundamental building fabric changes, we should proabably discard the record
+    # TODO [x]: Should we prune records that have an exceptionally large amount of time between them?
+    #           - leave for now and check performance after temporal features
+    # TODO [x]: If we have multiple EPCs lodged on the same day, should we remove them? Could be corrections?
+    #           - Leave for now
+    #
+
    for directory in tqdm(directories):

        filepath = directory / "certificates.csv"
@ -45,6 +54,7 @@ def app():
        df = data_processor.pre_process()
        cleaning_averages = data_processor.make_cleaning_averages()

+        data_by_urpn = []
        for uprn, property_data in df.groupby("UPRN", observed=True):

            # Fixed features - these are property attributes that shouldn't change over time
@ -60,9 +70,6 @@ def app():
                property_data[MANDATORY_FIXED_FEATURES].iloc[-1].to_dict()
            )

-            # Taking just the last row, which is the percentage change from the latest to previous one only
-            # property_data[AVERAGE_FIXED_FEATURES].fillna(value=0).pct_change().iloc[-1] > 0.1
-
            # Extract the columns that are not all None
            modified_property_data = DataProcessor.apply_averages_cleaning(
                data_to_clean=property_data,
@ -70,18 +77,6 @@ def app():
                cols_to_merge_on=COLUMNS_TO_MERGE_ON
            )

-            for field in AVERAGE_FIXED_FEATURES:
-
-                vals = list(modified_property_data[field].dropna().unique())
-                if len(vals) > 1:
-                    # Check the values are too far apart
-                    # TODO: we could have multiple values here, why only use the first two?
-                    if abs(vals[0] - vals[1]) / vals[0] > 0.1:
-                        # Take the more recent value since it's likely to be more accurate
-                        vals = [vals[-1]]
-
-                fixed_data[field] = np.mean(vals)
-
            # Combine all fields together
            fixed_data.update(mandatory_field_data)
            fixed_data.update(latest_field_data)
@ -89,8 +84,7 @@ def app():
            # We include the lodgement date here as we probably need to factor time into the
            # model, since EPC standards and rigour have changed over time
            variable_data = modified_property_data[
-                COMPONENT_FEATURES
-                + ["LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE]
+                COMPONENT_FEATURES + ["LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE]
                ]

            # Note: we look at changes between subsequent EPCS, however we could look at other permutations
@ -101,26 +95,32 @@ def app():
                if idx >= modified_property_data.shape[0] - 1:
                    break

-                starting_record = variable_data.iloc[idx]
-                ending_record = variable_data.iloc[idx + 1]
-                rdsap_change = (
-                    ending_record[RDSAP_RESPONSE] - starting_record[RDSAP_RESPONSE]
-                )
-                heat_demand_change = (
-                    ending_record[HEAT_DEMAND_RESPONSE]
-                    - starting_record[HEAT_DEMAND_RESPONSE]
-                )
+                earliest_record = variable_data.iloc[idx]
+                latest_record = variable_data.iloc[idx + 1]

-                # TODO: We need to pre-process the data. For instance, rather than using static for roofs, walls and
-                #       floors, we may want to use the U-value. We may also want to handle the (assumed) tags
-                #       within descriptions
+                # Check if the sap gets better or worse
+                gets_better = earliest_record[RDSAP_RESPONSE] <= latest_record[RDSAP_RESPONSE]

-                starting_record = starting_record[
-                    COMPONENT_FEATURES + ["LODGEMENT_DATE"]
-                    ].add_suffix("_STARTING")
-                ending_record = ending_record[
-                    COMPONENT_FEATURES + ["LODGEMENT_DATE"]
-                    ].add_suffix("_ENDING")
+                if gets_better:
+                    starting_sap = earliest_record[RDSAP_RESPONSE]
+                    starting_heat_demand = earliest_record[HEAT_DEMAND_RESPONSE]
+                    rdsap_change = latest_record[RDSAP_RESPONSE] - starting_sap
+                    heat_demand_change = latest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand
+                else:
+                    starting_sap = latest_record[RDSAP_RESPONSE]
+                    starting_heat_demand = latest_record[HEAT_DEMAND_RESPONSE]
+                    rdsap_change = earliest_record[RDSAP_RESPONSE] - starting_sap
+                    heat_demand_change = earliest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand
+
+                if rdsap_change == 0:
+                    continue
+
+                if gets_better:
+                    starting_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
+                    ending_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING")
+                else:
+                    starting_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
+                    ending_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING")

                features = pd.concat([starting_record, ending_record])

@ -129,12 +129,30 @@ def app():
                        "UPRN": uprn,
                        "RDSAP_CHANGE": rdsap_change,
                        "HEAT_DEMAND_CHANGE": heat_demand_change,
+                        "STARTING_SAP": starting_sap,
+                        "STARTING_HEAT_DEMAND": starting_heat_demand,
                        **fixed_data,
                        **features.to_dict(),
                    }
                )

-            dataset.append(property_model_data)
+            data_by_urpn.extend(property_model_data)
+
+        data_by_urpn_df = pd.DataFrame(data_by_urpn)
+        # Add some temporal features - we look at the days from the standard starting point in time
+        # for the starting and ending date so all records are from a fixed point
+        data_by_urpn_df["DAYS_TO_STARTING"] = (
+            pd.to_datetime(data_by_urpn_df["LODGEMENT_DATE_STARTING"]) - pd.to_datetime(EARLIEST_EPC_DATE)
+        ).dt.days
+        data_by_urpn_df["DAYS_TO_ENDING"] = (
+            pd.to_datetime(data_by_urpn_df["LODGEMENT_DATE_ENDING"]) - pd.to_datetime(EARLIEST_EPC_DATE)
+        ).dt.days
+
+        # TODO: We need to pre-process the data. For instance, rather than using static for roofs, walls and
+        #       floors, we may want to use the U-value. We may also want to handle the (assumed) tags
+        #       within descriptions
+
+        dataset.append(data_by_urpn_df)

        cleaning_averages["LOCAL_AUTHORITY"] = df["LOCAL_AUTHORITY"].values[0]
        cleaning_dataset.append(cleaning_averages)
@ -147,8 +165,12 @@ def app():
        file_key="sap_change_model/cleaning_dataset.parquet",
    )

-    output = pd.DataFrame(dataset)
-    output.to_parquet("./dataset.parquet")
+    output = pd.concat(dataset)
+    save_dataframe_to_s3_parquet(
+        df=output,
+        bucket_name="retrofit-data-dev",
+        file_key="sap_change_model/dataset.parquet",
+    )


 if __name__ == "__main__":
--- a/model_data/tests/test_data/test_floor_attributes_cases.py
+++ b/model_data/tests/test_data/test_floor_attributes_cases.py
@ -341,5 +341,29 @@ clean_floor_cases = [
     "another_property_below": False},
    {'original_description': 'To unheated space, no insulation (assumed)', 'thermal_transmittance': None,
     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False,
-     'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False}
+     'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False},
+    {'original_description': '(eiddo arall islaw)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None,
+     'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False,
+     'is_solid': False, 'insulation_thickness': None,
+     "another_property_below": True},
+    {'original_description': 'Solet, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': False, 'is_to_external_air': False,
+     'is_suspended': False, 'is_solid': True, 'insulation_thickness': 'below average', "another_property_below": False},
+    {'original_description': 'Crog, wediGÇÖi inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': False, 'is_to_external_air': False,
+     'is_suspended': True, 'is_solid': False, 'insulation_thickness': 'average', "another_property_below": False},
+    {'original_description': 'IGÇÖr awyr y tu allan, dim inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': False, 'is_to_external_air': True,
+     'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False},
+    {'original_description': 'I ofod heb ei wresogi, wediGÇÖi inswleiddio (rhagdybiaeth)',
+     'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False,
+     'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'average', "another_property_below": False},
+    {'original_description': 'Solet, wediGÇÖi inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': False, 'is_to_external_air': False,
+     'is_suspended': False, 'is_solid': True, 'insulation_thickness': 'average', "another_property_below": False},
+    {'original_description': 'I ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False,
+     'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False},
 ]
--- a/model_data/tests/test_data/test_hot_water_attributes_cases.py
+++ b/model_data/tests/test_data/test_hot_water_attributes_cases.py
@ -130,8 +130,93 @@ hotwater_cases = [
     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
     'assumed': False, "appliance": None},
-    {'original_description': 'Oil boiler/circulator', 'heater_type': None, 'system_type': 'oil boiler',
+    {'original_description': 'Oil boiler/circulator', 'heater_type': 'oil boiler', 'system_type': None,
     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
     'extra_features': None, 'chp_systems': None, 'distribution_system': 'circulator', 'no_system_present': None,
-     'assumed': False, "appliance": None}
+     'assumed': False, "appliance": None},
+    {'original_description': 'Solid fuel range cooker', 'heater_type': 'solid fuel range cooker', 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': 'OGÇÖr brif system, dim thermostat ar y silindr', 'heater_type': None,
+     'system_type': 'from main system', 'thermostat_characteristics': 'no cylinder thermostat', 'heating_scope': None,
+     'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None,
+     'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'Twymwr tanddwr, an-frig', 'heater_type': 'electric immersion', 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': 'off-peak',
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': 'OGÇÖr brif system, gydag ynniGÇÖr haul', 'heater_type': None,
+     'system_type': 'from main system',
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': 'plus solar', 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': 'Twymwr tanddwr, tarriff safonol', 'heater_type': 'electric immersion',
+     'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': 'standard tariff', 'extra_features': None, 'chp_systems': None, 'distribution_system': None,
+     'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'Trydan ar unwaith yn y fan lle maeGÇÖn cael ei ddefnyddio',
+     'heater_type': 'electric instantaneous',
+     'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None,
+     'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'O gynllun cymunedol', 'heater_type': None, 'system_type': 'community scheme',
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': "O'r brif system", 'heater_type': None, 'system_type': 'from main system',
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': "Trydan ar unwaith yn y fan lle mae'n cael ei ddefnyddio",
+     'heater_type': 'electric instantaneous',
+     'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None,
+     'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'Oil range cooker', 'heater_type': None, 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": "oil range cooker"},
+    {'original_description': 'Oil range cooker, no cylinder thermostat', 'heater_type': None, 'system_type': None,
+     'thermostat_characteristics': 'no cylinder thermostat', 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": "oil range cooker"},
+    {'original_description': 'Popty estynedig olew, dim thermostat ar y silindr', 'heater_type': None,
+     'system_type': None,
+     'thermostat_characteristics': 'no cylinder thermostat', 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": "oil range cooker"},
+    {'original_description': 'Cynllun cymunedol', 'heater_type': None, 'system_type': 'community scheme',
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': 'Nwy wrth fwy nag un pwynt', 'heater_type': 'gas multipoint', 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": None},
+    {'original_description': 'Oil range cooker', 'heater_type': None, 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": "oil range cooker"},
+    {'original_description': 'Popty estynedig olew', 'heater_type': None, 'system_type': None,
+     'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None,
+     'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+     'assumed': False, "appliance": "oil range cooker"},
+    {'original_description': 'Dim system ar gael: rhagdybir bod twymwr tanddwr trydan',
+     'heater_type': 'electric immersion',
+     'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+     'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None,
+     'no_system_present': 'no system present', 'assumed': True, "appliance": None},
+    {'original_description': "O'r brif system, dim thermostat ar y silindr", 'heater_type': None,
+     'system_type': 'from main system', 'thermostat_characteristics': 'no cylinder thermostat', 'heating_scope': None,
+     'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None,
+     'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'Trydan ar unwaith yn y fan lle maeGÇÖn cael ei ddefnyddio, adfer gwres d+¦r gwastraff',
+     'heater_type': 'electric instantaneous', 'system_type': None, 'thermostat_characteristics': None,
+     'heating_scope': None, 'energy_recovery': 'waste water heat recovery', 'tariff_type': None, 'extra_features': None,
+     'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None},
 ]
--- a/model_data/tests/test_data/test_lighting_attributes_cases.py
+++ b/model_data/tests/test_data/test_lighting_attributes_cases.py
@ -30,5 +30,9 @@ test_cases = [
    {'original_description': 'Excellent lighting efficiency', 'low_energy_proportion': 1.0},
    {'original_description': 'Low energy lighting in 2% of fixed outlets', 'low_energy_proportion': 0.02},
    {'original_description': 'No Low energy lighting', 'low_energy_proportion': 0},
-    {'original_description': 'Goleuadau ynni-isel mewn 60% oGÇÖr mannau gosod', 'low_energy_proportion': 0.6}
+    {'original_description': 'Goleuadau ynni-isel mewn 60% oGÇÖr mannau gosod', 'low_energy_proportion': 0.6},
+    {'original_description': 'Goleuadau ynni-isel ym mhob un oGÇÖr mannau gosod', 'low_energy_proportion': 1},
+    {'original_description': 'Dim goleuadau ynni-isel', 'low_energy_proportion': 0},
+    {'original_description': 'Excellent lighting efficiency', 'low_energy_proportion': 1},
+    {'original_description': "Goleuadau ynni-isel ym mhob un o'r mannau gosod", 'low_energy_proportion': 1},
 ]
--- a/model_data/tests/test_data/test_main_fuel_attributes_cases.py
+++ b/model_data/tests/test_data/test_main_fuel_attributes_cases.py
@ -60,5 +60,27 @@ mainfuel_cases = [
    {'original_description': 'wood chips', 'fuel_type': 'wood chips', 'tariff_type': None, 'is_community': False,
     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
    {'original_description': 'wood pellets', 'fuel_type': 'wood pellets', 'tariff_type': None, 'is_community': False,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {'original_description': 'Solid fuel: dual fuel appliance (mineral and wood)',
+     'fuel_type': 'dual fuel appliance mineral and wood',
+     'tariff_type': None, 'is_community': False,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {'original_description': 'coal (community)',
+     'fuel_type': 'coal',
+     'tariff_type': None, 'is_community': True,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {'original_description': 'B30D (community)',
+     'fuel_type': 'b30d',
+     'tariff_type': None, 'is_community': True,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {'original_description': 'Solid fuel: manufactured smokeless fuel', 'fuel_type': 'manufactured smokeless fuel',
+     'tariff_type': None,
+     'is_community': False,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {'original_description': "LNG", 'fuel_type': 'lng', 'tariff_type': None, 'is_community': False,
+     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None},
+    {"original_description": "Community heating schemes: heat from electric heat pump",
+     'fuel_type': 'electric heat pump', 'tariff_type': None, 'is_community': True,
     'no_individual_heating_or_community_network': False, 'complex_fuel_type': None}
+
 ]
--- a/model_data/tests/test_data/test_mainheat_attributes_cases.py
+++ b/model_data/tests/test_data/test_mainheat_attributes_cases.py
--- a/model_data/tests/test_data/test_mainheat_control_attributes_cases.py
+++ b/model_data/tests/test_data/test_mainheat_control_attributes_cases.py
@ -58,7 +58,7 @@ mainheat_control_cases = [
     'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
     'auxiliary_systems': None, 'trvs': None},
    {'original_description': 'Flat rate charging, programmer, no room thermostat',
-     'thermostatic_control': 'room thermostat', 'charging_system': 'flat rate charging', 'switch_system': 'programmer',
+     'thermostatic_control': None, 'charging_system': 'flat rate charging', 'switch_system': 'programmer',
     'no_control': 'no room thermostat', 'dhw_control': None, 'community_heating': None,
     'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
    {'original_description': 'Flat rate charging, room thermostat only', 'thermostatic_control': 'room thermostat',
@ -102,7 +102,7 @@ mainheat_control_cases = [
    {'original_description': 'Programmer, TRVs and flow switch', 'thermostatic_control': None, 'charging_system': None,
     'switch_system': 'programmer', 'no_control': None, 'dhw_control': None, 'community_heating': None,
     'multiple_room_thermostats': False, 'auxiliary_systems': 'flow switch', 'trvs': 'trvs'},
-    {'original_description': 'Programmer, no room thermostat', 'thermostatic_control': 'room thermostat',
+    {'original_description': 'Programmer, no room thermostat', 'thermostatic_control': None,
     'charging_system': None, 'switch_system': 'programmer', 'no_control': 'no room thermostat', 'dhw_control': None,
     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
    {'original_description': 'Programmer, room thermostat and TRVs', 'thermostatic_control': 'room thermostat',
@ -127,6 +127,90 @@ mainheat_control_cases = [
    {'original_description': 'Celect-type controls', 'thermostatic_control': 'celect-type control',
     'charging_system': None, 'switch_system': None, 'no_control': None,
     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None,
-     'trvs': None}
-
+     'trvs': None},
+    {'original_description': 'Celect controls', 'thermostatic_control': 'celect-type control', 'charging_system': None,
+     'switch_system': None, 'no_control': None,
+     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None,
+     'trvs': None},
+    {'original_description': 'Rhaglennydd, dim thermostat ystafell',
+     'thermostatic_control': None, 'charging_system': None,
+     'switch_system': "programmer", 'no_control': 'no room thermostat',
+     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None,
+     'trvs': None},
+    {'original_description': 'Rhaglennydd a thermostat ystafell', 'thermostatic_control': 'room thermostat',
+     'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'RheoliGÇÖr t+ól +ó llaw', 'thermostatic_control': None,
+     'charging_system': 'manual charge control', 'switch_system': None, 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Rheolaeth amser a rheolaeth parthau tymheredd',
+     'thermostatic_control': 'time and temperature zone control', 'charging_system': None, 'switch_system': None,
+     'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
+     'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Programmer, room thermostat', 'thermostatic_control': 'room thermostat',
+     'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Rhaglennydd a thermostatau ar y cyfarpar', 'thermostatic_control': 'room thermostat',
+     'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Rheolyddion i wresogyddion storio syGÇÖn cadw llawer o wres',
+     'thermostatic_control': None,
+     'charging_system': 'high heat retention storage heaters', 'switch_system': None, 'no_control': None,
+     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None,
+     'trvs': None},
+    {'original_description': 'Flat rate charging, TRVs', 'thermostatic_control': None,
+     'charging_system': 'flat rate charging', 'switch_system': None, 'no_control': None,
+     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
+     'auxiliary_systems': None, 'trvs': 'trvs'},
+    {'original_description': 'Single rate heating, programmer and room thermostat',
+     'thermostatic_control': 'room thermostat', 'charging_system': None,
+     'switch_system': 'programmer',
+     'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
+     'auxiliary_systems': None, 'trvs': None, 'rate_control': 'single rate heating'},
+    {'original_description': 'T+ól un gyfradd, rhaglennydd a thermostat ystafell',
+     'thermostatic_control': 'room thermostat', 'charging_system': None,
+     'switch_system': 'programmer',
+     'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
+     'auxiliary_systems': None, 'trvs': None, 'rate_control': 'single rate heating'},
+    {'original_description': 'Rhaglennydd ac o leiaf ddau thermostat ystafell',
+     'thermostatic_control': 'room thermostats',
+     'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': True, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Thermostat ystafell yn unig', 'thermostatic_control': 'room thermostat',
+     'charging_system': None,
+     'switch_system': None, 'no_control': None, 'dhw_control': None, 'community_heating': None,
+     'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Dim rheolaeth amser na rheolaeth thermostatig ar dymheredd yr ystafell',
+     'thermostatic_control': None,
+     'charging_system': None, 'switch_system': None, 'no_control': 'no time or thermostatic control',
+     'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None,
+     'trvs': None},
+    {'original_description': 'Rheoli gwefr drydanol yn awtomatig', 'thermostatic_control': None,
+     'charging_system': 'automatic charge control', 'switch_system': None, 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': "Rheoli'r t+ól +ó llaw", 'thermostatic_control': None,
+     'charging_system': 'manual charge control', 'switch_system': None, 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'System dalu wediGÇÖi chysylltu +ó defnyddio gwres cymunedol, thermostat ystafell yn unig',
+     'thermostatic_control': 'room thermostat', 'charging_system': 'charging system', 'switch_system': None,
+     'no_control': None, 'dhw_control': None, 'community_heating': 'use of community heating',
+     'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Dim', 'thermostatic_control': None, 'charging_system': None, 'switch_system': None,
+     'no_control': 'none', 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
+     'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Dim rheolaeth thermostatig ar dymheredd yr ystafell', 'thermostatic_control': None,
+     'charging_system': None, 'switch_system': None, 'no_control': 'no thermostatic control', 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Thermostatau ar y cyfarpar', 'thermostatic_control': 'appliance thermostats',
+     'charging_system': None, 'switch_system': None, 'no_control': None, 'dhw_control': None, 'community_heating': None,
+     'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {'original_description': 'Rhaglennydd a thermostatau ystafell', 'thermostatic_control': 'room thermostats',
+     'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None,
+     'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
+    {
+        'original_description': 'System dalu wediGÇÖi chysylltu +ó defnyddio gwres cymunedol, rhaglennydd a '
+                                'thermostat ystafell',
+        'thermostatic_control': 'room thermostat', 'charging_system': 'charging system', 'switch_system': 'programmer',
+        'no_control': None, 'dhw_control': None, 'community_heating': 'use of community heating',
+        'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': None},
 ]
--- a/model_data/tests/test_data/test_roof_attributes_cases.py
+++ b/model_data/tests/test_data/test_roof_attributes_cases.py
@ -349,5 +349,49 @@ clean_roof_test_cases = [
     'thermal_transmittance_unit': None,
     'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
     'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
-     'insulation_thickness': 'none'}
+     'insulation_thickness': 'none'},
+    {'original_description': 'Ar oleddf, dim inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'none'},
+    {'original_description': 'Yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': True,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'below average'},
+    {'original_description': 'Ar oleddf, wediGÇÖi inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'average'},
+    {'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True,
+     'insulation_thickness': None},
+    {'original_description': 'Ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'below average'},
+    {'original_description': 'Ystafell(oedd) to, wediGÇÖi hinswleiddio', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': True, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'average'},
+    {'original_description': "Ar oleddf, wediGÇÖi inswleiddio wrth y trawstiau", 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': True, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'average'},
+    {'original_description': 'Yn wastad, wediGÇÖi inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': True,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'average'},
+    {'original_description': 'Ystafell(oedd) to, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': True, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'below average'},
+    {'original_description': 'Ystafell(oedd) to, nenfwd wediGÇÖi inswleiddio', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': True, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'average'},
+    {'original_description': 'Ystafell(oedd) to, dim inswleiddio (rhagdybiaeth)', 'thermal_transmittance': None,
+     'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': True, 'is_loft': False, 'is_flat': False,
+     'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+     'insulation_thickness': 'none'},
 ]
--- a/model_data/tests/test_data/test_window_attributes_cases.py
+++ b/model_data/tests/test_data/test_window_attributes_cases.py
@ -16,6 +16,8 @@ windows_cases = [
     'glazing_type': 'secondary', 'no_data': False},
    {'original_description': 'Mostly triple glazing', 'has_glazing': True, 'glazing_coverage': 'most',
     'glazing_type': 'triple', 'no_data': False},
+    {'original_description': 'Gwydrau triphlyg mwyaf', 'has_glazing': True, 'glazing_coverage': 'most',
+     'glazing_type': 'triple', 'no_data': False},
    {'original_description': 'Multiple glazing throughout', 'has_glazing': True, 'glazing_coverage': 'full',
     'glazing_type': 'multiple', 'no_data': False},
    {'original_description': 'Partial double glazing', 'has_glazing': True, 'glazing_coverage': 'partial',
@ -26,6 +28,8 @@ windows_cases = [
     'glazing_type': 'secondary', 'no_data': False},
    {'original_description': 'Partial triple glazing', 'has_glazing': True, 'glazing_coverage': 'partial',
     'glazing_type': 'triple', 'no_data': False},
+    {'original_description': 'Gwydrau triphlyg rhannol', 'has_glazing': True, 'glazing_coverage': 'partial',
+     'glazing_type': 'triple', 'no_data': False},
    {'original_description': 'Single glazed', 'has_glazing': True, 'glazing_coverage': 'full', 'glazing_type': 'single',
     'no_data': False},
    {'original_description': 'Some double glazing', 'has_glazing': True, 'glazing_coverage': 'partial',
@ -37,5 +41,23 @@ windows_cases = [
    {'original_description': 'Some triple glazing', 'has_glazing': True, 'glazing_coverage': 'partial',
     'glazing_type': 'triple', 'no_data': False},
    {'original_description': 'Gwydrau dwbl llawn', 'has_glazing': True, 'glazing_coverage': 'full',
-     'glazing_type': 'double', 'no_data': False}
+     'glazing_type': 'double', 'no_data': False},
+    {'original_description': 'Gwydrau dwbl rhannol', 'has_glazing': True, 'glazing_coverage': 'partial',
+     'glazing_type': 'double', 'no_data': False},
+    {'original_description': 'Gwydrau dwbl gan mwyaf', 'has_glazing': True, 'glazing_coverage': 'most',
+     'glazing_type': 'double', 'no_data': False},
+    {'original_description': 'Gwydrau sengl', 'has_glazing': True, 'glazing_coverage': 'full', 'glazing_type': 'single',
+     'no_data': False},
+    {'original_description': 'Ffenestri perfformiad uchel', 'has_glazing': True, 'glazing_coverage': 'full',
+     'glazing_type': 'high performance', 'no_data': False},
+    {'original_description': 'Rhai gwydrau dwbl', 'has_glazing': True, 'glazing_coverage': 'partial',
+     'glazing_type': 'double', 'no_data': False},
+    {'original_description': 'Gwydrau triphlyg llawn', 'has_glazing': True, 'glazing_coverage': 'full',
+     'glazing_type': 'triple', 'no_data': False},
+    {'original_description': 'Gwydrau eilaidd llawn', 'has_glazing': True, 'glazing_coverage': 'full',
+     'glazing_type': 'secondary', 'no_data': False},
+    {'original_description': 'Gwydrau eilaidd mwyaf', 'has_glazing': True, 'glazing_coverage': 'most',
+     'glazing_type': 'secondary', 'no_data': False},
+    {'original_description': 'Gwydrau eilaidd rhannol', 'has_glazing': True, 'glazing_coverage': 'partial',
+     'glazing_type': 'secondary', 'no_data': False},
 ]
--- a/model_data/tests/test_downloader.py
+++ b/model_data/tests/test_downloader.py
@ -17,7 +17,7 @@ class TestDownloader:
    def mock_client(self, mocker):
        # mocker is a wrapper around unittest.mock.Mock, extending with
        # additional features specific to pytest
-        client = mocker.Mock(spec=EpcClient())
+        client = mocker.Mock(spec=EpcClient(auth_token="123"))
        client.domestic.search.return_value = mock_epc_response
        return client

--- a/model_data/tests/test_hotwater_attributes.py
+++ b/model_data/tests/test_hotwater_attributes.py
@ -30,9 +30,13 @@ class TestHotWaterAttributes:
        invalid_descriptions = [
            "invalid description",
            "description with no known hotwater data_types",
-            ""
        ]

        for description in invalid_descriptions:
            with pytest.raises(ValueError):
                HotWaterAttributes(description).process()
+
+    def test_empty_description(self):
+        processed = HotWaterAttributes("").process()
+        for _, x in processed.items():
+            assert x is None
--- a/model_data/tests/test_lighting_attributes.py
+++ b/model_data/tests/test_lighting_attributes.py
@ -5,13 +5,11 @@ from model_data.epc_attributes.LightingAttributes import LightingAttributes

 # An example averages dataset to use in tests. It is a dictionary where the key is a lighting description and the
 # value is the expected proportion.
-averages = pd.DataFrame(
-    [
-        {"lighting-description": "good lighting efficiency", "low-energy-lighting": 0.75},
-        {"lighting-description": "excellent lighting efficiency", "low-energy-lighting": 1.0},
-        {"lighting-description": "below average lighting efficiency", "low-energy-lighting": 0.25}
-    ]
-)
+averages = [
+    {"lighting-description": "good lighting efficiency", "low-energy-lighting": 0.75},
+    {"lighting-description": "excellent lighting efficiency", "low-energy-lighting": 1.0},
+    {"lighting-description": "below average lighting efficiency", "low-energy-lighting": 0.25}
+]


 class TestLightingAttributes:
--- a/model_data/tests/test_mainheat_attributes.py
+++ b/model_data/tests/test_mainheat_attributes.py
@ -27,6 +27,12 @@ class TestMainHeatAttributes:
        expected_result = test_case.copy()
        del expected_result["original_description"]
        result = MainHeatAttributes(test_case['original_description']).process()
+
+        # Some of the expected_result test data was produced before some attributes were added to the code
+        # base so we need to filter out some of the keys. The test is still valid
+        result = {k: v for k, v in result.items() if v}
+        expected_result = {k: v for k, v in expected_result.items() if v}
+
        assert sorted(result.items()) == sorted(expected_result.items())

    def test_invalid_description(self):
--- a/model_data/tests/test_mainheat_controls_attributes.py
+++ b/model_data/tests/test_mainheat_controls_attributes.py
@ -23,6 +23,10 @@ class TestMainHeatControlAttributes:
        expected_result = test_case.copy()
        del expected_result["original_description"]
        result = MainheatControlAttributes(test_case['original_description']).process()
+        # Some of the expected_result test data was produced before some attributes were added to the code
+        # base so we need to filter out some of the keys. The test is still valid
+        result = {k: v for k, v in result.items() if v}
+        expected_result = {k: v for k, v in expected_result.items() if v}
        assert sorted(result.items()) == sorted(expected_result.items())

    def test_invalid_description(self):
@ -53,5 +57,6 @@ class TestMainHeatControlAttributes:
            "community_heating": False,
            "multiple_room_thermostats": False,
            "auxiliary_systems": False,
-            "trvs": False
+            "trvs": False,
+            "rate_control": False
        }
--- a/model_data/utils.py
+++ b/model_data/utils.py
@ -1,4 +1,5 @@
 import boto3
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
 import pandas as pd
 from io import BytesIO
 import re
@ -47,3 +48,45 @@ def save_dataframe_to_s3_parquet(df, bucket_name, file_key):

    # Upload the Parquet file to S3
    client.put_object(Bucket=bucket_name, Key=file_key, Body=parquet_buffer.getvalue())
+
+
+def save_data_to_s3(data, bucket_name, s3_file_name):
+    """
+    Save an object to an S3 bucket
+
+    :param data: The data to save
+    :param bucket_name: The name of the S3 bucket
+    :param s3_file_name: The file name to use for the saved data in S3
+    """
+    # Ensure you have AWS credentials set up - either via environment variables, AWS CLI, or IAM roles
+    try:
+        s3 = boto3.client('s3')
+    except NoCredentialsError:
+        print("Credentials not available.")
+        return
+    except PartialCredentialsError:
+        print("Incomplete credentials provided.")
+        return
+
+    try:
+        s3.put_object(Bucket=bucket_name, Key=s3_file_name, Body=data)
+        print(f'Successfully uploaded data to {bucket_name}/{s3_file_name}')
+    except Exception as e:
+        print(f'Failed to upload data to {bucket_name}/{s3_file_name}: {str(e)}')
+
+
+def read_from_s3(bucket_name, s3_file_name):
+    """
+    Read an object from s3. Decoding of the data is left for outside of this function
+
+    :param bucket_name: The name of the S3 bucket
+    :param s3_file_name: The file name to use for the saved data in S3
+    """
+    # Initialize a session using Amazon S3
+    s3 = boto3.resource('s3')
+
+    # Get the MessagePack data from S3
+    obj = s3.Object(bucket_name, s3_file_name)
+    data = obj.get()['Body'].read()
+
+    return data
--- a/pytest.ini
+++ b/pytest.ini
@ -1,3 +1,4 @@
 [pytest]
+pythonpath = .
 addopts = --cov-report term-missing --cov=model_data --cov=recommendations
 testpaths = model_data/tests recommendations/tests
--- a/recommendations/FloorRecommendations.py
+++ b/recommendations/FloorRecommendations.py
@ -9,66 +9,6 @@ from recommendations.recommendation_utils import (
    get_recommended_part, get_uvalue_estimate
 )

-suspended_floor_insulation_parts = [
-    {
-        # Example product
-        # https://www.insulationsuperstore.co.uk/product/recticel-eurothane-general-purpose-pir-insulation-board-2400
-        # -x-1200-x-100mm.html
-        # All product data_types here:
-        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/recticel/filterby/application/floors.html
-        "type": "suspended_floor_insulation",
-        "description": "Rigid Insulation Foam Boards",
-        "depths": [25, 30, 40, 50, 60, 70, 75, 80, 90, 100, 110, 120, 130, 140, 150],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.04545454545454546,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.022,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.insulationsuperstore.co.uk/product/rockwool-rwa45-acoustic-insulation-slab-100mm-2-88m2-pack.html
-        # All product data_types here:
-        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/rockwool/filterby/application/floors
-        # /material/mineral-wool.html
-        "type": "suspended_floor_insulation",
-        "description": "Mineral Wool Floor Insulation",
-        "depths": [25, 40, 50, 60, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.035,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-]
-
-solid_floor_insulation_parts = [
-    {
-        # Example product
-        # https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation/k103-100mm
-        # All product data_types here:
-        # https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation?brand=7015&p=1
-        # Example screed https://www.screwfix.com/p/mapei-ultraplan-3240-self-levelling-compound-25kg/4959f
-        "type": "solid_floor_insulation",
-        "description": "Rigid Insulation Foam Boards with floor screed",
-        "depths": [25, 50, 70, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.04545454545454546,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.052631578947368425,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-
-]
-
-parts = suspended_floor_insulation_parts + solid_floor_insulation_parts
-

 class FloorRecommendations(Definitions):
    # part L building regulations indicate that any rennovations on an existing property's walls should
@ -101,7 +41,7 @@ class FloorRecommendations(Definitions):
        property_instance: Property,
        uvalue_estimates: List,
        total_floor_area_group_decile: str,
-        materials: List = None,
+        materials: List,
    ):
        self.property = property_instance
        self.uvalue_estimates = uvalue_estimates
@ -112,10 +52,7 @@ class FloorRecommendations(Definitions):
        # Will contains a list of recommended measures
        self.recommendations = []

-        if materials:
-            self.materials = materials
-        else:
-            self.materials = parts
+        self.materials = materials

        self.suspended_floor_insulation_parts = [
            part for part in self.materials if part["type"] == "suspended_floor_insulation"
--- a/recommendations/WallRecommendations.py
+++ b/recommendations/WallRecommendations.py
@ -1,5 +1,6 @@
 import itertools
 import math
+from typing import List

 from datatypes.enums import QuantityUnits
 from backend.Property import Property
@ -9,181 +10,6 @@ from recommendations.recommendation_utils import (
    get_recommended_part, get_uvalue_estimate
 )

-external_wall_insulation_parts = [
-    {
-        # Example product
-        # https://insulationgo.co.uk/100mm-rockwool-external-wall-insulation-dual-density-slabs-a1-non-combustible
-        # -slab-ewi-render-fire/
-        "type": "external_wall_insulation",
-        "description": "Mineral Wool External Wall Insulation",
-        "depths": [30, 50, 70, 80, 90, 100, 150, 200],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.0278,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.036,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.insulationking.co.uk/products/polystyrene-eps70?variant=44156186558759
-        "type": "external_wall_insulation",
-        "description": "Expanded Polystyrene External Wall Insulation",
-        "depths": [25, 50, 100, 125],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02703,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.037,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.insulationshop.co/20mm_kooltherm_k5_external_wall_kingspan.html
-        "type": "external_wall_insulation",
-        "description": "Phenolic Foam External Wall Insulation",
-        "depths": [20, 50, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.043478260869565216,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.023,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-
-    },
-    {
-        "type": "external_wall_insulation",
-        "description": "Polyisocyanurate/Polyurethane Foam External Wall Insulation",
-        "depths": [],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": None,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": None,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.mikewye.co.uk/product/steico-duo-dry/
-        "type": "external_wall_insulation",
-        "description": "Wood Fiber External Wall Insulation",
-        "depths": [40, 60],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.023255813953488375,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.043,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.thermablok.co.uk/site/wp-content/uploads/2022/09/Thermablok-Aerogel-Insulation-Blanket-TDS-AIS
-        # -and-Steel-Related-Details.pdf
-        "type": "external_wall_insulation",
-        "description": "Aerogel External Wall Insulation",
-        "depths": [10, 20, 30, 40, 50, 60, 70],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.06666666666666667,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.015,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        "type": "external_wall_insulation",
-        "description": "Vacuum Insulation Panels External Wall Insulation",
-        "depths": [45, 60],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.16666666666666666,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.006,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    }
-]
-
-internal_wall_insulation_parts = [
-    {
-        # Example product
-        # https://www.insulationshop.co/25mm_polystyrene_insulation_eps_70jablite.html
-        "type": "internal_wall_insulation",
-        "description": "Rigid Insulation Boards Internal Wall Insulation",
-        "depths": [25, 40, 50, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.026315789473684213,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.038,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.rockwool.com/siteassets/rw-uk/downloads/datasheets/flexi.pdf
-        "type": "internal_wall_insulation",
-        "description": "Mineral Wool Internal Wall Insulation",
-        "depths": [140],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.035,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.kingspan.com/gb/en/products/insulation-boards/wall-insulation-boards/kooltherm-k118-insulated
-        # -plasterboard/
-        "type": "internal_wall_insulation",
-        "description": "Insulated Plasterboard Internal Wall Insulation",
-        "depths": [25, 80],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.019,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        "type": "internal_wall_insulation",
-        "description": "Reflective Internal Wall Insulation",
-        "depths": [],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": None,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": None,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-    {
-        # Example product
-        # https://www.insulationsuperstore.co.uk/product/vacutherm-vacupor-nt-b2-vacuum-insulated-panel-1m-x-600mm-x
-        # -30mm.html
-        "type": "internal_wall_insulation",
-        "description": "Vacuum Insulation Panels Wall Insulation",
-        "depths": [20, 30],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.125,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.008,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin"
-    },
-]
-
-wall_parts = external_wall_insulation_parts + internal_wall_insulation_parts
-

 class WallRecommendations(Definitions):
    YEAR_WALLS_BUILT_WITH_INSULATION = 1990
@ -217,7 +43,12 @@ class WallRecommendations(Definitions):
        "solid_brick": 2,
    }

-    def __init__(self, property_instance: Property, uvalue_estimates, total_floor_area_group_decile, materials=None):
+    def __init__(
+        self, property_instance: Property,
+        uvalue_estimates: List,
+        total_floor_area_group_decile: str,
+        materials: List
+    ):
        self.property = property_instance
        self.uvalue_estimates = uvalue_estimates
        self.total_floor_area_group_decile = total_floor_area_group_decile
@ -227,10 +58,7 @@ class WallRecommendations(Definitions):
        # Will contains a list of recommended measures
        self.recommendations = []

-        if materials:
-            self.materials = materials
-        else:
-            self.materials = wall_parts
+        self.materials = materials

    @property
    def ewi_valid(self):
--- a/recommendations/tests/test_floor_recommendations.py
+++ b/recommendations/tests/test_floor_recommendations.py
@ -4,7 +4,6 @@ import os
 from unittest.mock import Mock
 from recommendations.FloorRecommendations import FloorRecommendations

-
 # with open(
 #     os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
 # ) as f:
@ -16,6 +15,67 @@ from recommendations.FloorRecommendations import FloorRecommendations
 #     uvalue_estimates = pickle.load(f)


+suspended_floor_insulation_parts = [
+    {
+        # Example product
+        # https://www.insulationsuperstore.co.uk/product/recticel-eurothane-general-purpose-pir-insulation-board-2400
+        # -x-1200-x-100mm.html
+        # All product data_types here:
+        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/recticel/filterby/application/floors.html
+        "type": "suspended_floor_insulation",
+        "description": "Rigid Insulation Foam Boards",
+        "depths": [25, 30, 40, 50, 60, 70, 75, 80, 90, 100, 110, 120, 130, 140, 150],
+        "depth_unit": "mm",
+        "cost": [25, 30, 40, 50, 60, 70, 75, 80, 90, 100, 110, 120, 130, 140, 150],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.04545454545454546,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.022,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.insulationsuperstore.co.uk/product/rockwool-rwa45-acoustic-insulation-slab-100mm-2-88m2-pack.html
+        # All product data_types here:
+        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/rockwool/filterby/application/floors
+        # /material/mineral-wool.html
+        "type": "suspended_floor_insulation",
+        "description": "Mineral Wool Floor Insulation",
+        "depths": [25, 40, 50, 60, 75, 100],
+        "depth_unit": "mm",
+        "cost": [25, 40, 50, 60, 75, 100],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.02857142857142857,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.035,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+]
+
+solid_floor_insulation_parts = [
+    {
+        # Example product
+        # https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation/k103-100mm
+        # All product data_types here:
+        # https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation?brand=7015&p=1
+        # Example screed https://www.screwfix.com/p/mapei-ultraplan-3240-self-levelling-compound-25kg/4959f
+        "type": "solid_floor_insulation",
+        "description": "Rigid Insulation Foam Boards with floor screed",
+        "depths": [25, 50, 70, 75, 100],
+        "depth_unit": "mm",
+        "cost": [25, 40, 50, 60, 75, 100],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.04545454545454546,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.052631578947368425,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+
+]
+
+parts = suspended_floor_insulation_parts + solid_floor_insulation_parts
+
+
 class TestWallRecommendations:

    @pytest.fixture
@ -48,7 +108,8 @@ class TestWallRecommendations:
        obj = FloorRecommendations(
            property_instance=input_properties[0],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        assert obj
        assert obj.property
@ -59,7 +120,8 @@ class TestWallRecommendations:
        recommender = FloorRecommendations(
            property_instance=input_properties[0],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        recommender.recommend()
        assert recommender.property.floor["another_property_below"]
@ -71,10 +133,14 @@ class TestWallRecommendations:
        For a suspended floor without insulation, we use the rdsap methogology to estimate a U-value for the floor
        :return:
        """
+
+        input_properties[2].floor_area = 50
+
        recommender = FloorRecommendations(
            property_instance=input_properties[2],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        assert recommender.estimated_u_value is None
        recommender.recommend()
@ -95,7 +161,8 @@ class TestWallRecommendations:
        recommender = FloorRecommendations(
            property_instance=input_properties[3],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        assert recommender.estimated_u_value is None
        recommender.recommend()
@ -108,10 +175,14 @@ class TestWallRecommendations:
        """
        :return:
        """
+
+        input_properties[4].floor_area = 100
+
        recommender = FloorRecommendations(
            property_instance=input_properties[4],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        assert recommender.estimated_u_value is None
        recommender.recommend()
@ -132,7 +203,8 @@ class TestWallRecommendations:
        recommender = FloorRecommendations(
            property_instance=input_properties[6],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=parts
        )
        assert recommender.estimated_u_value is None
        recommender.recommend()
--- a/recommendations/tests/test_recommendation_utils.py
+++ b/recommendations/tests/test_recommendation_utils.py
@ -1,6 +1,7 @@
 import pytest
 from unittest.mock import MagicMock
 from recommendations import recommendation_utils
+from datatypes.enums import QuantityUnits


 class TestRecommendationUtils:
@ -38,7 +39,9 @@ class TestRecommendationUtils:

    def test_get_recommended_part(self):
        part = {'depths': [1, 2, 3]}
-        assert recommendation_utils.get_recommended_part(part, 1) == {'depths': [1]}
+        assert recommendation_utils.get_recommended_part(
+            part=part, selected_depth=1, selected_total_cost=50, quantity=99, quantity_unit="m2"
+        ) == {'depths': [1], 'estimated_cost': 50, 'quantity': 99, 'quantity_unit': QuantityUnits.m2.value}

    def test_get_uvalue_estimate(self, property_mock):
        uvalue_estimates = [
--- a/recommendations/tests/test_wall_recommendations.py
+++ b/recommendations/tests/test_wall_recommendations.py
@ -10,6 +10,192 @@ from model_data.analysis.UvalueEstimations import UvalueEstimations
 from backend.Property import Property
 from recommendations.recommendation_utils import is_diminishing_returns

+# with open(
+#     os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
+# ) as f:
+#     input_properties = pickle.load(f)
+#
+# with open(
+#     os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/uvalue_estimates.pkl", "rb"
+# ) as f:
+#     uvalue_estimates = pickle.load(f)
+
+
+external_wall_insulation_parts = [
+    {
+        # Example product
+        # https://insulationgo.co.uk/100mm-rockwool-external-wall-insulation-dual-density-slabs-a1-non-combustible
+        # -slab-ewi-render-fire/
+        "type": "external_wall_insulation",
+        "description": "Mineral Wool External Wall Insulation",
+        "depths": [30, 50, 70, 80, 90, 100, 150, 200],
+        "depth_unit": "mm",
+        "cost": [30, 50, 70, 80, 90, 100, 150, 200],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.0278,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.036,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.insulationking.co.uk/products/polystyrene-eps70?variant=44156186558759
+        "type": "external_wall_insulation",
+        "description": "Expanded Polystyrene External Wall Insulation",
+        "depths": [25, 50, 100, 125],
+        "depth_unit": "mm",
+        "cost": [25, 50, 100, 125],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.02703,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.037,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.insulationshop.co/20mm_kooltherm_k5_external_wall_kingspan.html
+        "type": "external_wall_insulation",
+        "description": "Phenolic Foam External Wall Insulation",
+        "depths": [20, 50, 100],
+        "depth_unit": "mm",
+        "cost": [20, 50, 100],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.043478260869565216,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.023,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+
+    },
+    {
+        "type": "external_wall_insulation",
+        "description": "Polyisocyanurate/Polyurethane Foam External Wall Insulation",
+        "depths": [],
+        "depth_unit": "mm",
+        "cost": [],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": None,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": None,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.mikewye.co.uk/product/steico-duo-dry/
+        "type": "external_wall_insulation",
+        "description": "Wood Fiber External Wall Insulation",
+        "depths": [40, 60],
+        "depth_unit": "mm",
+        "cost": [40, 60],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.023255813953488375,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.043,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.thermablok.co.uk/site/wp-content/uploads/2022/09/Thermablok-Aerogel-Insulation-Blanket-TDS-AIS
+        # -and-Steel-Related-Details.pdf
+        "type": "external_wall_insulation",
+        "description": "Aerogel External Wall Insulation",
+        "depths": [10, 20, 30, 40, 50, 60, 70],
+        "depth_unit": "mm",
+        "cost": [10, 20, 30, 40, 50, 60, 70],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.06666666666666667,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.015,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        "type": "external_wall_insulation",
+        "description": "Vacuum Insulation Panels External Wall Insulation",
+        "depths": [45, 60],
+        "depth_unit": "mm",
+        "cost": [45, 60],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.16666666666666666,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.006,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    }
+]
+
+internal_wall_insulation_parts = [
+    {
+        # Example product
+        # https://www.insulationshop.co/25mm_polystyrene_insulation_eps_70jablite.html
+        "type": "internal_wall_insulation",
+        "description": "Rigid Insulation Boards Internal Wall Insulation",
+        "depths": [25, 40, 50, 75, 100],
+        "depth_unit": "mm",
+        "cost": [25, 40, 50, 75, 100],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.026315789473684213,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.038,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.rockwool.com/siteassets/rw-uk/downloads/datasheets/flexi.pdf
+        "type": "internal_wall_insulation",
+        "description": "Mineral Wool Internal Wall Insulation",
+        "depths": [140],
+        "depth_unit": "mm",
+        "cost": [140],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.02857142857142857,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.035,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.kingspan.com/gb/en/products/insulation-boards/wall-insulation-boards/kooltherm-k118-insulated
+        # -plasterboard/
+        "type": "internal_wall_insulation",
+        "description": "Insulated Plasterboard Internal Wall Insulation",
+        "depths": [25, 80],
+        "depth_unit": "mm",
+        "cost": [25, 80],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.02857142857142857,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.019,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        "type": "internal_wall_insulation",
+        "description": "Reflective Internal Wall Insulation",
+        "depths": [],
+        "depth_unit": "mm",
+        "cost": [],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": None,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": None,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+    {
+        # Example product
+        # https://www.insulationsuperstore.co.uk/product/vacutherm-vacupor-nt-b2-vacuum-insulated-panel-1m-x-600mm-x
+        # -30mm.html
+        "type": "internal_wall_insulation",
+        "description": "Vacuum Insulation Panels Wall Insulation",
+        "depths": [20, 30],
+        "depth_unit": "mm",
+        "cost": [20, 30],
+        "cost_unit": "gbp_sq_meter",
+        "r_value_per_mm": 0.125,
+        "r_value_unit": "square_meter_kelvin_per_watt",
+        "thermal_conductivity": 0.008,
+        "thermal_conductivity_unit": "watt_per_meter_kelvin"
+    },
+]
+
+wall_parts = external_wall_insulation_parts + internal_wall_insulation_parts
+

 class TestWallRecommendations:

@ -36,14 +222,17 @@ class TestWallRecommendations:

        uvalue_estimates_mock = Mock()

-        mock_wall_rec_instance = WallRecommendations(property_mock, uvalue_estimates_mock, "Decile 1")
+        mock_wall_rec_instance = WallRecommendations(
+            property_mock, uvalue_estimates_mock, "Decile 1", materials=wall_parts
+        )
        return mock_wall_rec_instance

    def test_init(self, input_properties, uvalue_estimates):
        obj = WallRecommendations(
            property_instance=input_properties[0],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=wall_parts
        )
        assert obj
        assert obj.property
@ -63,7 +252,8 @@ class TestWallRecommendations:
        recommender = WallRecommendations(
            property_instance=input_properties[0],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=wall_parts
        )
        assert recommender.property.walls["original_description"] == "Average thermal transmittance 0.16 W/m-¦K"
        recommender.recommend()
@ -80,10 +270,13 @@ class TestWallRecommendations:
        This property is not in a conservation area, however it's a flat so we don't recommend external wall insulation
        """
        input_properties[1].year_built = 1930
+        input_properties[1].insulation_wall_area = 100
+
        recommender = WallRecommendations(
            property_instance=input_properties[1],
            uvalue_estimates=uvalue_estimates,
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=wall_parts
        )
        assert recommender.property.walls["original_description"] == "Solid brick, as built, no insulation (assumed)"
        assert not recommender.ewi_valid
@ -110,10 +303,7 @@ class TestWallRecommendations:
        However, we're told this property is solid brik so we assume no cavity.
        We're also told that it has some insulation already

-        We'll need to estimate the u-value for this property since we don't know what the insulation is
-        and how thick it is. We'll estimate the u-value based on similar properties, by matching properties
-        with similar attributes such as the walls energy efficiency rating, the property type, the number of rooms,
-        etc
+        Since the walls are already insulated, we don't recommend further measures

        This property is not in a conservation area, however it's a flat so we don't recommend external wall insulation
        """
@ -122,7 +312,8 @@ class TestWallRecommendations:
        recommender = WallRecommendations(
            property_instance=input_properties[6],
            uvalue_estimates=uvalue_estimates.walls.to_dict("records"),
-            total_floor_area_group_decile="Decile 1"
+            total_floor_area_group_decile="Decile 1",
+            materials=wall_parts
        )

        assert recommender.property.walls["original_description"] == "Solid brick, as built, insulated (assumed)"
@ -133,21 +324,8 @@ class TestWallRecommendations:

        recommender.recommend()

-        # We should result in some recommendations, all of which should be internal wall insulation
-        assert recommender.recommendations
-        assert recommender.estimated_u_value == 0.4115686274509804
-
-        rec_types = {part["type"] for rec in recommender.recommendations for part in rec["parts"]}
-        assert rec_types == {"internal_wall_insulation"}
-
-        # Check the recommendations provide a u value below the minimum
-        assert all(
-            rec["new_u_value"] < WallRecommendations.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE for rec in
-            recommender.recommendations
-        )
-
-        for rec in recommender.recommendations:
-            assert rec["new_u_value"] < 0.3
+        assert not recommender.recommendations
+        assert not recommender.estimated_u_value

    def test_is_diminishing_returns_no_recommendations(self):
        # We have no recommendations but the new u value is below the diminishing returns threshold
@ -238,7 +416,9 @@ class TestWallRecommendationsBase:

    @pytest.fixture
    def wall_recommendations_instance(self, property_mock, uvalue_estimations_mock):
-        wall_recommendations_instance = WallRecommendations(property_mock, uvalue_estimations_mock, "Decile 1")
+        wall_recommendations_instance = WallRecommendations(
+            property_mock, uvalue_estimations_mock, "Decile 1", materials=wall_parts
+        )
        wall_recommendations_instance.uvalue_estimates.walls_decile_data = {
            "decile_labels": MagicMock(),
            "decile_boundaries": MagicMock()
--- a/sapmodel.serverless.yml
+++ b/sapmodel.serverless.yml
@ -58,4 +58,5 @@ functions:
      - http:
          path: /predict
          method: POST
+          async: true  # Enable async for long running tasks
    timeout: 120 # Set max run time to 2 minutes - we shouldn't need this much time so this can be reviewed