junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #258 from Hestia-Homes/main

Browse source 
Huge update - multiple new recommendations, new cost data etl process, new infra for heat and carbon models
This commit is contained in:
KhalimCK 2023-11-27 14:55:17 +00:00 • committed by GitHub
commit b457df4c63
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 4552 additions and 528 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/unit_tests.yml vendored
View file

@ -26,6 +26,12 @@ jobs:
run: |
python -m pip install --upgrade pip
pip install -r model_data/requirements/requirements.txt
- name: Set dev AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- name: Run tests with pytest
run: |
pip install -r model_data/requirements/dev.txt

3
.idea/misc.xml generated
View file

@ -1,5 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />

76
backend/Property.py
View file

@ -4,13 +4,16 @@ import os
import pandas as pd
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import POTENTIAL_COLUMNS, EFFICIENCY_FEATURES
from etl.epc_clean.epc_attributes.all_cleaners import all_cleaner_map
from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet
from epc_api.client import EpcClient
from BaseUtility import Definitions
from recommendations.rdsap_tables import england_wales_age_band_lookup
from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area
from recommendations.recommendation_utils import (
estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area
)
ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev')
EPC_AUTH_TOKEN = os.environ.get('EPC_AUTH_TOKEN')
@ -78,6 +81,8 @@ class Property(Definitions):
self.floor_height = None
self.insulation_wall_area = None
self.floor_area = None
self.pitched_roof_area = None
self.insulation_floor_area = None
if epc_client:
self.epc_client = epc_client
@ -284,10 +289,16 @@ class Property(Definitions):
for description, attribute in cleaned.items():
if self.data[description] in self.DATA_ANOMALY_MATCHES:
template = cleaned[description][0]
fill_dict = dict(zip(template.keys(), [None] * len(template)))
fill_dict.update({
"original_description": self.data[description],
"clean_description": self.data[description],
})
setattr(
self,
self.ATTRIBUTE_MAP[description],
{"original_description": self.data[description], "clean_description": self.data[description]}
fill_dict,
)
continue
@ -324,8 +335,28 @@ class Property(Definitions):
raise ValueError("Property does not contain data")
self.construction_age_band = DataProcessor.clean_construction_age_band(self.data["construction-age-band"])
if self.construction_age_band in self.DATA_ANOMALY_MATCHES:
if self.old_data:
# Take the most recent
max_datetime = max(
[x["lodgement-datetime"] for x in self.old_data if
x["construction-age-band"] not in self.DATA_ANOMALY_MATCHES]
)
most_recent = [x for x in self.old_data if x["lodgement-datetime"] == max_datetime]
self.construction_age_band = DataProcessor.clean_construction_age_band(
most_recent[0]["construction-age-band"]
)
self.age_band = england_wales_age_band_lookup.get(self.construction_age_band)
if (self.data["transaction-type"] == "new dwelling") and (self.age_band is None):
self.age_band = "L"
self.construction_age_band = 'England and Wales: 2012 onwards'
if self.age_band is None:
raise ValueError("age_band is missing")
def set_spatial(self, spatial: pd.DataFrame):
"""
Sets whether the property is in a conservation area given the output of the ConservationAreaClient
@ -565,7 +596,7 @@ class Property(Definitions):
self.number_of_rooms = float(self.data["number-habitable-rooms"])
if self.data["property-type"] == "House":
self.number_of_floors = estimate_floors(self.floor_area, self.number_of_rooms)
self.number_of_floors = 2
elif self.data["property-type"] in ["Flat", "Bungalow"]:
self.number_of_floors = 1
elif self.data["property-type"] == "Maisonette":
@ -582,8 +613,17 @@ class Property(Definitions):
self.floor_area / self.number_of_floors, self.number_of_rooms / self.number_of_floors
)
self.insulation_wall_area = estimate_wall_area(
num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter
self.insulation_wall_area = estimate_external_wall_area(
num_floors=self.number_of_floors,
floor_height=self.floor_height,
perimeter=self.perimeter,
built_form=self.data["built-form"],
)
self.insulation_floor_area = self.floor_area / self.number_of_floors
self.pitched_roof_area = esimtate_pitched_roof_area(
floor_area=self.insulation_floor_area, floor_height=self.floor_height
)
def set_wall_type(self):
@ -596,14 +636,31 @@ class Property(Definitions):
def set_floor_type(self):
"""
This method sets the floor type of the property, which is used for calculating u-values
:return:
Section 5.6 of the BRE indicates that
"to simplify data collection no distinction is made in terms of U-value between an exposed floor (to
outside air below) and a semi-exposed floor (to an enclosed but unheated space below)
and the U-values in Table S12 are used.
Therefore, we treat the exposed floor and suspended floor as the same type of floor, which is used for
calculating u-values
"""
self.floor_type = "suspended" if self.floor["is_suspended"] else "solid"
if self.floor["is_suspended"] | self.floor["another_property_below"]:
self.floor_type = "suspended"
elif self.floor["is_solid"]:
self.floor_type = "solid"
elif self.floor["is_to_unheated_space"] | self.floor["is_to_external_air"]:
self.floor_type = "exposed_floor"
elif self.floor["thermal_transmittance"] is not None:
self.floor_type = "solid"
else:
raise NotImplementedError("Implement this floor type")
@staticmethod
def _extract_component(component_data, component_rename_cols, component_drop_cols, rename_prefix=None):
for k in component_rename_cols:
component_data[f"{rename_prefix}_{k}"] = component_data[k]
component_data[f"{rename_prefix}_{k}"] = component_data.get(k)
component_data = {
k: v for k, v in component_data.items() if k not in component_drop_cols + component_rename_cols
@ -640,7 +697,7 @@ class Property(Definitions):
# We'll need to clean second heating
second_heating = self.data["secondheat-description"]
epc_raw_columns = [
epc_raw_columns = POTENTIAL_COLUMNS + EFFICIENCY_FEATURES + [
'TRANSACTION_TYPE',
'ENERGY_TARIFF',
'PROPERTY_TYPE',
@ -697,6 +754,7 @@ class Property(Definitions):
"TOTAL_FLOOR_AREA": self.floor_area,
**epc_raw_data,
"BUILT_FORM": built_form,
"POSTCODE": self.data["postcode"],
}
return property_data

5
backend/app/db/functions/materials_functions.py
View file

@ -1,4 +1,5 @@
from backend.app.db.models.materials import Material
from backend.app.db.utils import row2dict
from functools import lru_cache
@ -16,4 +17,6 @@ def get_materials(session):
materials = session.query(Material).filter(Material.is_active).all()
return materials if materials else []
materials = materials if materials else []
return [row2dict(material) for material in materials]

2
backend/app/db/functions/portfolio_functions.py
View file

@ -8,6 +8,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int):
aggregates = (
session.query(
func.sum(Recommendation.estimated_cost).label("cost"),
func.sum(Recommendation.total_work_hours).label("total_work_hours"),
# For future usage we will aggregate multiple fields in this step
# func.sum(Recommendation.heat_demand).label("total_heat_demand"),
# func.sum(Recommendation.energy_savings).label("total_energy_savings")
@ -20,6 +21,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int):
aggregates_dict = {
"cost": aggregates.cost or 0,
"total_work_hours": aggregates.total_work_hours or 0,
# "total_heat_demand": aggregates.total_heat_demand or 0,
# "total_energy_savings": aggregates.total_energy_savings or 0
}

62
backend/app/db/functions/recommendations_functions.py
View file

@ -1,10 +1,14 @@
from sqlalchemy import insert
from sqlalchemy import insert, delete
from sqlalchemy.orm import Session
from backend.app.db.models.recommendations import Plan, Recommendation, RecommendationMaterials, PlanRecommendations
from backend.app.db.models.portfolio import PropertyModel, PropertyTargetsModel, PropertyDetailsMeter, \
PropertyDetailsEpcModel
def create_plan(session, plan):
def create_plan(session: Session, plan):
"""
This function will create a record for the plan in the database if it does not exist.
:param session: The database session
:param plan: dictionary of data representing a plan to be created
"""
@ -15,7 +19,7 @@ def create_plan(session, plan):
return new_plan.id
def create_recommendation(session, recommendation):
def create_recommendation(session: Session, recommendation):
"""
This function will create a record for the recommendation in the database if it does not exist.
:param session: The database session
@ -29,7 +33,7 @@ def create_recommendation(session, recommendation):
return new_recommendation.id
def create_recommendation_material(session, recommendation_id, material_id, depth):
def create_recommendation_material(session: Session, recommendation_id, material_id, depth):
"""
This function will create a record for the recommendation_material in the database if it does not exist.
:param session: The databse session
@ -49,9 +53,10 @@ def create_recommendation_material(session, recommendation_id, material_id, dept
return new_recommendation_material.id
def create_plan_recommendations(session, plan_id, recommendation_ids):
def create_plan_recommendations(session: Session, plan_id, recommendation_ids):
"""
This function will create records for the plan_recommendation in the database.
:param session: The database session
:param plan_id: ID of the plan
:param recommendation_ids: list of recommendation IDs
"""
@ -63,18 +68,19 @@ def create_plan_recommendations(session, plan_id, recommendation_ids):
session.execute(insert(PlanRecommendations).values(data))
def upload_recommendations(session, recommendations_to_upload, property_id):
def upload_recommendations(session: Session, recommendations_to_upload, property_id):
# Prepare data for bulk insert for Recommendation
recommendations_data = [
{
"property_id": property_id,
"type": rec["type"],
"description": rec["description"],
"estimated_cost": rec["cost"],
"estimated_cost": rec["total"],
"default": rec["default"],
"starting_u_value": rec.get("starting_u_value"),
"new_u_value": rec.get("new_u_value"),
"sap_points": rec["sap_points"]
"sap_points": rec["sap_points"],
"total_work_hours": rec["labour_hours"],
}
for rec in recommendations_to_upload
]
@ -97,10 +103,10 @@ def upload_recommendations(session, recommendations_to_upload, property_id):
{
"recommendation_id": recommendation_id,
"material_id": part["id"],
"depth": part["depths"][0] if part["depths"] else None,
"depth": int(part["depth"]) if part["depth"] else None,
"quantity": part["quantity"],
"quantity_unit": part["quantity_unit"],
"estimated_cost": part["estimated_cost"],
"estimated_cost": part["total"],
}
for rec, recommendation_id in zip(recommendations_to_upload, uploaded_recommendation_ids)
for part in rec["parts"]
@ -112,3 +118,39 @@ def upload_recommendations(session, recommendations_to_upload, property_id):
session.flush()
return uploaded_recommendation_ids
def clear_portfolio(session: Session, portfolio_id: int):
# Fetch all property IDs associated with the given portfolio
property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all()
property_ids = [p.id for p in property_ids]
# Fetch all recommendation IDs associated with the properties
recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all()
recommendation_ids = [r.id for r in recommendation_ids]
# Delete all entries from RecommendationMaterials for these recommendations
session.execute(
delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
)
# Delete all entries from PlanRecommendations that reference plans in the portfolio
session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_(
session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar()
)))
# Delete all Plans associated with the portfolio
session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id))
# Delete all Recommendations associated with the properties
session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids)))
# Now, delete the PropertyModels and related details
# Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel
session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id))
# session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids)))
session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id))
session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id))
# Commit the changes
session.commit()

35
backend/app/db/models/materials.py
View file

@ -13,6 +13,25 @@ class MaterialType(enum.Enum):
external_wall_insulation = "external_wall_insulation"
internal_wall_insulation = "internal_wall_insulation"
cavity_wall_insulation = "cavity_wall_insulation"
mechanical_ventilation = "mechanical_ventilation"
loft_insulation = "loft_insulation"
exposed_floor_insulation = "exposed_floor_insulation"
flat_roof_insulation = "flat_roof_insulation"
room_roof_insulation = "room_roof_insulation"
iwi_wall_demolition = "iwi_wall_demolition"
iwi_vapour_barrier = "iwi_vapour_barrier"
iwi_redecoration = "iwi_redecoration"
suspended_floor_demolition = "suspended_floor_demolition"
suspended_floor_redecoration = "suspended_floor_redecoration"
suspended_floor_vapour_barrier = "suspended_floor_vapour_barrier"
solid_floor_demolition = "solid_floor_demolition"
solid_floor_preparation = "solid_floor_preparation"
solid_floor_vapour_barrier = "solid_floor_vapour_barrier"
solid_floor_redecoration = "solid_floor_redecoration"
ewi_wall_demolition = "ewi_wall_demolition"
ewi_wall_preparation = "ewi_wall_preparation"
ewi_wall_redecoration = "ewi_wall_redecoration"
class DepthUnit(enum.Enum):
@ -21,6 +40,8 @@ class DepthUnit(enum.Enum):
class CostUnit(enum.Enum):
gbp_sq_meter = "gbp_sq_meter"
gbp_per_unit = "gbp_per_unit"
gbp_per_m2 = "gbp_per_m2"
class RValueUnit(enum.Enum):
@ -35,9 +56,11 @@ class Material(Base):
__tablename__ = 'material'
id = Column(Integer, primary_key=True, autoincrement=True)
type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x]), nullable=False)
type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x], create_constraint=False),
nullable=False)
description = Column(String, nullable=False)
depths = Column(String) # You may want to use a specific JSON type depending on the database
depth = Column(String) # You may want to use a specific JSON type depending on the database
depth_unit = Column(Enum(DepthUnit, values_callable=lambda x: [e.value for e in x]), nullable=False)
cost = Column(String)
cost_unit = Column(Enum(CostUnit, values_callable=lambda x: [e.value for e in x]), nullable=False)
@ -51,3 +74,11 @@ class Material(Base):
link = Column(String)
created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())
is_active = Column(Boolean, nullable=False, default=True)
prime_material_cost = Column(Float)
material_cost = Column(Float)
labour_cost = Column(Float)
labour_hours_per_unit = Column(Float)
plant_cost = Column(Float)
total_cost = Column(Float)
notes = Column(String)

83
backend/app/plan/router.py
View file

@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup
from backend.app.dependencies import validate_token
from backend.app.plan.schemas import PlanTriggerRequest
from backend.app.plan.utils import (
create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id
create_recommendation_scoring_data, get_cleaned, insert_temp_recommendation_id
)
from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3
@ -30,13 +30,15 @@ from backend.Property import Property
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from recommendations.FloorRecommendations import FloorRecommendations
from recommendations.RoofRecommendations import RoofRecommendations
from recommendations.VentilationRecommendations import VentilationRecommendations
from recommendations.FireplaceRecommendations import FireplaceRecommendations
from recommendations.optimiser.CostOptimiser import CostOptimiser
from recommendations.optimiser.GainOptimiser import GainOptimiser
from recommendations.optimiser.optimiser_functions import prepare_input_measures
from recommendations.WallRecommendations import WallRecommendations
from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet
from tqdm import tqdm
logger = setup_logger()
@ -71,7 +73,8 @@ async def trigger_plan(body: PlanTriggerRequest):
input_properties = []
for config in plan_input:
# We validate each record in the file. If the record is NOT valid, we need to handle this accordingly
# TODO: implment validation
# TODO: implment validation. We should also standardise postcode and address in some fashion as
# a postcode of abcdef would be considered different to ABCDEF
# Create a record in db
property_id, is_new = create_property(
session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode']
@ -111,7 +114,6 @@ async def trigger_plan(body: PlanTriggerRequest):
# the same data
logger.info("Reading in materials and cleaned datasets")
materials = get_materials(session)
materials_by_type = filter_materials(materials)
cleaned = get_cleaned()
logger.info("Getting components and epc recommendations")
@ -119,25 +121,18 @@ async def trigger_plan(body: PlanTriggerRequest):
# TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers
# in as a dependency and then the optimisers can take the input measures in as part of the setup() method
# import pickle
# with open("input_properties.pickle", "rb") as f:
# input_properties = pickle.load(f)
recommendations = {}
recommendations_scoring_data = []
for p in input_properties:
property_recommendations = []
# Property recommendations
p.get_components(cleaned)
property_recommendations = []
# Floor recommendations
floor_recommender = FloorRecommendations(
property_instance=p,
materials=materials_by_type["floor"],
)
floor_recommender = FloorRecommendations(property_instance=p, materials=materials)
floor_recommender.recommend()
if floor_recommender.recommendations:
@ -145,15 +140,36 @@ async def trigger_plan(body: PlanTriggerRequest):
# Wall recommendations
wall_recomender = WallRecommendations(
property_instance=p,
materials=materials_by_type["walls"]
)
wall_recomender = WallRecommendations(property_instance=p, materials=materials)
wall_recomender.recommend()
if wall_recomender.recommendations:
property_recommendations.append(wall_recomender.recommendations)
# Roof recommendations
roof_recommender = RoofRecommendations(property_instance=p, materials=materials)
roof_recommender.recommend()
if roof_recommender.recommendations:
property_recommendations.append(roof_recommender.recommendations)
# Ventilation recommendations
ventilation_recomender = VentilationRecommendations(
property_instance=p,
materials=[part for part in materials if part["type"] == "mechanical_ventilation"]
)
ventilation_recomender.recommend()
if ventilation_recomender.recommendation:
property_recommendations.append(ventilation_recomender.recommendation)
# Fireplace sealing recommendations
fireplace_recommender = FireplaceRecommendations(property_instance=p)
fireplace_recommender.recommend()
if fireplace_recommender.recommendation:
property_recommendations.append(fireplace_recommender.recommendation)
# We insert temporary ids into the recommendations which is important for the optimiser later
property_recommendations = insert_temp_recommendation_id(property_recommendations)
@ -196,20 +212,28 @@ async def trigger_plan(body: PlanTriggerRequest):
logger.info("Preparing data for scoring in sap change api")
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
# Perform the same cleaning as in the model
# Perform the same cleaning as in the model - first clean number of room variables though
recommendations_scoring_data = DataProcessor.apply_averages_cleaning(
data_to_clean=recommendations_scoring_data,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'],
colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"],
)
recommendations_scoring_data = DataProcessor.apply_averages_cleaning(
data_to_clean=recommendations_scoring_data,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"],
).drop(columns=["LOCAL_AUTHORITY"])
recommendations_scoring_data = DataProcessor.clean_missings_after_description_process(
recommendations_scoring_data, [
c for c in recommendations_scoring_data.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
recommendations_scoring_data,
ignore_cols=[c for c in recommendations_scoring_data.columns if ("thermal_transmittance" in c) or (
"insulation_thickness" in c) or ("ENERGY_EFF" in c)]
)
recommendations_scoring_data = DataProcessor.clean_efficiency_variables(recommendations_scoring_data)
sap_change_model_api = SAPChangeModelAPI(portfolio_id=body.portfolio_id, timestamp=created_at)
file_location = sap_change_model_api.upload_scoring_data(
df=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET
@ -287,19 +311,24 @@ async def trigger_plan(body: PlanTriggerRequest):
# 3) the recommendations
logger.info("Uploading recommendations to the database")
for i in tqdm(range(0, len(input_properties), BATCH_SIZE)):
session.commit()
for i in range(0, len(input_properties), BATCH_SIZE):
try:
# Take a slice of the input_properties list to make a batch
batch_properties = input_properties[i:i + BATCH_SIZE]
for p in batch_properties:
# Your existing operations
property_details_epc = p.get_property_details_epc(
portfolio_id=body.portfolio_id, rating_lookup=rating_lookup
)
create_property_details_epc(session, property_details_epc)
# TODO: TEMP
if p.data["uprn"] == "":
print("Get rid of me!")
p.data["uprn"] = 0
property_data = p.get_full_property_data()
update_property_data(
session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data
@ -338,7 +367,7 @@ async def trigger_plan(body: PlanTriggerRequest):
# the portfolion level impact
aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id)
# Commit all changes at once
# Commit final changes
session.commit()
except IntegrityError:
logger.error("Database integrity error occurred", exc_info=True)

176
backend/app/plan/temp_script_for_flight.py
View file

@ -1,176 +0,0 @@
from datetime import datetime
import pandas as pd
from epc_api.client import EpcClient
from fastapi import APIRouter, Depends
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import sessionmaker
from starlette.responses import Response
from backend.app.config import get_settings
from backend.app.db.connection import db_engine
from backend.app.db.functions.materials_functions import get_materials
from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations
from backend.app.db.functions.property_functions import (
create_property, create_property_details_epc, create_property_targets, update_property_data
)
from backend.app.db.functions.recommendations_functions import (
create_plan, create_plan_recommendations, upload_recommendations
)
from backend.app.db.models.portfolio import rating_lookup
from backend.app.dependencies import validate_token
from backend.app.plan.schemas import PlanTriggerRequest
from backend.app.plan.utils import (
create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id
)
from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3
from backend.ml_models.sap_change_model.api import SAPChangeModelAPI
from backend.Property import Property
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from recommendations.FloorRecommendations import FloorRecommendations
from recommendations.optimiser.CostOptimiser import CostOptimiser
from recommendations.optimiser.GainOptimiser import GainOptimiser
from recommendations.optimiser.optimiser_functions import prepare_input_measures
from recommendations.WallRecommendations import WallRecommendations
from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet
logger = setup_logger()
import pickle
with open('local_data.pickle', 'rb') as f:
local_data = pickle.load(f)
with open("property_dimensions.pickle", "rb") as f:
property_dimensions = pickle.load(f)
with open("sap_change_dataset.pickle", "rb") as f:
sap_change_dataset = pickle.load(f)
created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
plan_input = local_data["plan_input"]
uprn_filenames = local_data["uprn_filenames"]
local_property_data = local_data["local_property_data"]
materials = local_data["materials"]
materials_by_type = filter_materials(materials)
cleaned = local_data["cleaned"]
cleaning_data = local_data["cleaning_data"]
# Need to find some proper materials
materials_by_type["walls"] += [
{'id': 4, 'type': 'cavity_wall_insulation', 'description': 'Example Material 1',
'depths': None,
'depth_unit': None, 'cost': 20,
'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.0278, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.036, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': None, 'created_at': None, 'is_active': True},
{'id': 10, 'type': "cavity_wall_insulation", 'description': 'Example Material 2',
'depths': None, 'depth_unit': None, 'cost': 25, 'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.02631579, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': None,
'created_at': None, 'is_active': True}
]
epc_client = EpcClient(auth_token="NO-TOKEN")
input_properties = []
for i, config in enumerate(plan_input):
property_id = local_property_data[i]["id"]
input_properties.append(
Property(
postcode=config['postcode'],
address1=config['address'],
epc_client=epc_client,
id=property_id
)
)
logger.info("Getting EPC, and spatial data")
for i, p in enumerate(input_properties):
p.data = local_property_data[i]["data"]
p.uprn = local_property_data[i]["uprn"]
p.id = local_property_data[i]["id"]
p.full_sap_epc = local_property_data[i]["full_sap_epc"]
p.old_data = local_property_data[i]["old_data"]
p.is_listed = False
p.in_conservation_area = False
p.is_heritage = False
p.set_year_built()
# TODO: TESTING
p.data['number-habitable-rooms'] = 3
recommendations = {}
recommendations_scoring_data = []
for p in input_properties:
property_recommendations = []
# Property recommendations
p.get_components(cleaned)
# Floor recommendations
floor_recommender = FloorRecommendations(
property_instance=p,
materials=materials_by_type["floor"],
)
floor_recommender.recommend()
if floor_recommender.recommendations:
property_recommendations.append(floor_recommender.recommendations)
# Wall recommendations
wall_recomender = WallRecommendations(
property_instance=p,
materials=materials_by_type["walls"]
)
wall_recomender.recommend()
if wall_recomender.recommendations:
property_recommendations.append(wall_recomender.recommendations)
# We insert temporary ids into the recommendations which is important for the optimiser later
property_recommendations = insert_temp_recommendation_id(property_recommendations)
if not property_recommendations:
continue
recommendations[p.id] = property_recommendations
# Finally, we'll prepare data for predicting the impact on SAP
# TODO: We should use the cleaned data from get_components in the data rather than the raw
# values. We should create a method in Property which takes the EPC data and inserts the cleaned
# data
data_processor = DataProcessor(None, newdata=True)
data_processor.insert_data(pd.DataFrame([p.data.copy()]))
data_processor.pre_process()
starting_epc_data = data_processor.get_component_features(suffix="_STARTING")
ending_epc_data = data_processor.get_component_features(suffix="_ENDING")
fixed_data = data_processor.get_fixed_features()
# We update the ending record with the recommended updates and we set lodgement date to today
ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at
for recommendations_by_type in property_recommendations:
for rec in recommendations_by_type:
scoring_dict = create_recommendation_scoring_data(
property=p,
recommendation=rec,
starting_epc_data=starting_epc_data,
ending_epc_data=ending_epc_data,
fixed_data=fixed_data,
)
recommendations_scoring_data.append(scoring_dict)
# cleanup
del data_processor

79
backend/app/plan/utils.py
View file

@ -1,32 +1,13 @@
import pandas as pd
from backend.Property import Property
from collections import defaultdict
from utils.s3 import read_from_s3
from recommendations.config import UPGRADES_MAP
from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value
from backend.app.db.utils import row2dict
from backend.app.config import get_settings
import msgpack
def filter_materials(materials):
materials_by_type = defaultdict(list)
mapping = {
"walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"],
"floor": ["suspended_floor_insulation", "solid_floor_insulation"]
}
materials = [row2dict(material) for material in materials]
for component, types in mapping.items():
materials_by_type[component] = [part for part in materials if part["type"] in types]
return dict(materials_by_type)
def insert_temp_recommendation_id(property_recommendations):
"""
Creates a temporary recommendation id which is needed for
@ -87,7 +68,7 @@ def create_recommendation_scoring_data(
}
# Set staring u-values if we don't have them
if not scoring_dict["walls_thermal_transmittance"]:
if scoring_dict["walls_thermal_transmittance"] is None:
scoring_dict["walls_thermal_transmittance"] = get_wall_u_value(
clean_description=property.walls["clean_description"],
age_band=property.age_band,
@ -95,7 +76,7 @@ def create_recommendation_scoring_data(
is_sandstone_or_limestone=property.walls["is_sandstone_or_limestone"]
)
if not scoring_dict["floor_thermal_transmittance"]:
if scoring_dict["floor_thermal_transmittance"] is None:
scoring_dict["floor_thermal_transmittance"] = get_floor_u_value(
floor_type=property.floor_type,
area=property.floor_area,
@ -105,7 +86,7 @@ def create_recommendation_scoring_data(
age_band=property.age_band,
)
if not scoring_dict["roof_thermal_transmittance"]:
if scoring_dict["roof_thermal_transmittance"] is None:
scoring_dict["roof_thermal_transmittance"] = get_roof_u_value(
insulation_thickness=property.roof["insulation_thickness"],
has_dwelling_above=property.roof["has_dwelling_above"],
@ -130,8 +111,9 @@ def create_recommendation_scoring_data(
# insulation thickness
scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
scoring_dict["walls_insulation_thickness_ENDING"] = "above average"
scoring_dict["WALLS_ENERGY_EFF_ENDING"] = "Good"
else:
if not scoring_dict["walls_thermal_transmittance_ENDING"]:
if scoring_dict["walls_thermal_transmittance_ENDING"] is None:
scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value(
clean_description=property.walls["clean_description"],
age_band=property.age_band,
@ -144,15 +126,15 @@ def create_recommendation_scoring_data(
# Update description to indicate it's insulate
if recommendation["type"] == "floor_insulation":
if len(recommendation["parts"]) > 1:
raise NotImplementedError("Have more than 1 floor insulation part - handle this case")
scoring_dict["floor_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
# We don't really see above average for this in the training data
scoring_dict["floor_insulation_thickness_ENDING"] = "average"
scoring_dict["FLOOR_ENERGY_EFF_ENDING"] = "Good"
else:
if not scoring_dict["floor_thermal_transmittance_ENDING"]:
if scoring_dict["floor_thermal_transmittance_ENDING"] is None:
scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value(
floor_type=property.floor_type,
area=property.floor_area,
@ -165,23 +147,42 @@ def create_recommendation_scoring_data(
if scoring_dict["floor_insulation_thickness_ENDING"] is None:
scoring_dict["floor_insulation_thickness_ENDING"] = "none"
if recommendation["type"] not in ["wall_insulation", "floor_insulation"]:
raise NotImplementedError("Implement me")
if recommendation["type"] == "roof_insulation":
scoring_dict["roof_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
if not scoring_dict["roof_thermal_transmittance_ENDING"]:
scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value(
insulation_thickness=property.roof["insulation_thickness"],
has_dwelling_above=property.roof["has_dwelling_above"],
is_loft=property.roof["is_loft"],
is_roof_room=property.roof["is_roof_room"],
is_thatched=property.roof["is_thatched"],
age_band=property.age_band,
is_flat=property.roof["is_flat"],
is_pitched=property.roof["is_pitched"],
is_at_rafters=property.roof["is_at_rafters"],
)
parts = recommendation["parts"]
if len(parts) != 1:
raise ValueError("More than one part for roof insulation - investiage me")
scoring_dict["roof_insulation_thickness_ENDING"] = str(int(parts[0]["depth"]))
scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good"
else:
# Fill missing roof u-values - this fill is not based on recommended upgrades
if scoring_dict["roof_thermal_transmittance_ENDING"] is None:
scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value(
insulation_thickness=property.roof["insulation_thickness"],
has_dwelling_above=property.roof["has_dwelling_above"],
is_loft=property.roof["is_loft"],
is_roof_room=property.roof["is_roof_room"],
is_thatched=property.roof["is_thatched"],
age_band=property.age_band,
is_flat=property.roof["is_flat"],
is_pitched=property.roof["is_pitched"],
is_at_rafters=property.roof["is_at_rafters"],
)
if scoring_dict["roof_insulation_thickness_ENDING"] is None:
scoring_dict["roof_insulation_thickness_ENDING"] = "none"
if recommendation["type"] == "mechanical_ventilation":
scoring_dict["MECHANICAL_VENTILATION_ENDING"] = 'mechanical, extract only'
if recommendation["type"] == "sealing_open_fireplace":
scoring_dict["NUMBER_OPEN_FIREPLACES_ENDING"] = 0
if recommendation["type"] not in [
"wall_insulation", "floor_insulation", "roof_insulation", "mechanical_ventilation", "sealing_open_fireplace",
]:
raise NotImplementedError("Implement me")
return scoring_dict

4
backend/requirements/dev.txt Normal file
View file

@ -0,0 +1,4 @@
pytest
mock
pytest-cov
pytest-mock

63
backend/tests/test_property.py
View file

@ -13,6 +13,7 @@ mock_epc_response = {
"number-habitable-rooms": 5,
"property-type": "House",
"inspection-date": "2023-06-01",
'lodgement-datetime': '2023-06-01 20:29:01',
"some-other-key": "some-value",
"roof-description": "Roof Description",
"walls-description": "Walls Description",
@ -33,7 +34,8 @@ mock_epc_response = {
"mains-gas-flag": "Y",
"floor-height": 2.5,
"total-floor-area": 100,
"construction-age-band": "England and Wales: 1967-1975"
"construction-age-band": "England and Wales: 1967-1975",
"floor-description": "Floor Description"
},
{
"lmk-key": 2,
@ -41,6 +43,7 @@ mock_epc_response = {
"number-habitable-rooms": 5,
"property-type": "House",
"inspection-date": "2023-05-01",
'lodgement-datetime': '2023-05-01 20:29:01',
"some-other-key": "some-other-value",
"roof-description": "Roof Description",
"walls-description": "Walls Description",
@ -61,7 +64,8 @@ mock_epc_response = {
"mains-gas-flag": "Y",
"floor-height": 2.5,
"total-floor-area": 100,
"construction-age-band": "England and Wales: 1967-1975"
"construction-age-band": "England and Wales: 1967-1975",
"floor-description": "Floor Description"
}
]
}
@ -73,7 +77,9 @@ mock_epc_response_dupe = {
"uprn": 1,
"number-habitable-rooms": 5,
"property-type": "House",
'inspection-date': '2023-06-01', 'some-other-key': 'some-value', 'roof-description': 'Roof Description',
'inspection-date': '2023-06-01',
'lodgement-datetime': '2023-06-01 20:29:01',
'some-other-key': 'some-value', 'roof-description': 'Roof Description',
'walls-description': 'Walls Description', 'windows-description': 'Windows Description',
'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description',
"transaction-type": "rental",
@ -90,14 +96,17 @@ mock_epc_response_dupe = {
"mains-gas-flag": "Y",
"floor-height": 2.5,
"total-floor-area": 100,
"construction-age-band": "England and Wales: 1967-1975"
"construction-age-band": "England and Wales: 1967-1975",
"floor-description": "Floor Description"
},
{
"lmk-key": 2,
"uprn": 2,
"number-habitable-rooms": 5,
"property-type": "House",
'inspection-date': '2023-05-01', 'some-other-key': 'some-other-value',
'inspection-date': '2023-05-01',
'lodgement-datetime': '2023-05-01 20:29:01',
'some-other-key': 'some-other-value',
'roof-description': 'Roof Description', 'walls-description': 'Walls Description',
'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description',
'hotwater-description': 'Hot Water Description',
@ -115,14 +124,17 @@ mock_epc_response_dupe = {
"mains-gas-flag": "Y",
"floor-height": 2.5,
"total-floor-area": 100,
"construction-age-band": "England and Wales: 1967-1975"
"construction-age-band": "England and Wales: 1967-1975",
"floor-description": "Floor Description"
},
{
"lmk-key": 3,
"uprn": 3,
"number-habitable-rooms": 5,
"property-type": "House",
'inspection-date': '2023-06-01', 'some-other-key': 'duplicate-date',
'inspection-date': '2023-06-01',
'lodgement-datetime': '2023-06-01 20:29:01',
'some-other-key': 'duplicate-date',
'roof-description': 'Roof Description',
'walls-description': 'Walls Description', 'windows-description': 'Windows Description',
'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description',
@ -140,7 +152,8 @@ mock_epc_response_dupe = {
"mains-gas-flag": "Y",
"floor-height": 2.5,
"total-floor-area": 100,
"construction-age-band": "England and Wales: 1967-1975"
"construction-age-band": "England and Wales: 1967-1975",
"floor-description": "Floor Description"
}
]
}
@ -187,7 +200,8 @@ class TestProperty:
{"mainheat-description": "Main Heating Description"},
{"hotwater-description": "Hot Water Description"},
{"lighting-description": "Good Lighting Efficiency"},
{"low-energy-lighting": 0}
{"low-energy-lighting": 0},
{"floor-description": "Floor Description"}
],
lighting_averages=lighting_averages
)
@ -212,7 +226,9 @@ class TestProperty:
"windows-description": [{"original_description": "Windows Description"}],
"mainheat-description": [{"original_description": "Main Heating Description"}],
"hotwater-description": [{"original_description": "Hot Water Description"}],
"lighting-description": [{"original_description": "Good Lighting Efficiency"}]
"lighting-description": [{"original_description": "Good Lighting Efficiency"}],
"floor-description": [
{"original_description": "Floor Description", "is_suspended": True, "another_property_below": False}]
}
return mock_cleaner
@ -288,14 +304,35 @@ class TestProperty:
"roof-description": []
}
property_instance.search_address_epc()
property_instance.data["roof-description"] = "Pitched, no insulation"
property_instance.walls = {
"original_description": "Walls Description",
"is_cavity_wall": True,
"is_solid_brick": False,
"is_timber_frame": False,
"is_system_built": False,
"is_park_home": False,
"is_cob": False,
"is_sandstone_or_limestone": False,
"is_granite_or_whinstone": False,
}
# Verify that ValueError is raised when no attributes are found
with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"):
property_instance.get_components(mock_cleaner.cleaned)
property_instance.floor = {
"is_suspended": False,
"another_property_below": False,
"is_solid": True
}
# Assert backup cleaning has been applied
property_instance.get_components(mock_cleaner.cleaned)
assert property_instance.roof["clean_description"] == "Pitched, no insulation"
assert property_instance.roof["is_pitched"]
def test_get_components_multiple_attributes(self, property_instance, mock_cleaner):
# This shouldn't happen - it would mean a cleaning error
property_instance.search_address_epc()
property_instance.data["roof-description"] = "Roof Description"
cleaned = {
"roof-description": [
{"original_description": "Roof Description"},

989
backend/tests/test_sap_model_prep.py
View file

@ -0,0 +1,989 @@
from backend.Property import Property
from etl.epc.DataProcessor import DataProcessor
from backend.app.plan.utils import create_recommendation_scoring_data, get_cleaned
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from epc_api.client import EpcClient
import pandas as pd
import pytest
import msgpack
from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
from tqdm import tqdm
# Handy code for selecting testing data
# import pickle
#
# with open("sap_dataset.pickle", "rb") as f:
# sap_change_dataset = pickle.load(f)
#
# search_from = sap_change_dataset[
# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) &
# sap_change_dataset["is_to_unheated_space"]
# ]
# search_from = search_from[
# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) &
# (search_from["floor_thermal_transmittance_ENDING"] != search_from["floor_thermal_transmittance"]) &
# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) &
# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) &
# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"])
# ]
#
# # Find a record where the only difference is cavity wall getting filled
# ending_cols = [c for c in search_from.columns if "_ENDING" in c]
#
# ignore = [
# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING",
# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING"
# ]
#
# ending_cols = [c for c in ending_cols if c not in ignore]
#
# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]):
#
# same = True
# starting_cols = []
# for c in ending_cols:
#
# starting_col = c.replace("_ENDING", "")
# if starting_col not in search_from.columns:
# starting_col = c.replace("_ENDING", "_STARTING")
# if starting_col not in search_from.columns:
# raise Exception("something went wrong")
#
# starting_cols.append(starting_col)
#
# # We want them to be different
# if c == "floor_thermal_transmittance_ENDING":
# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"):
# same = False
# break
# else:
# continue
#
# # We now check if the starting and ending values are the same
# if row[c] != row[starting_col]:
# same = False
# break
#
# if same:
# raise Exception("We found one!")
#
# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols]
#
# import pandas as pd
#
# start = row[["SAP_STARTING"] + starting_cols]
# start.index = [c.replace("_STARTING", "") for c in start.index]
# end = row[["SAP_ENDING"] + ending_cols]
# end.index = [c.replace("_ENDING", "") for c in end.index]
# start["type"] = "starting"
# end["type"] = "ending"
#
# compare = pd.concat([start, end], axis=1)
#
# ending_lmk = "1252008839062019090910572351658131"
# starting_lmk = "1252008819542014122308482236142128"
#
# client = EpcClient(auth_token=EPC_AUTH_TOKEN)
# result = client.domestic.search(params={"address": "Flat 14 Charles House, Freemens Way", "postcode": "CT14 9DL"})
# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0]
# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0]
# with open(
# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaned.pickle", "rb"
# ) as f:
# cleaned = pickle.load(f)
# with open(
# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaning_data.pickle", "rb"
# ) as f:
# cleaning_data = pickle.load(f)
# TODO: Need to do floors, suspended and solid and to unheated space
class TestSapModelPrep:
@pytest.fixture
def cleaning_data(self):
return read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
@pytest.fixture
def cleaned(self):
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
return cleaned
def test_fill_cavity_wall(self, cleaned, cleaning_data):
"""
We ensure that the process that prepares the data in the engine code results in the same data as
the model is trained on
"""
# This is an actual starting EPC
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '26, Vicarage Lane, Eaton',
'uprn-source': 'Address Matched', 'floor-height': '2.39', 'heating-cost-potential': '942',
'unheated-corridor-length': '', 'hot-water-cost-potential': '97',
'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'D',
'mainheat-energy-eff': 'Average', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Average',
'environment-impact-potential': '53',
'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '1475',
'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs',
'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Melton',
'fixed-lighting-outlets-count': '', 'energy-tariff': 'Single',
'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96', 'county': 'Leicestershire',
'postcode': 'NG32 1SP', 'solar-water-heating-flag': 'Y', 'constituency': 'E14000909',
'co2-emissions-potential': '5.7', 'number-heated-rooms': '7',
'floor-description': 'Suspended, no insulation (assumed)',
'energy-consumption-potential': '177', 'local-authority': 'E07000133', 'built-form': 'Detached',
'number-open-fireplaces': '1', 'windows-description': 'Fully double glazed',
'glazed-area': 'Normal', 'inspection-date': '2016-09-22', 'mains-gas-flag': 'N',
'co2-emiss-curr-per-floor-area': '87', 'address1': '26, Vicarage Lane',
'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '',
'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Very Poor',
'total-floor-area': '116.0', 'building-reference-number': '4940047478',
'environment-impact-current': '29', 'co2-emissions-current': '10.0',
'roof-description': 'Pitched, limited insulation (assumed)', 'floor-energy-eff': 'NO DATA!',
'number-habitable-rooms': '7', 'address2': 'Eaton', 'hot-water-env-eff': 'Good',
'posttown': 'GRANTHAM', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'oil (not community)',
'lighting-env-eff': 'Average', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A',
'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 31% of fixed outlets',
'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Poor', 'photo-supply': '',
'lighting-cost-potential': '69', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
'main-heating-controls': '2106', 'lodgement-datetime': '2016-09-23 20:29:01',
'flat-top-storey': '', 'current-energy-rating': 'F',
'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', 'walls-env-eff': 'Poor',
'transaction-type': 'marketed sale', 'uprn': '100030534042', 'current-energy-efficiency': '34',
'energy-consumption-current': '343', 'mainheat-description': 'Boiler and radiators, oil',
'lighting-cost-current': '117', 'lodgement-date': '2016-09-23', 'extension-count': '2',
'mainheatc-env-eff': 'Good', 'lmk-key': '1481856849902016092320290148762028',
'wind-turbine-count': '0', 'tenure': 'owner-occupied', 'floor-level': 'NODATA!',
'potential-energy-efficiency': '64', 'hot-water-energy-eff': 'Good',
'low-energy-lighting': '31',
'walls-description': 'Cavity wall, as built, no insulation (assumed)',
'hotwater-description': 'From main system, plus solar'
}
# This is the training data as we prepare it in the engine
# This is an actual record from the training data
row = {
'UPRN': '100030534042', 'RDSAP_CHANGE': 12, 'HEAT_DEMAND_CHANGE': -72,
'CARBON_CHANGE': -2.0999999999999996, 'SAP_STARTING': 34, 'SAP_ENDING': 46, 'HEAT_DEMAND_STARTING': 343,
'HEAT_DEMAND_ENDING': 271, 'CARBON_STARTING': 10.0, 'CARBON_ENDING': 7.9, 'PROPERTY_TYPE': 'House',
'BUILT_FORM': 'Detached', 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 7.0,
'NUMBER_HEATED_ROOMS': 7.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 21.0,
'CONSTRUCTION_AGE_BAND': 'England and Wales: 1967-1975', 'TRANSACTION_TYPE_STARTING': 'marketed sale',
'MECHANICAL_VENTILATION_STARTING': 'natural',
'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)',
'ENERGY_TARIFF_STARTING': 'Single', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'Y',
'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_STARTING': 100.0, 'LOW_ENERGY_LIGHTING_STARTING': 31.0,
'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, 'EXTENSION_COUNT_STARTING': 2.0,
'TOTAL_FLOOR_AREA_STARTING': 116.0, 'FLOOR_HEIGHT_STARTING': 2.39,
'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural',
'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)',
'ENERGY_TARIFF_ENDING': 'Single', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'Y', 'PHOTO_SUPPLY_ENDING': 0.0,
'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_ENDING': 100.0, 'LOW_ENERGY_LIGHTING_ENDING': 31.0,
'NUMBER_OPEN_FIREPLACES_ENDING': 1.0, 'EXTENSION_COUNT_ENDING': 2.0, 'TOTAL_FLOOR_AREA_ENDING': 116.0,
'FLOOR_HEIGHT_ENDING': 2.41, 'DAYS_TO_STARTING': 784, 'DAYS_TO_ENDING': 867,
'walls_thermal_transmittance': 1.5, 'is_cavity_wall': True, 'is_filled_cavity': False,
'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False,
'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False,
'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none',
'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.7,
'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average',
'external_insulation_ENDING': False, 'internal_insulation_ENDING': False,
'floor_thermal_transmittance': 0.64, 'is_to_unheated_space': False, 'is_to_external_air': False,
'is_suspended': True, 'is_solid': False, 'another_property_below': False,
'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.64,
'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 1.5, 'is_pitched': True,
'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'has_dwelling_above': False, 'roof_insulation_thickness': 'below average',
'roof_thermal_transmittance_ENDING': 1.5, 'roof_insulation_thickness_ENDING': 'below average',
'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown',
'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown',
'extra_features': 'plus solar', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown',
'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'Unknown',
'system_type_ENDING': 'from main system', 'thermostat_characteristics_ENDING': 'Unknown',
'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown',
'hotwater_tariff_type_ENDING': 'Unknown', 'extra_features_ENDING': 'plus solar',
'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown',
'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': True,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False,
'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': True,
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True,
'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False,
'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False,
'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False,
'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False,
'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False,
'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False,
'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False,
'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False,
'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False,
'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False,
'has_electric_ENDING': False, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False,
'has_coal_ENDING': False, 'has_oil_ENDING': True, 'has_wood_pellets_ENDING': False,
'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False,
'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False,
'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False,
'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat',
'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown',
'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False,
'auxiliary_systems': 'Unknown', 'trvs': 'trvs', 'rate_control': 'Unknown',
'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown',
'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown',
'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False,
'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'trvs', 'rate_control_ENDING': 'Unknown',
'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'oil',
'main-fuel_tariff_type': 'Unknown', 'is_community': False,
'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
'fuel_type_ENDING': 'oil', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False,
'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown',
'estimated_perimeter_STARTING': 44.77882152472145, 'estimated_perimeter_ENDING': 44.77882152472145,
'HOT_WATER_ENERGY_EFF_STARTING': "Good",
"FLOOR_ENERGY_EFF_STARTING": "Unknown",
"WINDOWS_ENERGY_EFF_STARTING": "Good",
"WALLS_ENERGY_EFF_STARTING": "Poor",
"SHEATING_ENERGY_EFF_STARTING": "Unknown",
"ROOF_ENERGY_EFF_STARTING": "Very Poor",
"MAINHEAT_ENERGY_EFF_STARTING": "Average",
"MAINHEATC_ENERGY_EFF_STARTING": "Good",
"LIGHTING_ENERGY_EFF_STARTING": "Average",
"POTENTIAL_ENERGY_EFFICIENCY": 64,
"ENVIRONMENT_IMPACT_POTENTIAL": 53,
"ENERGY_CONSUMPTION_POTENTIAL": 177.0,
"CO2_EMISSIONS_POTENTIAL": 5.7,
"HOT_WATER_ENERGY_EFF_ENDING": "Good",
"FLOOR_ENERGY_EFF_ENDING": "Unknown",
"WINDOWS_ENERGY_EFF_ENDING": "Good",
"WALLS_ENERGY_EFF_ENDING": "Good",
"SHEATING_ENERGY_EFF_ENDING": "Unknown",
"ROOF_ENERGY_EFF_ENDING": "Very Poor",
"MAINHEAT_ENERGY_EFF_ENDING": "Average",
"MAINHEATC_ENERGY_EFF_ENDING": "Good",
"LIGHTING_ENERGY_EFF_ENDING": "Average",
}
home = Property(
id=0,
postcode=starting_epc["postcode"],
address1=starting_epc["address1"],
epc_client=EpcClient(auth_token="notoken"),
data=starting_epc
)
home.get_components(cleaned)
data_processor = DataProcessor(None, newdata=True)
data_processor.insert_data(pd.DataFrame([home.get_model_data()]))
data_processor.pre_process()
starting_epc_data = data_processor.get_component_features(suffix="_STARTING")
ending_epc_data = data_processor.get_component_features(suffix="_ENDING")
fixed_data = data_processor.get_fixed_features()
ending_lodgement_date = '2016-12-15'
ending_epc_data["DAYS_TO_ENDING"] = data_processor.calculate_days_to(ending_lodgement_date)
recommendation = {
"recommendation_id": 0,
"new_u_value": 0.7,
"type": "wall_insulation"
}
test_record = create_recommendation_scoring_data(
property=home,
recommendation=recommendation,
starting_epc_data=starting_epc_data,
ending_epc_data=ending_epc_data,
fixed_data=fixed_data,
)
test_record = pd.DataFrame([test_record])
# Test the final cleaning:
test_record = DataProcessor.apply_averages_cleaning(
data_to_clean=test_record,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
).drop(columns=["LOCAL_AUTHORITY"])
test_record = DataProcessor.clean_missings_after_description_process(
test_record, [
c for c in test_record.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
)
# Test that the data has been set up correctly
# Things to fix:
# [] Filled cavity should have an average insulation thickness in the cleaned data
for c in test_record.columns:
if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]:
continue
if c == "FLOOR_HEIGHT_ENDING":
assert (row[c] - test_record[c].values[0]) <= 0.020001
continue
if c == "walls_insulation_thickness_ENDING":
assert row[c] == "average"
assert test_record[c].values[0] == "above average"
continue
assert test_record[c].values[0] == row[c]
def test_solid_wall_insulation(self, cleaned, cleaning_data):
starting_epc2 = {
'low-energy-fixed-light-count': '2', 'address': 'FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY',
'uprn-source': 'Energy Assessor', 'floor-height': '3.64', 'heating-cost-potential': '465',
'unheated-corridor-length': '', 'hot-water-cost-potential': '185',
'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C',
'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Poor',
'environment-impact-potential': '51', 'glazed-type': 'double glazing installed during or after 2002',
'heating-cost-current': '1223', 'address3': '3 WESTERN GATEWAY',
'mainheatcont-description': 'Programmer and appliance thermostats', 'sheating-energy-eff': 'N/A',
'property-type': 'Flat', 'local-authority-label': 'Newham', 'fixed-lighting-outlets-count': '12',
'energy-tariff': 'off-peak 7 hour', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '342',
'county': '', 'postcode': 'E16 1BD', 'solar-water-heating-flag': 'N', 'constituency': 'E14001032',
'co2-emissions-potential': '3.6', 'number-heated-rooms': '2', 'floor-description': '(other premises below)',
'energy-consumption-potential': '307', 'local-authority': 'E09000025', 'built-form': 'Mid-Terrace',
'number-open-fireplaces': '0', 'windows-description': 'Partial double glazing', 'glazed-area': 'Normal',
'inspection-date': '2020-10-14', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '66',
'address1': 'FLAT 12', 'heat-loss-corridor': 'heated corridor', 'flat-storey-count': '',
'constituency-label': 'West Ham', 'roof-energy-eff': 'N/A', 'total-floor-area': '70.0',
'building-reference-number': '10000539740', 'environment-impact-current': '42',
'co2-emissions-current': '4.6', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '2', 'address2': 'WAREHOUSE W', 'hot-water-env-eff': 'Poor', 'posttown': 'LONDON',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Poor',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 17% of fixed outlets', 'roof-env-eff': 'N/A',
'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '67',
'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '61', 'main-heating-controls': '',
'lodgement-datetime': '2020-10-14 00:00:00', 'flat-top-storey': 'N', 'current-energy-rating': 'F',
'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'marketed sale',
'uprn': '10012839482', 'current-energy-efficiency': '33', 'energy-consumption-current': '393',
'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '110',
'lodgement-date': '2020-10-14', 'extension-count': '0', 'mainheatc-env-eff': 'Good',
'lmk-key': 'b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '01', 'potential-energy-efficiency': '71',
'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '17',
'walls-description': 'Solid brick, as built, no insulation (assumed)',
'hotwater-description': 'Electric immersion, standard tariff'
}
row2 = {
'UPRN': '10012839482', 'RDSAP_CHANGE': 8, 'HEAT_DEMAND_CHANGE': -59,
'CARBON_CHANGE': -0.5999999999999996, 'SAP_STARTING': 33, 'SAP_ENDING': 41, 'HEAT_DEMAND_STARTING': 393,
'HEAT_DEMAND_ENDING': 334, 'CARBON_STARTING': 4.6, 'CARBON_ENDING': 4.0, 'PROPERTY_TYPE': 'Flat',
'BUILT_FORM': 'Mid-Terrace', 'CONSTITUENCY': 'E14001032', 'NUMBER_HABITABLE_ROOMS': 2.0,
'NUMBER_HEATED_ROOMS': 2.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 12.0,
'CONSTRUCTION_AGE_BAND': 'England and Wales: 1996-2002', 'TRANSACTION_TYPE_STARTING': 'marketed sale',
'MECHANICAL_VENTILATION_STARTING': 'natural', 'SECONDHEAT_DESCRIPTION_STARTING': 'None',
'ENERGY_TARIFF_STARTING': 'off-peak 7 hour', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N',
'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_STARTING': 61.0, 'LOW_ENERGY_LIGHTING_STARTING': 17.0,
'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, 'EXTENSION_COUNT_STARTING': 0.0,
'TOTAL_FLOOR_AREA_STARTING': 70.0, 'FLOOR_HEIGHT_STARTING': 3.64,
'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural',
'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'off-peak 7 hour',
'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0,
'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0,
'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0,
'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307,
'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, 'is_filled_cavity': False,
'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False,
'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False,
'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none',
'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.21,
'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average',
'external_insulation_ENDING': False, 'internal_insulation_ENDING': False,
'floor_thermal_transmittance': 0.0, 'is_to_unheated_space': False, 'is_to_external_air': False,
'is_suspended': False, 'is_solid': False, 'another_property_below': True,
'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.0,
'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.0, 'is_pitched': False,
'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'has_dwelling_above': True, 'roof_insulation_thickness': 'none',
'roof_thermal_transmittance_ENDING': 0.0, 'roof_insulation_thickness_ENDING': 'none',
'heater_type': 'electric immersion', 'system_type': 'Unknown', 'thermostat_characteristics': 'Unknown',
'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'standard tariff',
'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown',
'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'electric immersion',
'system_type_ENDING': 'Unknown', 'thermostat_characteristics_ENDING': 'Unknown',
'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown',
'hotwater_tariff_type_ENDING': 'standard tariff', 'extra_features_ENDING': 'Unknown',
'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown',
'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': False,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False,
'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': True,
'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': False,
'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False,
'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False,
'has_boiler_ENDING': False, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': True,
'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False,
'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False,
'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False,
'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False,
'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False,
'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False,
'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False,
'has_electric_ENDING': True, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False,
'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False,
'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False,
'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False,
'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False,
'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'appliance thermostats',
'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown',
'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False,
'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown',
'thermostatic_control_ENDING': 'appliance thermostats', 'charging_system_ENDING': 'Unknown',
'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown',
'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False,
'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown',
'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'electricity',
'main-fuel_tariff_type': 'Unknown', 'is_community': False,
'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
'fuel_type_ENDING': 'electricity', 'main-fuel_tariff_type_ENDING': 'Unknown',
'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False,
'complex_fuel_type_ENDING': 'Unknown', 'estimated_perimeter_STARTING': 35.4964786985977,
'estimated_perimeter_ENDING': 35.4964786985977,
'HOT_WATER_ENERGY_EFF_STARTING': "Very Poor",
"FLOOR_ENERGY_EFF_STARTING": "Unknown",
"WINDOWS_ENERGY_EFF_STARTING": "Average",
"WALLS_ENERGY_EFF_STARTING": "Very Poor",
"SHEATING_ENERGY_EFF_STARTING": "Unknown",
"ROOF_ENERGY_EFF_STARTING": "Unknown",
"MAINHEAT_ENERGY_EFF_STARTING": "Very Poor",
"MAINHEATC_ENERGY_EFF_STARTING": "Good",
"LIGHTING_ENERGY_EFF_STARTING": "Poor",
"POTENTIAL_ENERGY_EFFICIENCY": 71,
"ENVIRONMENT_IMPACT_POTENTIAL": 51,
"ENERGY_CONSUMPTION_POTENTIAL": 307,
"CO2_EMISSIONS_POTENTIAL": 3.6,
'HOT_WATER_ENERGY_EFF_ENDING': "Very Poor",
"FLOOR_ENERGY_EFF_ENDING": "Unknown",
"WINDOWS_ENERGY_EFF_ENDING": "Average",
"WALLS_ENERGY_EFF_ENDING": "Good",
"SHEATING_ENERGY_EFF_ENDING": "Unknown",
"ROOF_ENERGY_EFF_ENDING": "Unknown",
"MAINHEAT_ENERGY_EFF_ENDING": "Very Poor",
"MAINHEATC_ENERGY_EFF_ENDING": "Good",
"LIGHTING_ENERGY_EFF_ENDING": "Poor",
}
home2 = Property(
id=0,
postcode=starting_epc2["postcode"],
address1=starting_epc2["address1"],
epc_client=EpcClient(auth_token="notoken"),
data=starting_epc2
)
home2.get_components(cleaned)
data_processor2 = DataProcessor(None, newdata=True)
data_processor2.insert_data(pd.DataFrame([home2.get_model_data()]))
data_processor2.pre_process()
starting_epc_data2 = data_processor2.get_component_features(suffix="_STARTING")
ending_epc_data2 = data_processor2.get_component_features(suffix="_ENDING")
fixed_data2 = data_processor2.get_fixed_features()
ending_lodgement_date2 = '2020-11-24'
ending_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2)
recommendation2 = {
"recommendation_id": 0,
"new_u_value": 0.21,
"type": "wall_insulation"
}
test_record2 = create_recommendation_scoring_data(
property=home2,
recommendation=recommendation2,
starting_epc_data=starting_epc_data2,
ending_epc_data=ending_epc_data2,
fixed_data=fixed_data2,
)
test_record2 = pd.DataFrame([test_record2])
# Test the final cleaning:
test_record2 = DataProcessor.apply_averages_cleaning(
data_to_clean=test_record2,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
).drop(columns=["LOCAL_AUTHORITY"])
test_record2 = DataProcessor.clean_missings_after_description_process(
test_record2, [
c for c in test_record2.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
)
for c in test_record2.columns:
if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]:
continue
if c == "FLOOR_HEIGHT_ENDING":
assert (row2[c] - test_record2[c].values[0]) <= 0.020001
continue
if c == "walls_insulation_thickness_ENDING":
assert row2[c] == "average"
assert test_record2[c].values[0] == "above average"
continue
if c == "CONSTRUCTION_AGE_BAND":
# For this, we have different values in the original data
assert row2[c] == "England and Wales: 1996-2002"
assert test_record2[c].values[0] == "England and Wales: 1900-1929"
continue
assert test_record2[c].values[0] == row2[c]
def test_ventilation(self, cleaned, cleaning_data):
starting_epc3 = {
'low-energy-fixed-light-count': '', 'address': '45 Shepperson Road', 'uprn-source': 'Energy Assessor',
'floor-height': '1.87', 'heating-cost-potential': '645', 'unheated-corridor-length': '',
'hot-water-cost-potential': '69', 'construction-age-band': 'England and Wales: 1900-1929',
'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
'lighting-energy-eff': 'Average', 'environment-impact-potential': '75',
'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '1028', 'address3': '',
'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'Sheffield', 'fixed-lighting-outlets-count': '21',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96',
'county': '', 'postcode': 'S6 4FG', 'solar-water-heating-flag': 'N', 'constituency': 'E14000921',
'co2-emissions-potential': '2.9', 'number-heated-rooms': '5',
'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '152',
'local-authority': 'E08000019', 'built-form': 'Enclosed Mid-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-06-13',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '59', 'address1': '45 Shepperson Road',
'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Sheffield, Brightside and Hillsborough', 'roof-energy-eff': 'Very Poor',
'total-floor-area': '107.0', 'building-reference-number': '10002892085', 'environment-impact-current': '46',
'co2-emissions-current': '6.3', 'roof-description': 'Pitched, no insulation (assumed)',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good',
'posttown': 'SHEFFIELD', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'mains gas (not community)',
'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A',
'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 43% of fixed outlets',
'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0',
'lighting-cost-potential': '83', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
'main-heating-controls': '', 'lodgement-datetime': '2023-05-27 12:15:21', 'flat-top-storey': '',
'current-energy-rating': 'E', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor',
'transaction-type': 'marketed sale', 'uprn': '100051073214', 'current-energy-efficiency': '54',
'energy-consumption-current': '335', 'mainheat-description': 'Boiler and radiators, mains gas',
'lighting-cost-current': '131', 'lodgement-date': '2023-05-27', 'extension-count': '1',
'mainheatc-env-eff': 'Average',
'lmk-key': 'dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247', 'wind-turbine-count': '0',
'tenure': 'Not defined - use in the case of a new dwelling for which the intended tenure in not known. It '
'is not to be used for an existing dwelling',
'floor-level': '', 'potential-energy-efficiency': '80', 'hot-water-energy-eff': 'Good',
'low-energy-lighting': '43',
'walls-description': 'Sandstone or limestone, as built, no insulation (assumed)',
'hotwater-description': 'From main system'
}
row3 = {
'UPRN': '100051073214', 'RDSAP_CHANGE': 2, 'HEAT_DEMAND_CHANGE': -22, 'CARBON_CHANGE': -0.39999999999999947,
'SAP_STARTING': 54, 'SAP_ENDING': 56, 'HEAT_DEMAND_STARTING': 335, 'HEAT_DEMAND_ENDING': 313,
'CARBON_STARTING': 6.3, 'CARBON_ENDING': 5.9, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Mid-Terrace',
'CONSTITUENCY': 'E14000921', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0,
'FIXED_LIGHTING_OUTLETS_COUNT': 21.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1900-1929',
'TRANSACTION_TYPE_STARTING': 'marketed sale', 'MECHANICAL_VENTILATION_STARTING': 'natural',
'SECONDHEAT_DESCRIPTION_STARTING': 'None', 'ENERGY_TARIFF_STARTING': 'Single',
'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 0.0,
'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0,
'LOW_ENERGY_LIGHTING_STARTING': 43.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 0.0,
'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 107.0, 'FLOOR_HEIGHT_STARTING': 1.87,
'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'mechanical, extract only',
'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'Single',
'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0,
'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0,
'LOW_ENERGY_LIGHTING_ENDING': 43.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 1.0,
'TOTAL_FLOOR_AREA_ENDING': 107.0, 'FLOOR_HEIGHT_ENDING': 1.87, 'DAYS_TO_STARTING': 3221,
'DAYS_TO_ENDING': 2874, 'walls_thermal_transmittance': 2.0, 'is_cavity_wall': False,
'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False,
'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': True,
'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 2.0, 'is_park_home_ENDING': False,
'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False,
'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.62, 'is_to_unheated_space': False,
'is_to_external_air': False, 'is_suspended': True, 'is_solid': False, 'another_property_below': False,
'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.62,
'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 2.3, 'is_pitched': True,
'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'has_dwelling_above': False, 'roof_insulation_thickness': 'none', 'roof_thermal_transmittance_ENDING': 2.3,
'roof_insulation_thickness_ENDING': 'none', 'heater_type': 'Unknown', 'system_type': 'from main system',
'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown',
'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown',
'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown',
'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system',
'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown',
'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown',
'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown',
'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown',
'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False,
'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False,
'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False,
'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False,
'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True,
'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False,
'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False,
'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False,
'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False,
'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False,
'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False,
'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False,
'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False,
'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False,
'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False,
'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False,
'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False,
'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False,
'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False,
'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False,
'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'Unknown', 'charging_system': 'Unknown',
'switch_system': 'programmer', 'no_control': 'Unknown', 'dhw_control': 'Unknown',
'community_heating': 'Unknown', 'multiple_room_thermostats': False, 'auxiliary_systems': 'bypass',
'trvs': 'trvs', 'rate_control': 'Unknown', 'thermostatic_control_ENDING': 'Unknown',
'charging_system_ENDING': 'Unknown', 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown',
'dhw_control_ENDING': 'Unknown', 'community_heating_ENDING': 'Unknown',
'multiple_room_thermostats_ENDING': False, 'auxiliary_systems_ENDING': 'bypass', 'trvs_ENDING': 'trvs',
'rate_control_ENDING': 'Unknown', 'glazing_type': 'double', 'glazing_type_ENDING': 'double',
'fuel_type': 'mains gas', 'main-fuel_tariff_type': 'Unknown', 'is_community': False,
'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False,
'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown',
'estimated_perimeter_STARTING': 41.634120622393354, 'estimated_perimeter_ENDING': 41.634120622393354,
'HOT_WATER_ENERGY_EFF_STARTING': "Good",
"FLOOR_ENERGY_EFF_STARTING": "Unknown",
"WINDOWS_ENERGY_EFF_STARTING": "Average",
"WALLS_ENERGY_EFF_STARTING": "Very Poor",
"SHEATING_ENERGY_EFF_STARTING": "Unknown",
"ROOF_ENERGY_EFF_STARTING": "Very Poor",
"MAINHEAT_ENERGY_EFF_STARTING": "Good",
"MAINHEATC_ENERGY_EFF_STARTING": "Average",
"LIGHTING_ENERGY_EFF_STARTING": "Average",
"POTENTIAL_ENERGY_EFFICIENCY": 80,
"ENVIRONMENT_IMPACT_POTENTIAL": 75,
"ENERGY_CONSUMPTION_POTENTIAL": 152,
"CO2_EMISSIONS_POTENTIAL": 2.9,
'HOT_WATER_ENERGY_EFF_ENDING': "Good",
"FLOOR_ENERGY_EFF_ENDING": "Unknown",
"WINDOWS_ENERGY_EFF_ENDING": "Average",
"WALLS_ENERGY_EFF_ENDING": "Very Poor",
"SHEATING_ENERGY_EFF_ENDING": "Unknown",
"ROOF_ENERGY_EFF_ENDING": "Very Poor",
"MAINHEAT_ENERGY_EFF_ENDING": "Good",
"MAINHEATC_ENERGY_EFF_ENDING": "Average",
"LIGHTING_ENERGY_EFF_ENDING": "Average",
}
home3 = Property(
id=0,
postcode=starting_epc3["postcode"],
address1=starting_epc3["address1"],
epc_client=EpcClient(auth_token="notoken"),
data=starting_epc3
)
home3.get_components(cleaned)
data_processor3 = DataProcessor(None, newdata=True)
data_processor3.insert_data(pd.DataFrame([home3.get_model_data()]))
data_processor3.pre_process()
starting_epc_data3 = data_processor3.get_component_features(suffix="_STARTING")
ending_epc_data3 = data_processor3.get_component_features(suffix="_ENDING")
fixed_data3 = data_processor3.get_fixed_features()
ending_lodgement_date3 = '2022-06-14'
ending_epc_data3["DAYS_TO_ENDING"] = data_processor3.calculate_days_to(ending_lodgement_date3)
recommendation3 = {
"recommendation_id": 0,
"type": "mechanical_ventilation"
}
test_record3 = create_recommendation_scoring_data(
property=home3,
recommendation=recommendation3,
starting_epc_data=starting_epc_data3,
ending_epc_data=ending_epc_data3,
fixed_data=fixed_data3,
)
test_record3 = pd.DataFrame([test_record3])
# Test the final cleaning:
test_record3 = DataProcessor.apply_averages_cleaning(
data_to_clean=test_record3,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
).drop(columns=["LOCAL_AUTHORITY"])
test_record3 = DataProcessor.clean_missings_after_description_process(
test_record3, [
c for c in test_record3.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
)
for c in test_record3.columns:
if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]:
continue
assert test_record3[c].values[0] == row3[c]
def test_fireplaces(self, cleaned, cleaning_data):
starting_epc4 = {
'low-energy-fixed-light-count': '', 'address': '9 Glebe Road, Asfordby Hill',
'uprn-source': 'Energy Assessor', 'floor-height': '2.4', 'heating-cost-potential': '501',
'unheated-corridor-length': '', 'hot-water-cost-potential': '70',
'construction-age-band': 'England and Wales: 1930-1949', 'potential-energy-rating': 'C',
'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average',
'environment-impact-potential': '76', 'glazed-type': 'double glazing, unknown install date',
'heating-cost-current': '723', 'address3': '',
'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'Melton',
'fixed-lighting-outlets-count': '14', 'energy-tariff': 'dual',
'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98',
'county': 'Leicestershire', 'postcode': 'LE14 3QT', 'solar-water-heating-flag': 'N',
'constituency': 'E14000909', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '5',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '153',
'local-authority': 'E07000133', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '1',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
'inspection-date': '2022-06-27', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '46',
'address1': '9 Glebe Road', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Good',
'total-floor-area': '87.0', 'building-reference-number': '10002396876',
'environment-impact-current': '60', 'co2-emissions-current': '4.0',
'roof-description': 'Pitched, 200 mm loft insulation', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '5', 'address2': 'Asfordby Hill', 'hot-water-env-eff': 'Good',
'posttown': 'MELTON MOWBRAY', 'mainheatc-energy-eff': 'Average',
'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Average',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 29% of fixed outlets', 'roof-env-eff': 'Good',
'walls-energy-eff': 'Very Poor', 'photo-supply': '15.0', 'lighting-cost-potential': '79',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2022-06-27 15:28:18', 'flat-top-storey': '',
'current-energy-rating': 'D',
'secondheat-description': 'Room heaters, dual fuel (mineral and wood)',
'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030539619',
'current-energy-efficiency': '66', 'energy-consumption-current': '256',
'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '135',
'lodgement-date': '2022-06-27', 'extension-count': '1', 'mainheatc-env-eff': 'Average',
'lmk-key': '736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef',
'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
'potential-energy-efficiency': '78', 'hot-water-energy-eff': 'Good',
'low-energy-lighting': '29',
'walls-description': 'Solid brick, as built, no insulation (assumed)',
'hotwater-description': 'From main system'
}
row4 = {
'UPRN': '100030539619', 'RDSAP_CHANGE': 7, 'HEAT_DEMAND_CHANGE': -41, 'CARBON_CHANGE': -0.5,
'SAP_STARTING': 66, 'SAP_ENDING': 73, 'HEAT_DEMAND_STARTING': 256, 'HEAT_DEMAND_ENDING': 215,
'CARBON_STARTING': 4.0, 'CARBON_ENDING': 3.5, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Semi-Detached',
'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0,
'FIXED_LIGHTING_OUTLETS_COUNT': 14.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1930-1949',
'TRANSACTION_TYPE_STARTING': 'eco assessment', 'MECHANICAL_VENTILATION_STARTING': 'natural',
'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)',
'ENERGY_TARIFF_STARTING': 'dual', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 15.0,
'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0,
'LOW_ENERGY_LIGHTING_STARTING': 29.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0,
'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 87.0, 'FLOOR_HEIGHT_STARTING': 2.4,
'TRANSACTION_TYPE_ENDING': 'eco assessment', 'MECHANICAL_VENTILATION_ENDING': 'natural',
'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)',
'ENERGY_TARIFF_ENDING': 'dual', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 15.0,
'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0,
'LOW_ENERGY_LIGHTING_ENDING': 29.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0, 'EXTENSION_COUNT_ENDING': 1.0,
'TOTAL_FLOOR_AREA_ENDING': 87.0, 'FLOOR_HEIGHT_ENDING': 2.4, 'DAYS_TO_STARTING': 2887,
'DAYS_TO_ENDING': 2960, 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False,
'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False,
'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False,
'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 1.7, 'is_park_home_ENDING': False,
'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False,
'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.66, 'is_to_unheated_space': False,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, 'another_property_below': False,
'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.66,
'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.21, 'is_pitched': True,
'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'has_dwelling_above': False, 'roof_insulation_thickness': '200', 'roof_thermal_transmittance_ENDING': 0.21,
'roof_insulation_thickness_ENDING': '200', 'heater_type': 'Unknown', 'system_type': 'from main system',
'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown',
'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown',
'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown',
'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system',
'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown',
'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown',
'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown',
'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown',
'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False,
'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False,
'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False,
'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False,
'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True,
'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False,
'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False,
'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False,
'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False,
'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False,
'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False,
'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False,
'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False,
'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False,
'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False,
'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False,
'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False,
'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False,
'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False,
'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False,
'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat',
'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown',
'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False,
'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown',
'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown',
'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown',
'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False,
'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown',
'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'mains gas',
'main-fuel_tariff_type': 'Unknown', 'is_community': False,
'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False,
'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown',
'estimated_perimeter_STARTING': 37.54197650630557, 'estimated_perimeter_ENDING': 37.54197650630557,
'HOT_WATER_ENERGY_EFF_STARTING': "Good",
"FLOOR_ENERGY_EFF_STARTING": "Unknown",
"WINDOWS_ENERGY_EFF_STARTING": "Average",
"WALLS_ENERGY_EFF_STARTING": "Very Poor",
"SHEATING_ENERGY_EFF_STARTING": "Unknown",
"ROOF_ENERGY_EFF_STARTING": "Good",
"MAINHEAT_ENERGY_EFF_STARTING": "Good",
"MAINHEATC_ENERGY_EFF_STARTING": "Average",
"LIGHTING_ENERGY_EFF_STARTING": "Average",
"POTENTIAL_ENERGY_EFFICIENCY": 78,
"ENVIRONMENT_IMPACT_POTENTIAL": 76,
"ENERGY_CONSUMPTION_POTENTIAL": 153,
"CO2_EMISSIONS_POTENTIAL": 2.4,
'HOT_WATER_ENERGY_EFF_ENDING': "Good",
"FLOOR_ENERGY_EFF_ENDING": "Unknown",
"WINDOWS_ENERGY_EFF_ENDING": "Average",
"WALLS_ENERGY_EFF_ENDING": "Very Poor",
"SHEATING_ENERGY_EFF_ENDING": "Unknown",
"ROOF_ENERGY_EFF_ENDING": "Good",
"MAINHEAT_ENERGY_EFF_ENDING": "Good",
"MAINHEATC_ENERGY_EFF_ENDING": "Average",
"LIGHTING_ENERGY_EFF_ENDING": "Average",
}
home4 = Property(
id=0,
postcode=starting_epc4["postcode"],
address1=starting_epc4["address1"],
epc_client=EpcClient(auth_token="notoken"),
data=starting_epc4
)
home4.get_components(cleaned)
data_processor4 = DataProcessor(None, newdata=True)
data_processor4.insert_data(pd.DataFrame([home4.get_model_data()]))
data_processor4.pre_process()
starting_epc_data4 = data_processor4.get_component_features(suffix="_STARTING")
ending_epc_data4 = data_processor4.get_component_features(suffix="_ENDING")
fixed_data4 = data_processor4.get_fixed_features()
ending_lodgement_date4 = '2022-09-08'
ending_epc_data4["DAYS_TO_ENDING"] = data_processor4.calculate_days_to(ending_lodgement_date4)
recommendation4 = {
"recommendation_id": 0,
"type": "sealing_open_fireplace"
}
test_record4 = create_recommendation_scoring_data(
property=home4,
recommendation=recommendation4,
starting_epc_data=starting_epc_data4,
ending_epc_data=ending_epc_data4,
fixed_data=fixed_data4,
)
test_record4 = pd.DataFrame([test_record4])
# Test the final cleaning:
test_record4 = DataProcessor.apply_averages_cleaning(
data_to_clean=test_record4,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
).drop(columns=["LOCAL_AUTHORITY"])
test_record4 = DataProcessor.clean_missings_after_description_process(
test_record4, [
c for c in test_record4.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
)
for c in test_record4.columns:
if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]:
continue
assert test_record4[c].values[0] == row4[c]

1
datatypes/enums.py
View file

@ -3,3 +3,4 @@ import enum
class QuantityUnits(enum.Enum):
m2 = "m2"
part = "part"

35
etl/costs/README.md Normal file
View file

@ -0,0 +1,35 @@
### Costs ETL Application
This is a simple application to push the materials costs data to the database.
#### How to run
Ensure you have a .env file in the base Model directory with the following variables
```
DB_HOST="Your db host"
DB_PORT="Your db port"
DB_USER="Your db user"
DB_PASSWORD="Your db password"
DB_NAME="Your db name"
```
Make sure your python path environment variable pouints to the base Model directory. To set the
`PYTHONPATH` environment variable, run the following command from the base Model directory
```
export PYTHONPATH=`pwd`
```
From the base Model directory, install the requirements by running the following command
```
pip install -r etl/costs/requirements.txt
```
Then run the following command to run the application
```
python etl/costs/app.py
```

106
etl/costs/app.py Normal file
View file

@ -0,0 +1,106 @@
import os
import dotenv
import pandas as pd
import numpy as np
from pathlib import Path
from sqlalchemy.orm import Session
from sqlalchemy import create_engine
from backend.app.db.models.materials import Material
from recommendations.recommendation_utils import calculate_r_value_per_mm
DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx"
# Environment file is at the same level as this file
ENV_FILE = Path(__file__).parent / "etl" / "costs" / ".env"
dotenv.load_dotenv(ENV_FILE)
DB_USERNAME = os.getenv('DB_USERNAME')
DB_PASSWORD = os.getenv('DB_PASSWORD')
DB_HOST = os.getenv('DB_HOST')
DB_PORT = os.getenv('DB_PORT')
DB_NAME = os.getenv('DB_NAME')
def push_costs_to_db(engine, costs_df):
"""
Push costs DataFrame to the database.
:param engine: The SQLAlchemy engine connected to your database.
:param costs_df: The DataFrame containing cost data.
"""
materials = []
for _, row in costs_df.iterrows():
row_dict = row.to_dict()
# Add other necessary transformations here
# Create Material object and add it to the list
materials.append(Material(**row_dict))
# Use SQLAlchemy session for bulk insert
with Session(engine) as session:
session.bulk_save_objects(materials)
session.commit()
def app():
"""
This application uploads the cost data to our database
The most recent cost data can be found in OneDrive, in the
shared folder > 04. Product Development > Cost data > Hestia Materials.xlsx
For the moment, the data is uploaded manually. In the future, we will automate this so the data can be
stored locally and then is uploaded from the local_data folder
:return:
"""
connection_string = "postgresql+{drivername}://{username}:{password}@{server}:{port}/{dbname}"
db_string = connection_string.format(
drivername="psycopg2", # You'll need to use psycopg2 driver for PostgreSQL
username=DB_USERNAME,
password=DB_PASSWORD,
server=DB_HOST,
port=DB_PORT,
dbname=DB_NAME,
)
db_engine = create_engine(db_string, pool_size=5, max_overflow=5)
cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0)
loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0)
iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0)
suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0)
solid_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="solid_floor_insulation", header=0)
ewi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="external_wall_insulation", header=0)
# Form a single table to be uploaded
costs = pd.concat(
[
cwi_costs,
loft_insulation_costs,
iwi_costs,
suspended_floor_costs,
solid_floor_costs,
ewi_costs,
]
)
costs = costs.replace({np.nan: None})
costs["depth"] = costs["depth"].fillna(0)
costs["depth"] = costs["depth"].astype(str)
costs["r_value_per_mm"] = costs.apply(
lambda row: calculate_r_value_per_mm(float(row["depth"]), row["thermal_conductivity"]), axis=1
)
costs["r_value_unit"] = "square_meter_kelvin_per_watt"
for col in ["material_cost", "labour_cost", "labour_hours_per_unit", "plant_cost"]:
costs[col] = costs[col].fillna(0)
# Push the costs to the database
push_costs_to_db(db_engine, costs)
if __name__ == "__main__":
app()

5
etl/costs/requirements.txt Normal file
View file

@ -0,0 +1,5 @@
pandas==1.5.3
sqlalchemy==2.0.19
python-dotenv
psycopg2-binary
openpyxl

54
etl/epc/DataProcessor.py
View file

@ -16,7 +16,9 @@ from etl.epc.settings import (
fill_na_map,
STARTING_SUFFIX_COMPONENT_COLS,
NO_SUFFIX_COMPONENT_COLS,
ENDING_SUFFIX_COMPONENT_COLS
ENDING_SUFFIX_COMPONENT_COLS,
POTENTIAL_COLUMNS,
EFFICIENCY_FEATURES,
)
from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
@ -177,7 +179,6 @@ class DataProcessor:
# We have some non-standard construction age bands which we'll clean for matching
if not self.newdata:
self.standardise_construction_age_band()
self.clean_missing_rooms()
self.recast_df_columns(
@ -203,6 +204,8 @@ class DataProcessor:
# Final re-casting after data transformed and prepared
coltypes = {k: v for k, v in COLUMNTYPES.items() if k in self.data.columns} if self.newdata else COLUMNTYPES
for k, v in coltypes.items():
self.data[k] = self.data[k].astype(v)
self.data = self.data.astype(coltypes)
self.na_remapping()
@ -447,7 +450,7 @@ class DataProcessor:
self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0)
@staticmethod
def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on):
def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on, colnames=None):
"""
Clean the input DataFrame using averages from a cleaning DataFrame.
@ -455,11 +458,16 @@ class DataProcessor:
:param cleaning_data: DataFrame containing data for cleaning.
:param cols_to_merge_on: Columns on which merging is based. We pass cols_to_merge_on to this function as this
differs depending on where the function is being used.
:param colnames: If specified can be used to state exactly which columns to clean
:return: Cleaned DataFrame.
"""
# The desired colnames to clean - which may not be present
if colnames is None:
colnames = ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"]
cols_to_clean = [
c for c in ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"] if
c for c in colnames if
c in data_to_clean.columns
]
@ -488,6 +496,8 @@ class DataProcessor:
for col in cols_to_clean:
data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"], inplace=True)
data_to_clean.drop(columns=[f"{col}_AVERAGE"], inplace=True)
# If we still have missings
data_to_clean[col].fillna(data_to_clean[col].mean(), inplace=True)
return data_to_clean
@ -504,12 +514,14 @@ class DataProcessor:
raise Exception("Suffix should be one of _STARTING or _ENDING")
if suffix == "_STARTING":
starting_cols = self.data[STARTING_SUFFIX_COMPONENT_COLS].copy().add_suffix(suffix)
fixed_cols = self.data[NO_SUFFIX_COMPONENT_COLS].copy()
starting_cols = self.data[STARTING_SUFFIX_COMPONENT_COLS + EFFICIENCY_FEATURES].copy().add_suffix(suffix)
fixed_cols = self.data[NO_SUFFIX_COMPONENT_COLS + POTENTIAL_COLUMNS].copy()
return pd.concat([starting_cols, fixed_cols], axis=1)
return self.data[ENDING_SUFFIX_COMPONENT_COLS].copy().add_suffix(suffix)
return self.data[
ENDING_SUFFIX_COMPONENT_COLS + EFFICIENCY_FEATURES
].copy().add_suffix(suffix)
def get_fixed_features(self) -> pd.DataFrame:
"""
@ -569,3 +581,31 @@ class DataProcessor:
df[col] = df[col].fillna("Unknown")
return df
@staticmethod
def clean_efficiency_variables(df):
"""
These is scope to clean this by the model per corresponding description.
E.g. for WALLS_ENG_EFF we could look at the mode efficiency rating by description and
fill in the missing values with this.
When looking at this initially, there are a large volume of records with missing energy efficiency
values and therefore a simpler approach was taken just to test including these variables
:param df:
:return:
"""
missings = pd.isnull(df).sum()
missings = missings[missings >= 1]
if len(missings) == 0:
return df
# Make sure they are all efficiency columns
if any(~missings.index.str.contains("ENERGY_EFF")):
raise ValueError("Non efficiency columns are missing")
for m in missings.index:
df[m] = df[m].fillna("NO_RATING")
return df

88
etl/epc/property_change_app.py
View file

@ -12,6 +12,10 @@ from etl.epc.settings import (
HEAT_DEMAND_RESPONSE,
COLUMNS_TO_MERGE_ON,
CARBON_RESPONSE,
CORE_COMPONENT_FEATURES,
EFFICIENCY_FEATURES,
POTENTIAL_COLUMNS,
MINIMUM_FLOOR_HEIGHT
)
from etl.epc.DataProcessor import DataProcessor
from utils.s3 import save_dataframe_to_s3_parquet, read_from_s3
@ -254,6 +258,13 @@ def make_uvalues(df):
# Roof
# ~~~~~~~~~~~~~~~~~~
if x["has_dwelling_above"]:
if x["roof_thermal_transmittance"] != 0:
raise ValueError("Should have 0 u-value for roof")
if x["roof_thermal_transmittance_ENDING"] != 0:
raise ValueError("Should have 0 u-value for roof")
starting_roof_uvalue = x["roof_thermal_transmittance"]
if pd.isnull(starting_roof_uvalue):
starting_roof_uvalue = get_roof_u_value(
@ -297,6 +308,11 @@ def make_uvalues(df):
wall_type = get_wall_type(**x)
if x["another_property_below"]:
if x["floor_thermal_transmittance"] != 0:
raise ValueError("Should have 0 u-value for floor")
if x["floor_thermal_transmittance_ENDING"] != 0:
raise ValueError("Should have 0 u-value for floor")
starting_floor_uvalue, ending_floor_uvalue = 0, 0
else:
starting_floor_uvalue = x["floor_thermal_transmittance"]
@ -363,6 +379,25 @@ def make_uvalues(df):
return df
def compare_records(earliest_record: pd.Series, latest_record: pd.Series, columns: list):
"""
For a list of columns, check if the earliest and latest record are the same
If they are the same, we indicate this, because we have example of SAP scores changing
without any feature changes
:param earliest_record: pd.Series
:param latest_record: pd.Series
:param columns: list of columns to compare
:return: boolean indicating whether or not all features are the same
"""
all_equal = True
for col in columns:
if earliest_record[col] != latest_record[col]:
return False
if all_equal:
return True
def app():
# Get all the files in the directory
@ -376,15 +411,15 @@ def app():
dataset = []
cleaning_dataset = []
# Keep track of the all equals
all_equal_rows = []
for directory in tqdm(directories):
filepath = directory / "certificates.csv"
data_processor = DataProcessor(filepath=filepath)
df = data_processor.pre_process()
df[df["WALLS_DESCRIPTION"].str.contains("Cavity")]["WALLS_DESCRIPTION"].unique()
cleaning_averages = data_processor.make_cleaning_averages()
@ -422,7 +457,9 @@ def app():
# We include the lodgement date here as we probably need to factor time into the
# model, since EPC standards and rigour have changed over time
variable_data = property_data[
COMPONENT_FEATURES + ["LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE, CARBON_RESPONSE]
COMPONENT_FEATURES + EFFICIENCY_FEATURES + POTENTIAL_COLUMNS + [
"LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE, CARBON_RESPONSE
]
]
# Note: we look at changes between subsequent EPCS, however we could look at other permutations
@ -439,6 +476,8 @@ def app():
# Check if the sap gets better or worse
gets_better = earliest_record[RDSAP_RESPONSE] <= latest_record[RDSAP_RESPONSE]
component_variables = COMPONENT_FEATURES + EFFICIENCY_FEATURES
if gets_better:
starting_sap = earliest_record[RDSAP_RESPONSE]
starting_heat_demand = earliest_record[HEAT_DEMAND_RESPONSE]
@ -452,8 +491,8 @@ def app():
heat_demand_change = latest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand
carbon_change = latest_record[CARBON_RESPONSE] - starting_carbon
starting_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
ending_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING")
starting_record = earliest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
ending_record = latest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_ENDING")
else:
starting_sap = latest_record[RDSAP_RESPONSE]
starting_heat_demand = latest_record[HEAT_DEMAND_RESPONSE]
@ -467,12 +506,23 @@ def app():
heat_demand_change = earliest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand
carbon_change = earliest_record[CARBON_RESPONSE] - starting_carbon
starting_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
ending_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING")
starting_record = latest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_STARTING")
ending_record = earliest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_ENDING")
if rdsap_change == 0:
continue
all_equal = compare_records(
earliest_record=earliest_record,
latest_record=latest_record,
columns=CORE_COMPONENT_FEATURES
)
if all_equal:
# Keep track of this for the moment so we can analyse
all_equal_rows.append({"uprn": uprn, "directory_name": directory.name})
continue
features = pd.concat([starting_record, ending_record])
property_model_data.append(
@ -487,6 +537,10 @@ def app():
"HEAT_DEMAND_ENDING": ending_heat_demand,
"CARBON_STARTING": starting_carbon,
"CARBON_ENDING": ending_carbon,
"POTENTIAL_ENERGY_EFFICIENCY": earliest_record["POTENTIAL_ENERGY_EFFICIENCY"],
"ENVIRONMENT_IMPACT_POTENTIAL": earliest_record["ENVIRONMENT_IMPACT_POTENTIAL"],
"ENERGY_CONSUMPTION_POTENTIAL": earliest_record["ENERGY_CONSUMPTION_POTENTIAL"],
"CO2_EMISSIONS_POTENTIAL": earliest_record["CO2_EMISSIONS_POTENTIAL"],
**fixed_data,
**features.to_dict(),
}
@ -496,16 +550,18 @@ def app():
data_by_urpn_df = pd.DataFrame(data_by_urpn)
# Add some temporal features - we look at the days from the standard starting point in time
# for the starting and ending date so all records are from a fixed point
data_by_urpn_df["DAYS_TO_STARTING"] = DataProcessor.calculate_days_to(
data_by_urpn_df["LODGEMENT_DATE_STARTING"])
data_by_urpn_df["LODGEMENT_DATE_STARTING"]
)
data_by_urpn_df["DAYS_TO_ENDING"] = DataProcessor.calculate_days_to(
data_by_urpn_df["LODGEMENT_DATE_ENDING"])
data_by_urpn_df["LODGEMENT_DATE_ENDING"]
)
data_by_urpn_df = data_by_urpn_df.drop(columns=["LODGEMENT_DATE_STARTING", "LODGEMENT_DATE_ENDING"])
data_by_urpn_df = DataProcessor.clean_efficiency_variables(data_by_urpn_df)
# We look for key building fabric features that have changed from one EPC to the next.
# if, for example, we see that a home has gone from being a cavity wall to a solid wall, we
# remove this record, as it indicates that the quality of the EPC conducted in the first instance
@ -539,6 +595,8 @@ def app():
cleaning_averages["LOCAL_AUTHORITY"] = df["LOCAL_AUTHORITY"].values[0]
cleaning_dataset.append(cleaning_averages)
print("Final all equal count: %s" % str(len(all_equal_rows)))
# Store cleaning dataset in s3 as a parquet file
cleaning_dataset = pd.concat(cleaning_dataset)
save_dataframe_to_s3_parquet(
@ -565,6 +623,14 @@ def app():
file_key="sap_change_model/dataset.parquet",
)
# Store all_equal_rows
all_equal_rows = pd.DataFrame(all_equal_rows)
save_dataframe_to_s3_parquet(
df=all_equal_rows,
bucket_name="retrofit-data-dev",
file_key="sap_change_model/all_equal_rows.parquet",
)
if __name__ == "__main__":
app()

42
etl/epc/settings.py
View file

@ -85,8 +85,7 @@ FIXED_FEATURES = [
"FIXED_LIGHTING_OUTLETS_COUNT",
]
COMPONENT_FEATURES = [
"TRANSACTION_TYPE",
CORE_COMPONENT_FEATURES = [
"WALLS_DESCRIPTION",
"FLOOR_DESCRIPTION",
"LIGHTING_DESCRIPTION",
@ -96,21 +95,48 @@ COMPONENT_FEATURES = [
"MAIN_FUEL",
"MECHANICAL_VENTILATION",
"SECONDHEAT_DESCRIPTION",
"ENERGY_TARIFF", # Not sure if this is relevant
"SOLAR_WATER_HEATING_FLAG",
"PHOTO_SUPPLY",
"WINDOWS_DESCRIPTION",
"GLAZED_TYPE",
"MULTI_GLAZE_PROPORTION",
"LOW_ENERGY_LIGHTING",
"NUMBER_OPEN_FIREPLACES",
"MAINHEATCONT_DESCRIPTION",
"SOLAR_WATER_HEATING_FLAG",
"PHOTO_SUPPLY",
]
EFFICIENCY_FEATURES = [
'HOT_WATER_ENERGY_EFF',
'FLOOR_ENERGY_EFF',
'WINDOWS_ENERGY_EFF',
'WALLS_ENERGY_EFF',
'SHEATING_ENERGY_EFF',
'ROOF_ENERGY_EFF',
'MAINHEAT_ENERGY_EFF',
'MAINHEATC_ENERGY_EFF',
'LIGHTING_ENERGY_EFF'
]
COMPONENT_FEATURES = CORE_COMPONENT_FEATURES + [
"TRANSACTION_TYPE",
"ENERGY_TARIFF", # Not sure if this is relevant
"EXTENSION_COUNT",
"TOTAL_FLOOR_AREA",
"FLOOR_HEIGHT",
# 'GLAZED_AREA', # May not need this since we have MULTI_GLAZE_PROPORTION
]
POTENTIAL_COLUMNS = [
'POTENTIAL_ENERGY_EFFICIENCY',
'ENVIRONMENT_IMPACT_POTENTIAL',
'ENERGY_CONSUMPTION_POTENTIAL',
'CO2_EMISSIONS_POTENTIAL',
# We don't include cost features for the moment
# 'LIGHTING_COST_POTENTIAL',
# 'HEATING_COST_POTENTIAL',
# 'HOT_WATER_COST_POTENTIAL'
]
# For these fields, we take the latest value if we have multiple values
# Since more recent EPCs have been conducted with more rigour, we assume that the latest value is
# the most accurate
@ -168,6 +194,8 @@ COLUMNTYPES = {
'MAINHEATCONT_DESCRIPTION': 'object',
'EXTENSION_COUNT': 'float64',
'LODGEMENT_DATE': 'object',
**dict(zip(EFFICIENCY_FEATURES, ['object', ] * len(EFFICIENCY_FEATURES))),
**dict(zip(POTENTIAL_COLUMNS, ['float64', ] * len(POTENTIAL_COLUMNS)))
}
# For modelling, we don't allow records with more than 100 SAP points
@ -253,3 +281,7 @@ ENDING_SUFFIX_COMPONENT_COLS = [
'rate_control', 'glazing_type', 'fuel_type', 'main-fuel_tariff_type', 'is_community',
'no_individual_heating_or_community_network', 'complex_fuel_type', 'estimated_perimeter'
]
# We found that without performing any filtering, the bottom 0.5% of homes had a floor height of 1.65m. We'll therefore
# filter out any homes with a floor height below this
MINIMUM_FLOOR_HEIGHT = 1.65

4
etl/epc_clean/epc_attributes/FloorAttributes.py
View file

@ -107,4 +107,8 @@ class FloorAttributes(Definitions):
else:
result['insulation_thickness'] = None
if result["another_property_below"]:
result["thermal_transmittance"] = 0
result["thermal_transmittance_unit"] = 'w/m-¦k'
return result

4
etl/epc_clean/epc_attributes/RoofAttributes.py
View file

@ -138,4 +138,8 @@ class RoofAttributes(Definitions):
if "insulation_thickness" not in result:
result['insulation_thickness'] = None
if result["has_dwelling_above"]:
result["thermal_transmittance"] = 0
result["thermal_transmittance_unit"] = 'w/m-¦k'
return result

9
etl/epc_clean/epc_attributes/WallAttributes.py
View file

@ -133,4 +133,13 @@ class WallAttributes(Definitions):
result['external_insulation'] = 'external insulation' in description
result['internal_insulation'] = 'internal insulation' in description
if result["is_filled_cavity"]:
# If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation
if result["external_insulation"]:
result["insulation_thickness"] = "above average"
elif result["internal_insulation"]:
result["insulation_thickness"] = "above average"
else:
result["insulation_thickness"] = "average"
return result

16
etl/epc_clean/tests/test_data/test_floor_attributes_cases.py
View file

@ -1,14 +1,14 @@
clean_floor_cases = [
{'original_description': '(another dwelling below)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False,
{'original_description': '(another dwelling below)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None,
"another_property_below": True},
{'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False,
{'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None,
"another_property_below": True},
{'original_description': '(other premises below)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None,
{'original_description': '(other premises below)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k",
'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False,
'is_solid': False, 'insulation_thickness': None,
"another_property_below": True},
@ -342,8 +342,8 @@ clean_floor_cases = [
{'original_description': 'To unheated space, no insulation (assumed)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False,
'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False},
{'original_description': '(eiddo arall islaw)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None,
{'original_description': '(eiddo arall islaw)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k",
'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False,
'is_solid': False, 'insulation_thickness': None,
"another_property_below": True},

14
etl/epc_clean/tests/test_data/test_roof_attributes_cases.py
View file

@ -1,10 +1,11 @@
clean_roof_test_cases = [
{'original_description': '(another dwelling above)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
{'original_description': '(another dwelling above)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False,
'is_flat': False,
'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True,
'insulation_thickness': None},
{'original_description': '(other premises above)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None,
{'original_description': '(other premises above)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k",
'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True,
'insulation_thickness': None},
@ -362,8 +363,9 @@ clean_roof_test_cases = [
'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': 'average'},
{'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False,
{'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False,
'is_flat': False,
'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True,
'insulation_thickness': None},
{'original_description': 'Ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None,

10
etl/epc_clean/tests/test_data/test_wall_attributes_cases.py
View file

@ -567,17 +567,17 @@ wall_cases = [
{'original_description': 'Cavity wall, filled cavity', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average",
'external_insulation': False, 'internal_insulation': False},
{'original_description': 'Cavity wall, filled cavity and external insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average',
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average',
'external_insulation': True, 'internal_insulation': False},
{'original_description': 'Cavity wall, filled cavity and internal insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average',
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average',
'external_insulation': False, 'internal_insulation': True},
{'original_description': 'Cavity wall, with external insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': False, 'is_solid_brick': False,
@ -723,7 +723,7 @@ wall_cases = [
{'original_description': 'Waliau ceudod, ceudod wediGÇÖi lenwi', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average",
'external_insulation': False, 'internal_insulation': False},
{'original_description': 'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)',
'thermal_transmittance': None,
@ -778,7 +778,7 @@ wall_cases = [
{'original_description': 'Waliau ceudod, ynysydd allanol a llenwi ceudod', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False,
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average',
'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average',
'external_insulation': True, 'internal_insulation': False},
{'original_description': 'Gwenithfaen neu risgraig, gydag inswleiddio mewnol', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,

4
etl/epc_clean/tests/test_roof_attributes.py
View file

@ -75,8 +75,8 @@ class TestRoofAttributes:
"is_assumed": False,
"is_flat": False,
"is_thatched": False,
"thermal_transmittance": None,
"thermal_transmittance_unit": None,
"thermal_transmittance": 0,
"thermal_transmittance_unit": "w/m-¦k",
}
for k in expected_output:

22
infrastructure/terraform/main.tf
View file

@ -129,6 +129,18 @@ module "retrofit_sap_data" {
allowed_origins = var.allowed_origins
}
module "retrofit_carbon_predictions" {
source = "./modules/s3"
bucketname = "retrofit-carbon-predictions-${var.stage}"
allowed_origins = var.allowed_origins
}
module "retrofit_heat_predictions" {
source = "./modules/s3"
bucketname = "retrofit-heat-predictions-${var.stage}"
allowed_origins = var.allowed_origins
}
# Set up the route53 record for the API
module "route53" {
@ -160,3 +172,13 @@ module "eco_spreadsheet_ecr" {
ecr_name = "eco-spreadsheet-${var.stage}"
source = "./modules/ecr"
}
module "lambda_carbon_prediction_ecr" {
ecr_name = "lambda-carbon-prediction-${var.stage}"
source = "./modules/ecr"
}
module "lambda_heat_prediction_ecr" {
ecr_name = "lambda-heat-prediction-${var.stage}"
source = "./modules/ecr"
}

571
recommendations/Costs.py Normal file
View file

@ -0,0 +1,571 @@
import numpy as np
# This data comes from SPONs
regional_labour_variations = [
{"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00},
{"Region": "Inner London", "Adjustment_Factor": 1.05},
{"Region": "South East", "Adjustment_Factor": 0.96},
{"Region": "South West", "Adjustment_Factor": 0.90},
{"Region": "East of England", "Adjustment_Factor": 0.93},
{"Region": "East Midlands", "Adjustment_Factor": 0.88},
{"Region": "West Midlands", "Adjustment_Factor": 0.87},
{"Region": "North East", "Adjustment_Factor": 0.83},
{"Region": "North West", "Adjustment_Factor": 0.88},
{"Region": "Yorkshire and Humberside", "Adjustment_Factor": 0.86},
{"Region": "Wales", "Adjustment_Factor": 0.88},
{"Region": "Scotland", "Adjustment_Factor": 0.88},
{"Region": "Northern Ireland", "Adjustment_Factor": 0.76}
]
county_map = {
"Northamptonshire": "East Midlands",
"Hampshire": "South East",
}
class Costs:
"""
A class to calculate the costs associated with construction works,
specifically focusing on cavity wall insulation.
It includes contingency, preliminaries, profit margin, and VAT calculations.
As a sense check, there is a useful article from checkatrade on retrofitting and expected costs:
https://www.checkatrade.com/blog/cost-guides/retrofit-insulation-cost/
Another useful article for benchmarking the cost of floor insulation:
https://www.checkatrade.com/blog/cost-guides/floor-insulation-cost/
"""
# Contingency is a percentage of the total cost of the work and covers unforseen expenses
# We assume a conservative 10% contingency for all works which is a rate defined by SPONs
CONTINGENCY = 0.1
# Where there is more uncertainty, a higher contingency rate is used
HIGH_RISK_CONTINGENCY = 0.15
# When there is less uncertainty, a lower contingency rate is used
LOW_RISK_CONTINGENCY = 0.05
# Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs
# such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10%
# rate, on the total cost before VAT, as recommended by SPONs
PRELIMINARIES = 0.1
# For higher risk projects, a higher preliminaries rate is used. SPONs indicates that a higher risk project might
# have a preliminaries of 12-14% so we use 12% as the median for the preliminaries rate.
# For External wall insulation (EWI), we use 15% as the preliminaries rate if we think the property might
# need scaffolding, otherwise we use 12%. This is to account for any site preparation that might be required
EWI_NO_SCAFFOLDING_PRELIMINARIES = 0.12
EWI_SCAFFOLDING_PRELIMINARIES = 0.15
VAT_RATE = 0.2
PROFIT_MARGIN = 0.15
def __init__(self, property_instance):
"""
Initializes the Costs class with a property instance.
:param property_instance: Instance of a Property class containing relevant details like wall area.
"""
if not hasattr(property_instance, 'insulation_wall_area'):
raise ValueError("Property instance must have an 'insulation_wall_area' attribute")
self.property = property_instance
self.regional_labour_variations = regional_labour_variations
self.county = county_map.get(self.property.data["county"], None)
if self.county is None:
raise ValueError("County not found in county map")
self.labour_adjustment_factor = [
x["Adjustment_Factor"] for x in self.regional_labour_variations if
x["Region"] == self.county
][0]
if not self.labour_adjustment_factor:
raise ValueError("Labour adjustment factor not found")
def cavity_wall_insulation(self, wall_area, material):
"""
Calculates the total cost for cavity wall insulation based on material and labor costs,
including contingency, preliminaries, profit, and VAT.
Because of some limitations in the SPONs data, there are no materials that can be blown through a wall,
therefore we have adapted similar materials, basing our estimates on 75mm cavity slabs, and have halved the
labour time required. That is why we still price based on wall area despite volume actually being the correct
metric.
:return: A dictionary containing detailed cost breakdown.
"""
material_cost_per_m2 = material["material_cost"]
base_material_cost = material_cost_per_m2 * wall_area
labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor
subtotal_before_profit = base_material_cost + labour_cost
contingency_cost = subtotal_before_profit * self.CONTINGENCY
preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
labour_hours = material["labour_hours_per_unit"] * wall_area
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": base_material_cost,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_cost": labour_cost
}
def loft_insulation(self, floor_area, material):
"""
Calculates the total cost for cavity wall insulation based on material and labor costs,
including contingency, preliminaries, profit, and VAT.
:return: A dictionary containing detailed cost breakdown.
"""
material_cost_per_m2 = material["material_cost"]
base_material_cost = material_cost_per_m2 * floor_area
labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor
subtotal_before_profit = base_material_cost + labour_cost
contingency_cost = subtotal_before_profit * self.CONTINGENCY
preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
labour_hours = material["labour_hours_per_unit"] * floor_area
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": base_material_cost,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_cost": labour_cost
}
def internal_wall_insulation(self, wall_area, material, non_insulation_materials):
"""
Broadly speaking, the high level steps to an internal wall insulation job are the following:
1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles.
It's important to factor in the disposal of debris and the potential need for additional protective
measures to ensure the safety of the work area.
2) Insulation Installation: This is the core part of the process where the chosen insulation material is
applied. The choice of insulation material will depend on several factors including thermal performance,
wall construction, and space constraints.
3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation,
which can compromise its effectiveness and lead to mold growth.
4) Re-decoration: This involves applying plaster to the wall and then painting.
The quality of finish here is important for both aesthetic and functional reasons.
5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames,
or window sills might be necessary.
:return:
"""
# Extract and check the different types of data we'll need
demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"]
vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"]
redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"]
if not demolition_data:
raise ValueError("No data found for iwi_wall_demolition")
if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3):
raise ValueError("Incorrect number of data entries for non-insulation materials")
# Break out the individual material costs
# Since we don't know the exact wall construction, we take an average for demolition costs, since
# the cost will depend on the type of wall construction
demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data])
insulation_material_costs = material["material_cost"] * wall_area
vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area
redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data])
demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data])
# Again for demolition, we average since we aren't sure which demolition process will be used
demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data])
insulation_labour_costs = material["labour_cost"] * wall_area
vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area
redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data])
labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
redecoration_labour_costs)
labour_costs = labour_costs * self.labour_adjustment_factor
materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs +
redecoration_material_costs)
subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs
# We use high risk contingency for iwi
contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data])
insulation_labour_hours = material["labour_hours_per_unit"] * wall_area
vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area
redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data])
labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
redecoration_labour_hours)
# To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people
labour_days = (labour_hours / 8) / 4
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": materials_costs,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_days": labour_days,
"labour_cost": labour_costs
}
def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials):
"""
We characterise the steps for suspended floor insulation as the following tasks:
1) Removal of Carpet and Underfelt: Where necessary, remove existing floor coverings to access the floorboards.
2) Removal of Floor Boarding: Carefully remove floorboards to access the space beneath for insulation.
3) Installation of Vapour Barrier: Install a vapour barrier to prevent moisture from affecting
the insulation and floor structure.
4) Installation of Insulation: Fit the chosen insulation material between the joists in the floor void.
5) Refixing Floorboards: Replace and secure the floorboards after insulation installation.
6) Re-carpeting: Lay down the carpet or other floor coverings once the insulation and floorboards are in place.
:return:
"""
demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"]
vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"]
redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"]
if (len(demolition_data) != 2) or (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2):
raise ValueError("Incorrect number of data entries for non-insulation materials")
# Break out the individual material costs
demolition_material_costs = sum([x["material_cost"] * insulation_floor_area for x in demolition_data])
insulation_material_costs = material["material_cost"] * insulation_floor_area
vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area
redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data])
demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data])
insulation_labour_costs = material["labour_cost"] * insulation_floor_area
vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area
redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data])
labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
redecoration_labour_costs)
labour_costs = labour_costs * self.labour_adjustment_factor
materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs +
redecoration_material_costs)
subtotal_before_profit = labour_costs + materials_costs
contingency_cost = subtotal_before_profit * self.CONTINGENCY
preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data])
insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area
redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data])
labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
redecoration_labour_hours)
# Assume a team of 3 people for a small to medium size project
labour_days = (labour_hours / 8) / 3
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": materials_costs,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_days": labour_days,
"labour_cost": labour_costs
}
def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials):
"""
We characterise the steps for solid floor insulation as the following tasks:
1) Removal of Carpet and Underfelt: This is the initial stage where any existing floor coverings, like carpets,
tiles, or linoleum, are carefully removed. This exposes the solid floor beneath, which is typically concrete.
2) Preparation of Flooring: This step is critical. It involves:
- Cleaning the existing floor surface thoroughly to remove debris and ensure a flat surface.
- Assessing and repairing any damage to the concrete floor. This might include filling cracks or leveling
uneven areas.
3) Installation of a Damp Proof Membrane (DPM): Before installing insulation, a DPM is often laid down to
prevent moisture from rising into the insulation and the interior space. This step is crucial in areas prone to
dampness.
4) Install Insulation: The insulation, often in the form of rigid foam boards, is laid over the DPM.
The choice of insulation material will depend on the desired thermal properties and the available floor height.
Care is taken to minimize thermal bridges and ensure a snug fit between insulation boards.
5) Laying a New Subfloor: Over the insulation, a new subfloor is often installed. This could be a layer of
screed (a type of concrete) or wooden boarding, depending on the specific requirements and preferences.
6) Re-decoration and Finishing Touches: Once the subfloor is in place and has set or dried (if necessary),
the final floor finish can be applied. This might involve:
- Laying new tiles, wooden flooring, or other chosen materials.
- If you're planning to re-carpet, this would be the stage to do it.
- Skirting boards may need to be refitted or replaced.
7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door
thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments.
:param insulation_floor_area: Area of the floor to be insulated
:param material: Selected insulation material
:param non_insulation_materials: Non-insulation materials required for the job
:return:
"""
demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"]
preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"]
vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"]
redecoration_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_redecoration"]
if ((len(demolition_data) != 1) or (len(preparation_data) != 2) or (len(vapour_barrier_data) != 1) or
(len(redecoration_data) != 3)):
raise ValueError("Incorrect number of data entries for non-insulation materials")
# Break out the individual material costs
preparation_material_costs = sum([x["material_cost"] * insulation_floor_area for x in preparation_data])
insulation_material_costs = material["material_cost"] * insulation_floor_area
vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area
redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data])
demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data])
preparation_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in preparation_data])
insulation_labour_costs = material["labour_cost"] * insulation_floor_area
vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area
redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data])
preparation_plant_costs = sum([x["plant_cost"] * insulation_floor_area for x in preparation_data])
labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
redecoration_labour_costs + preparation_labour_costs)
labour_costs = labour_costs * self.labour_adjustment_factor
materials_cost = (preparation_material_costs + insulation_material_costs + vapour_barrier_material_costs +
redecoration_material_costs)
subtotal_before_profit = labour_costs + materials_cost + preparation_plant_costs
# We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors,
# as well as scope for damage to the existing floor during preparation.
contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data])
preparation_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in preparation_data])
insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area
redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data])
labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
redecoration_labour_hours + preparation_labour_hours)
# Assume a team of 3 people for a small to medium size project
labour_days = (labour_hours / 8) / 3
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": materials_cost,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_days": labour_days,
"labour_cost": labour_costs
}
def external_wall_insulation(self, wall_area, material, non_insulation_materials):
"""
We characterise external wall insulation as the following steps:
1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective
sheets to protect the flooring and landscaping around the work area.
2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or
detached houses. For terraced houses or lower-level work, scaffolding might not be necessary.
3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material,
and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement
of the top surface may be necessary.
4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation
boards and stabilize the wall surface, especially if it's old or weathered.
5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings,
or a combination of both.
6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation.
This layer provides strength and helps prevent cracking.
7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation
and provides an aesthetic look.
8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that
were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness.
9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system,
followed by cleaning up the site to remove all debris and materials.
In the actual materials data, at this point, we have costing for:
- wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition)
- wall surface cleaning and priming (ewi_wall_preparation)
- insulation (external_wall_insulation)
- basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh)
All of this data comes from SPONS, however there are some clear features missing. Because we could not find
suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding,
re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by
increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these
steps should be included in the materials data. We will look to improve this in the future, with data from
installers
:param wall_area:
:param material:
:param non_insulation_materials:
:return:
"""
# For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding
if self.property.data["property-type"] == "House":
if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]:
preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
else:
preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
elif self.property.data["property-type"] == "Maisonette":
preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
elif self.property.data["property-type"] == "Bungalow":
preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
else:
raise ValueError("Unsupported property type - haven't handled flats")
demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"]
preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"]
redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"]
if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1):
raise ValueError("Incorrect number of data entries for non-insulation materials")
# Break out the individual material costs
# Since we don't know the exact wall construction, we take an average for demolition costs, since
# the cost will depend on the type of wall construction
demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data])
insulation_material_costs = material["material_cost"] * wall_area
preparation_material_costs = preparation_data[0]["material_cost"] * wall_area
redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area
demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data])
demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data])
insulation_labour_costs = material["labour_cost"] * wall_area
preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area
redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area
labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs +
preparation_labour_costs)
labour_costs = labour_costs * self.labour_adjustment_factor
materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs +
redecoration_material_costs)
subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs
contingency_cost = subtotal_before_profit * self.CONTINGENCY
preliminaries_cost = subtotal_before_profit * preliminaries_rate
profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
vat_cost = subtotal_before_vat * self.VAT_RATE
total_cost = subtotal_before_vat + vat_cost
demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data])
insulation_labour_hours = material["labour_hours_per_unit"] * wall_area
preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area
redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area
labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours +
preparation_labour_hours)
# Assume a team of 3-5 people for a small to medium size project
labour_days = (labour_hours / 8) / 4
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat_cost,
"contingency": contingency_cost,
"preliminaries": preliminaries_cost,
"material": materials_costs,
"profit": profit_cost,
"labour_hours": labour_hours,
"labour_days": labour_days,
"labour_cost": labour_costs
}

50
recommendations/FireplaceRecommendations.py Normal file
View file

@ -0,0 +1,50 @@
import pandas as pd
from BaseUtility import Definitions
from backend.Property import Property
class FireplaceRecommendations(Definitions):
"""
For properties that have open fireplaces, we recommend sealing the fireplaces
"""
# This is our base assumption for the cost of the work
COST_OF_WORK = 300
def __init__(
self,
property_instance: Property,
):
self.property = property_instance
self.has_ventilaion = None
self.recommendation = None
def recommend(self):
"""
Based on the number of open fireplcaes found, we recommend sealing each one at a cost of
around £500
:return:
"""
number_open_fireplaces = int(self.property.data["number-open-fireplaces"])
if number_open_fireplaces == 0:
return
estimated_cost = number_open_fireplaces * self.COST_OF_WORK
# We recommend installing two mechanical ventilation systems
self.recommendation = [
{
"parts": [],
"type": "sealing_open_fireplace",
"description": "Seal %s open fireplaces" % str(number_open_fireplaces),
"starting_u_value": None,
"new_u_value": None,
"sap_points": None,
"total": estimated_cost,
# Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal
"labour_hours": 6 * number_open_fireplaces
}
]

112
recommendations/FloorRecommendations.py
View file

@ -1,5 +1,8 @@
import math
from typing import List
import pandas as pd
from BaseUtility import Definitions
from datatypes.enums import QuantityUnits
from backend.Property import Property
@ -8,6 +11,7 @@ from recommendations.recommendation_utils import (
get_recommended_part, get_floor_u_value
)
from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
from recommendations.Costs import Costs
class FloorRecommendations(Definitions):
@ -30,31 +34,51 @@ class FloorRecommendations(Definitions):
materials: List,
):
self.property = property_instance
self.costs = Costs(self.property)
# For audit purposes, when estimating u values we'll store it
self.estimated_u_value = None
# Will contains a list of recommended measures
self.recommendations = []
self.materials = materials
self.suspended_floor_insulation_materials = [
part for part in materials if part["type"] == "suspended_floor_insulation"
]
self.suspended_floor_insulation_parts = [
part for part in self.materials if part["type"] == "suspended_floor_insulation"
self.suspended_floor_non_insulation_materials = [
part for part in materials if part["type"] in [
"suspended_floor_demolition", "suspended_floor_redecoration", "suspended_floor_vapour_barrier"
]
]
self.solid_floor_insulation_parts = [
part for part in self.materials if part["type"] == "solid_floor_insulation"
self.solid_floor_insulation_materials = [
part for part in materials if part["type"] == "solid_floor_insulation"
]
self.solid_floor_non_insulation_materials = [
part for part in materials if part["type"] in [
"solid_floor_demolition", "solid_floor_preparation", "solid_floor_vapour_barrier",
"solid_floor_redecoration"
]
]
self.exposed_floor_insulation_materials = [
part for part in materials if part["type"] == "exposed_floor_insulation"
]
# TODO: To be completed
self.exposed_floor_non_insulation_materials = []
def recommend(self):
u_value = self.property.floor["thermal_transmittance"]
is_suspended = self.property.floor["is_suspended"]
is_solid = self.property.floor["is_solid"]
floor_level = (
FLOOR_LEVEL_MAP[self.property.data["floor-level"]] if
self.property.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None
)
property_type = self.property.data["property-type"]
floor_area = self.property.insulation_floor_area
year_built = self.property.year_built
if self.property.floor["another_property_below"] | (self.property.floor["insulation_thickness"] in [
@ -81,7 +105,7 @@ class FloorRecommendations(Definitions):
u_value = get_floor_u_value(
floor_type=self.property.floor_type,
area=float(self.property.data["total-floor-area"]),
area=floor_area,
perimeter=self.property.perimeter,
age_band=self.property.age_band,
insulation_thickness=self.property.floor["insulation_thickness"],
@ -89,29 +113,51 @@ class FloorRecommendations(Definitions):
)
self.estimated_u_value = u_value
if is_suspended:
# Given the U-value, we recommend underfloor insulation
self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts)
if u_value < self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
return
if is_solid:
if self.property.floor["is_suspended"]:
# Given the U-value, we recommend underfloor insulation
self.recommend_floor_insulation(
u_value=u_value,
insulation_materials=self.suspended_floor_insulation_materials,
non_insulation_materials=self.suspended_floor_non_insulation_materials
)
return
if self.property.floor["is_solid"]:
# Given the U-value, we recommend solid floor insulation options which are usually solid foam
self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts)
self.recommend_floor_insulation(
u_value=u_value,
insulation_materials=self.solid_floor_insulation_materials,
non_insulation_materials=self.solid_floor_non_insulation_materials
)
return
if self.property.floor["is_to_unheated_space"] or self.property.floor["is_to_external_air"]:
self.recommend_floor_insulation(u_value=u_value, parts=self.exposed_floor_insulation_parts)
return
raise NotImplementedError("Implement me!")
@staticmethod
def _make_floor_description(part, depth):
return f"Install {depth}{part['depth_unit']} {part['description']} insulation"
def _make_floor_description(material):
return f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation"
def recommend_floor_insulation(self, u_value, parts):
def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials):
"""
This method is tasked with estimating the impact of performing suspended floor insulation
:return:
"""
lowest_selected_u_value = None
for part in parts:
for depth, cost_per_unit in zip(part["depths"], part["cost"]):
insulation_materials = pd.DataFrame(insulation_materials)
part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"])
lowest_selected_u_value = None
for _, insulation_material_group in insulation_materials.groupby("description"):
for _, material in insulation_material_group.iterrows():
part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
_, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
new_u_value = math.ceil(new_u_value * 100.0) / 100.0
@ -123,24 +169,36 @@ class FloorRecommendations(Definitions):
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
estimated_cost = cost_per_unit * self.property.floor_area
if material["type"] == "suspended_floor_insulation":
cost_result = self.costs.suspended_floor_insulation(
insulation_floor_area=self.property.insulation_floor_area,
material=material.to_dict(),
non_insulation_materials=non_insulation_materials
)
elif material["type"] == "solid_floor_insulation":
cost_result = self.costs.solid_floor_insulation(
insulation_floor_area=self.property.insulation_floor_area,
material=material.to_dict(),
non_insulation_materials=non_insulation_materials
)
else:
raise NotImplementedError("Implement me!")
self.recommendations.append(
{
"parts": [
get_recommended_part(
part=part,
selected_depth=depth,
quantity=self.property.floor_area,
part=material.to_dict(),
quantity=self.property.insulation_floor_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=estimated_cost
cost_result=cost_result
),
],
"type": "floor_insulation",
"description": self._make_floor_description(part, depth),
"description": self._make_floor_description(material),
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
"cost": estimated_cost,
**cost_result
}
)

309
recommendations/RoofRecommendations.py Normal file
View file

@ -0,0 +1,309 @@
import math
import pandas as pd
from backend.Property import Property
from typing import List
from datatypes.enums import QuantityUnits
from recommendations.recommendation_utils import (
get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns,
update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric
)
from recommendations.Costs import Costs
class RoofRecommendations:
# part L building regulations indicate that any rennovations on an existing property's roof should
# achieve a U-value of no higher than 0.16
# This can be seen in table 4.3 in building regulations part L:
# https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/1133079
# /Approved_Document_L__Conservation_of_fuel_and_power__Volume_1_Dwellings__2021_edition_incorporating_2023_amendments.pdf
BUILDING_REGULATIONS_PART_L_MAX_U_VALUE = 0.16
DIMINISHING_RETURNS_U_VALUE = 0.14
# It is recommended that lofts should have at least 270mm of insulation
MINIMUM_LOFT_ISULATION_MM = 270
def __init__(
self,
property_instance: Property,
materials: List
):
self.property = property_instance
self.costs = Costs(self.property)
# For audit purposes, when estimating u values we'll store it
self.estimated_u_value = None
# Will contains a list of recommended measures
self.recommendations = []
self.loft_insulation_materials = [
part for part in materials if part["type"] == "loft_insulation"
]
self.loft_non_insulation_materials = []
def recommend(self):
if self.property.roof["has_dwelling_above"]:
return
u_value = self.property.roof["thermal_transmittance"]
insulation_thickness = convert_thickness_to_numeric(
self.property.roof["insulation_thickness"],
self.property.roof["is_pitched"]
)
# We check if the roof is already insulated and if so, we exit
# Building regulations part L recommend installing at least 270mm of insulation, however generally we
# experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation
# This only holds true for pitched roofs
if (insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]:
return
# If we have a u-value already, need to implement this
if u_value:
if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
# The Roof is already compliant
return
if self.property.data["transaction-type"] == "new dwelling":
return
raise NotImplementedError("Implement me")
u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band})
self.estimated_u_value = u_value
if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
# The Roof is already compliant
return
if self.property.roof["is_pitched"] or self.property.roof["is_flat"]:
self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof)
return
if self.property.roof["is_roof_room"]:
self.recommend_room_roof_insulation(u_value)
return
raise NotImplementedError("Implement me")
@staticmethod
def make_loft_insulation_description(material):
return f"Install {int(material['depth'])}{material['depth_unit']} of {material['description']} in your loft"
@staticmethod
def make_room_roof_insulation_description(material, depth):
return f"Insulate your room roof with {depth}{material['depth_unit']} of {material['description']}"
@staticmethod
def make_flat_roof_insulation_description(material):
return (f"Insulate the home's flat roof "
f"with {int(material['depth'])}{material['depth_unit']} of {material['description']}")
def recommend_roof_insulation(
self, u_value, insulation_thickness, roof
):
"""
This method will recommend which insulation materials to use
This function handles both the case of loft insulation and flat roof insulation
We also follow advide provided in this article on the Energy Saving Trust website, providing
high level guidance around roof insulation:
https://energysavingtrust.org.uk/advice/roof-and-loft-insulation/
The process roughly looks like the following:
- Remove the Existing Weatherproof Layer: If the roof is being replaced, remove the old weatherproof layer to
expose the timber roof surface.
- Install Insulation Boards: Lay the rigid insulation boards directly on the timber roof surface.
Ensure the boards fit tightly together to prevent thermal bridging (heat loss through the gaps).
- Add a Vapour Control Layer (VCL): This is crucial to prevent moisture from entering the insulation layer,
which can lead to dampness and rot. The VCL is placed over the insulation.
- Install a New Weatherproof Layer: On top of the insulation and VCL, install a new weatherproof layer. This
could be traditional roofing materials like bitumen-based felt, rubber membranes like EPDM, or fiberglass.
:param u_value: U-value of the roof before any retrofit measures have been installed
:param insulation_thickness: Existing Insulation thickness of the loft
:param roof: dictionary describing the make-up of the roof
:return:
"""
# With loft insulation, 100mm goes between the joists and the rest is rolled on top
# Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle
# from the base layer
if roof["is_pitched"]:
insulation_materials = self.loft_insulation_materials
non_insulation_materials = self.loft_non_insulation_materials
elif roof["is_flat"]:
raise ValueError("UPDATE ME")
else:
raise ValueError("Roof is not pitched or flat")
if not insulation_materials:
raise ValueError("No roof insulation materials found")
insulation_materials = pd.DataFrame(insulation_materials)
lowest_selected_u_value = None
recommendations = []
for _, insulation_material_group in insulation_materials.groupby("description"):
for _, material in insulation_material_group.iterrows():
# We make sure we hit a depth of 270mm. We should factor in any existing insulation if the
# loft is already partially insulated.
# Note: This requirement is only for loft insulation
if ((material["depth"] + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]:
continue
part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
_, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
new_u_value = math.ceil(new_u_value * 100.0) / 100.0
# If I have a lowest U value and my new u value is higher than that but lower than the
# diminishing returns threshold, it can be considered
# If I have a lowest U value and my new u value is lower than the lowest value, it's
# further into the diminishing returns threshold and can shouldn't be
if is_diminishing_returns(
recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE
):
continue
# We allow a small tolerance for error so we don't discount the recommendation entirely
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
if material["type"] == "loft_insulation":
cost_result = self.costs.loft_insulation(
floor_area=self.property.insulation_floor_area,
material=material
)
description = self.make_loft_insulation_description(material)
elif material["type"] == "flat_roof_insulation":
description = self.make_flat_roof_insulation_description(material)
raise ValueError("COMPLETE ME")
else:
raise ValueError("Invalid material type")
recommendations.append(
{
"parts": [
get_recommended_part(
part=material.to_dict(),
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
cost_result=cost_result
)
],
"type": "roof_insulation",
"description": description,
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
**cost_result
}
)
self.recommendations = recommendations
def recommend_room_roof_insulation(self, u_value):
"""
This method recommends room in roof insulation for properties that have been identified
to possess a room in roof.
Because we currently have limited data about the construction of the roof, we make the following
assumptions:
1) The room in roof has a sloped roof.
We will make some basic estimations about the area of the roof given the floor area and the height of the
floors
2) Insulation of external walls is covered by the wall recommendation class
3) We assume a "Gable" roof type
Further, we recommend internal roof insulation for the room in roof
The following document contains details around best practices for insulating a room in roof
https://assets.publishing.service.gov.uk/media/61d727d18fa8f50594b59305/retrofit-room-in-roof-insulation-best
-practice.pdf
Of particular interest are the following:
We also follow advide provided in this article on the Energy Saving Trust website, providing
high level guidance around roof insulation:
https://energysavingtrust.org.uk/advice/roof-and-loft-insulation/
To insulate a warm loft, the following advice is given
"An alternative way to insulate your loft is to fit rigid insulation boards between and over the rafters.
Rafters are the sloping timbers that make up the roof itself."
To then insulate a room roof, the following recommendation is provided:
"If you want to use your loft as a living space, or it is already being used as a living space,
then you need to make sure that all the walls and ceilings between a heated room and an unheated space
are insulated.
- Sloping ceilings can be insulated in the same way as for a warm roof,
but with a layer of plasterboard on the inside of the insulation.
- Vertical walls can be insulated in the same way.
- Flat ceilings can be insulated like a standard loft.
:param u_value: Current u-value of the roof
:return:
"""
roof_roof_insulation_materials = [m for m in self.materials if m["type"] == "room_roof_insulation"]
if not roof_roof_insulation_materials:
raise ValueError("No room in roof insulation materials found")
if self.property.pitched_roof_area is None:
raise ValueError("pitched_roof_area not included as property attribute")
lowest_selected_u_value = None
recommendations = []
for material in roof_roof_insulation_materials:
for depth, cost_per_unit in zip(material["depths"], material["cost"]):
part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"])
_, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
new_u_value = math.ceil(new_u_value * 100.0) / 100.0
# If I have a lowest U value and my new u value is higher than that but lower than the
# diminishing returns threshold, it can be considered
# If I have a lowest U value and my new u value is lower than the lowest value, it's
# further into the diminishing returns threshold and can shouldn't be
if is_diminishing_returns(
recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE
):
continue
# We allow a small tolerance for error so we don't discount the recommendation entirely
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
estimated_cost = cost_per_unit * self.property.pitched_roof_area
recommendations.append(
{
"parts": [
get_recommended_part(
part=material,
selected_depth=depth,
quantity=self.property.pitched_roof_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=estimated_cost
)
],
"type": "roof_insulation",
"description": self.make_room_roof_insulation_description(material, depth),
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
"cost": estimated_cost,
}
)
self.recommendations = recommendations

72
recommendations/VentilationRecommendations.py Normal file
View file

@ -0,0 +1,72 @@
import pandas as pd
from BaseUtility import Definitions
from backend.Property import Property
class VentilationRecommendations(Definitions):
"""
For properties that do not have ventilation, we recommend installing ventilaion
This is particularly important for properties that have insulated walls and is also
crucial for prevent overheating risks in warmer months
"""
VENTILATION_DESCRIPTIONS = [
'mechanical, extract only',
'mechanical, supply and extract'
]
def __init__(
self,
property_instance: Property,
materials
):
self.property = property_instance
self.has_ventilaion = None
self.recommendation = None
self.materials = materials
def identify_ventilation(self):
self.has_ventilaion = self.property.data["mechanical-ventilation"] in self.VENTILATION_DESCRIPTIONS
def recommend(self):
"""
If there is no ventilation, we recommend installing ventilation
Generally, best practice is to install controlled ventilation for insulated walls so we still recommend
ventilation if there is natural ventilation
:return:
"""
self.identify_ventilation()
if self.has_ventilaion:
return
if len(self.materials) != 1:
raise NotImplementedError("Only handled the case of having one venilation option")
# We recommend installing 2 units
n_units = 2
part = self.materials.copy()
estimated_cost = n_units * part[0]["cost"]
part[0]["total"] = estimated_cost
part[0]["quantity"] = n_units
part[0]["quantity_unit"] = "part"
# We recommend installing two mechanical ventilation systems
self.recommendation = [
{
"parts": part,
"type": part[0]["type"],
"description": f"Install {n_units} {part[0]['description']} units",
"starting_u_value": None,
"new_u_value": None,
"sap_points": None,
"total": estimated_cost,
# We use a very simple and rough estimate of 4 hours per unit
"labour_hours": 4 * n_units
}
]

221
recommendations/WallRecommendations.py
View file

@ -1,7 +1,8 @@
import itertools
import math
from typing import List
import pandas as pd
from datatypes.enums import QuantityUnits
from backend.Property import Property
from BaseUtility import Definitions
@ -10,6 +11,7 @@ from recommendations.recommendation_utils import (
get_recommended_part, get_wall_u_value
)
from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
from recommendations.Costs import Costs
from utils.logger import setup_logger
logger = setup_logger()
@ -24,11 +26,17 @@ class WallRecommendations(Definitions):
# part L building regulations indicate that any rennovations on an existing property's walls should
# achieve a U-value of no higher than 0.3
# This can be seen in table 4.3 in building regulations part L:
# https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/1133079
# /Approved_Document_L__Conservation_of_fuel_and_power__Volume_1_Dwellings__2021_edition_incorporating_2023_amendments.pdf
BUILDING_REGULATIONS_PART_L_MAX_U_VALUE = 0.3
# We don't recommend measures that are too low because it becomes expensive, therefore we aim to avoid
# diminishing returns. This value should be verified with Osmosis (TODO)
DIMINISHING_RETURNS_U_VALUE = 0.25
# Building regulations part L also indicates that cavity wall insulation should result in 0.55 u-value
BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE = 0.55
# Part L regulations indicate that any new build should have walls that achieve a u-value of no higher
# than 0.18.
BUILDING_REGULATIONS_PART_L_NEW_BUILD_MAX_U_VALUE = 0.18
@ -45,13 +53,36 @@ class WallRecommendations(Definitions):
materials: List
):
self.property = property_instance
self.costs = Costs(self.property)
# For audit purposes, when estimating u values we'll store it
self.estimated_u_value = None
# Will contains a list of recommended measures
self.recommendations = []
self.materials = materials
self.cavity_wall_insulation_materials = [
part for part in materials if part["type"] == "cavity_wall_insulation"
]
self.internal_wall_insulation_materials = [
part for part in materials if part["type"] == "internal_wall_insulation"
]
self.internal_wall_non_insulation_materials = [
part for part in materials if part["type"] in [
"iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"
]
]
self.external_wall_insulation_materials = [
part for part in materials if part["type"] == "external_wall_insulation"
]
self.external_wall_non_insulation_materials = [
part for part in materials if part["type"] in [
"ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"
]
]
@property
def ewi_valid(self):
@ -112,15 +143,6 @@ class WallRecommendations(Definitions):
self.estimated_u_value = u_value
if self.property.walls["is_solid_brick"]:
if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
self.find_insulation(u_value)
return
# If the u-value is within regulations, we don't do anything
return
if is_cavity_wall:
if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
# Test filling cavity
@ -128,9 +150,13 @@ class WallRecommendations(Definitions):
return
logger.error("Not implemented yet")
# Remaining wall types are treated with IWI or EWI
if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
self.find_insulation(u_value)
return
# If the u-value is within regulations, we don't do anything
return
# NotImplementedError("Not implemented yet")
def find_cavity_insulation(self, u_value, insulation_thickness):
"""
@ -154,7 +180,7 @@ class WallRecommendations(Definitions):
filled cavity wall
"""
cavity_wall_fills = [m for m in self.materials if m["type"] == "cavity_wall_insulation"]
insulation_materials = pd.DataFrame(self.cavity_wall_insulation_materials)
cavity_width = 75
if insulation_thickness == "below average":
cavity_width = cavity_width * (1 - PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION)
@ -162,8 +188,9 @@ class WallRecommendations(Definitions):
# Test the different fill options
lowest_selected_u_value = None
recommendations = []
for part in cavity_wall_fills:
part_u_value = r_value_per_mm_to_u_value(cavity_width, part["r_value_per_mm"])
for _, material in insulation_materials.iterrows():
part_u_value = r_value_per_mm_to_u_value(cavity_width, material["r_value_per_mm"])
_, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
new_u_value = math.ceil(new_u_value * 100.0) / 100.0
@ -173,41 +200,44 @@ class WallRecommendations(Definitions):
):
continue
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
estimated_cost = part["cost"] * self.property.insulation_wall_area
cost_result = self.costs.cavity_wall_insulation(
wall_area=self.property.insulation_wall_area,
material=material.to_dict(),
)
recommendations.append(
{
"parts": [
get_recommended_part(
part=part,
selected_depth=None,
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=estimated_cost
)
],
"type": "wall_insulation",
"description": f"Fill cavity with {part['description']}",
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
"cost": estimated_cost,
}
)
recommendations.append(
{
"parts": [
get_recommended_part(
part=material.to_dict(),
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
cost_result=cost_result
)
],
"type": "wall_insulation",
"description": f"Fill cavity with {material['description']}",
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
**cost_result
}
)
self.recommendations = recommendations
def _find_insulation(self, parts, u_value):
def _find_insulation(self, u_value, insulation_materials, non_insulation_materials):
lowest_selected_u_value = None
recommendations = []
for part in parts:
for _, insulation_material_group in insulation_materials.groupby("description"):
for depth, cost_per_unit in zip(part["depths"], part["cost"]):
part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"])
for _, material in insulation_material_group.iterrows():
part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
_, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
new_u_value = math.ceil(new_u_value * 100.0) / 100.0
@ -224,27 +254,40 @@ class WallRecommendations(Definitions):
# We allow a small tolerance for error so we don't discount the recommendation entirely
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
estimated_cost = cost_per_unit * self.property.insulation_wall_area
if material["type"] == "internal_wall_insulation":
cost_result = self.costs.internal_wall_insulation(
wall_area=self.property.insulation_wall_area,
material=material.to_dict(),
non_insulation_materials=non_insulation_materials
)
elif material["type"] == "external_wall_insulation":
cost_result = self.costs.external_wall_insulation(
wall_area=self.property.insulation_wall_area,
material=material.to_dict(),
non_insulation_materials=non_insulation_materials
)
else:
raise ValueError("Invalid material type")
recommendations.append(
{
"parts": [
get_recommended_part(
part=part,
selected_depth=depth,
part=material.to_dict(),
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=estimated_cost
cost_result=cost_result
)
],
"type": "wall_insulation",
"description": "Install " + self._make_description(part, depth),
"description": "Install " + self._make_description(material),
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
"cost": estimated_cost,
**cost_result
}
)
@ -257,84 +300,32 @@ class WallRecommendations(Definitions):
:return:
"""
ewi_parts = [
part for part in self.materials if part["type"] == "external_wall_insulation"
] if self.ewi_valid else []
iwi_parts = [part for part in self.materials if part["type"] == "internal_wall_insulation"]
# Recommend external and internal wall insulation separately
# Since external and internal wall insulation are sufficiently different,
# we separate the logic for for recommending them, therefore we don't
# consider diminishing returns between the two
ewi_recommendations = self._find_insulation(ewi_parts, u_value)
iwi_recommendations = self._find_insulation(iwi_parts, u_value)
ewi_recommendations = []
if self.ewi_valid:
ewi_recommendations = self._find_insulation(
u_value=u_value,
insulation_materials=pd.DataFrame(self.external_wall_insulation_materials),
non_insulation_materials=self.external_wall_non_insulation_materials
)
iwi_recommendations = self._find_insulation(
u_value=u_value,
insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials),
non_insulation_materials=self.internal_wall_non_insulation_materials
)
self.recommendations += ewi_recommendations + iwi_recommendations
# We also can recommend both internal and external wall insulation together
# By looping through ewi first, if there is nothing there, that ensures not combinations are tested
for ewi_part in ewi_parts:
for iwi_part in iwi_parts:
for (ewi_depth, ewi_cost_per_unit), (iwi_depth, iwi_cost_per_unit) in itertools.product(
zip(ewi_part["depths"], ewi_part["cost"]),
zip(iwi_part["depths"], iwi_part["cost"])
):
ewi_part_u_value = r_value_per_mm_to_u_value(ewi_depth, ewi_part["r_value_per_mm"])
iwi_part_u_value = r_value_per_mm_to_u_value(iwi_depth, iwi_part["r_value_per_mm"])
# First calculate the new U-value after applying external wall insulation
_, ewi_new_u_value = calculate_u_value_uplift(u_value, ewi_part_u_value)
# Then calculate the new U-value after applying internal wall insulation
_, combined_new_u_value = calculate_u_value_uplift(ewi_new_u_value, iwi_part_u_value)
combined_new_u_value = round(combined_new_u_value, 2)
if combined_new_u_value < self.DIMINISHING_RETURNS_U_VALUE:
# We don't recommend an overkill solution
continue
# Check if the combined new U-value meets the requirement
if combined_new_u_value - self.U_VALUE_ERROR <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
# Here you might want to define a way to add both recommendations together.
# For now, I'm adding them as separate items in the list
ewi_esimtated_cost = ewi_cost_per_unit * self.property.insulation_wall_area
iwi_esimtated_cost = iwi_cost_per_unit * self.property.insulation_wall_area
recommendation = {
"parts": [
get_recommended_part(
part=ewi_part,
selected_depth=ewi_depth,
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=ewi_esimtated_cost
),
get_recommended_part(
part=iwi_part,
selected_depth=iwi_depth,
quantity=self.property.insulation_wall_area,
quantity_unit=QuantityUnits.m2.value,
selected_total_cost=iwi_esimtated_cost
)
],
"type": "wall_insulation",
"description": (
"Install " + self._make_description(ewi_part, ewi_depth) + " and " +
self._make_description(iwi_part, iwi_depth)
),
"starting_u_value": u_value,
"new_u_value": combined_new_u_value,
"sap_points": None,
"cost": ewi_esimtated_cost + iwi_esimtated_cost,
}
self.recommendations.append(recommendation)
self.prune_diminishing_recommendations()
@staticmethod
def _make_description(part, depth):
return f"{depth}{part['depth_unit']} {part['description']}"
def _make_description(material):
return f"{int(material['depth'])}{material['depth_unit']} {material['description']}"
def prune_diminishing_recommendations(self):
# For any recommendations, if we have at least 1 reommendation that does not exhibit diminishing returns

2
recommendations/optimiser/optimiser_functions.py
View file

@ -22,7 +22,7 @@ def prepare_input_measures(property_recommendations, goal):
[
{
"id": rec["recommendation_id"],
"cost": rec["cost"],
"cost": rec["total"],
"gain": rec[goal_key],
"type": rec["type"]
}

28
recommendations/rdsap_tables.py
View file

@ -463,6 +463,34 @@ s11_list = [
table_s11 = pd.DataFrame(s11_list)
########################################################################################################################
# Table s12 is used for assigning the u-values of floors to unheated spaces or external air
# which can be found on page 26 of the BRE document, section 5.6
# https://bregroup.com/wp-content/uploads/2019/09/RdSAP_2012_9.94-20-09-2019.pdf
#
# the insulation_{thickness} fields indicate the u-value at that insulation thickness
########################################################################################################################
s12_list = [
{"age_band": "A", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "B", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "C", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "D", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "E", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "F", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "G", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "H", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "I", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22},
{"age_band": "J", "insulation_0": 0.25, "insulation_50": 0.25, "insulation_100": 0.25, "insulation_150": 0.22},
{"age_band": "K", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22},
{"age_band": "L", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22},
]
table_s12 = pd.DataFrame(s12_list)
########################################################################################################################
#

177
recommendations/recommendation_utils.py
View file

@ -1,11 +1,12 @@
import math
from copy import deepcopy
import numpy as np
import pandas as pd
from recommendations.rdsap_tables import (
epc_wall_description_map, wall_uvalues_df, default_wall_thickness, table_s9 as s9, table_s10 as s10,
table_s11 as s11
table_s11 as s11, table_s12 as s12
)
from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION, PARTIAL_CAVITY_DESCRIPTIONS
@ -108,22 +109,21 @@ def update_lowest_selected_u_value(lowest_selected_u_value, new_u_value):
return lowest_selected_u_value
def get_recommended_part(part, selected_depth, selected_total_cost, quantity, quantity_unit):
def get_recommended_part(part, cost_result, quantity, quantity_unit):
"""
Utility function to return a recommended part with the selected depth.
:param part: part to be recommended
:param selected_depth: depth of the selected part
:param selected_total_cost: Total cost of the selected part
:param cost_result: Total cost of the selected part, as returned by the Cost class
:param quantity: Quantity of the selected part
:param quantity_unit: Unit of the quantity
:return:
"""
recommended_part = deepcopy(part)
recommended_part["depths"] = [selected_depth]
recommended_part["estimated_cost"] = selected_total_cost
recommended_part["quantity"] = quantity
recommended_part["quantity_unit"] = quantity_unit
recommended_part.update(cost_result)
return recommended_part
@ -340,6 +340,34 @@ def estimate_perimeter(floor_area, num_rooms):
return perimeter
def get_exposed_floor_uvalue(insulation_thickness_str, age_band):
"""
We implement the methodology as defined in section 5.6 and table S12 of the RdSAP document
:param insulation_thickness_str:
:return:
"""
unknown_insulation_age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I"]
# As directed by the documentation, if the insulation thickness is not known, we assume it's
# 50mm for these age bands
if insulation_thickness_str in ["below average", "average", "above average"] and (
age_band in unknown_insulation_age_bands
):
insulation_thickness = 50
elif insulation_thickness_str in ["none", None]:
insulation_thickness = 0
elif insulation_thickness_str == "below average":
insulation_thickness = 50
elif insulation_thickness_str == "average":
insulation_thickness = 100
elif insulation_thickness_str == "above average":
insulation_thickness = 150
else:
insulation_thickness = int(insulation_thickness_str.replace("mm", ""))
return s12[s12["age_band"] == age_band][f"insulation_{insulation_thickness}"].values[0]
def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None):
"""
Estimate the u-value of a suspended floor, based on RdSap methodology
@ -372,6 +400,12 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati
0.701
"""
if floor_type == "exposed_floor":
# In this case, we extract the u-value from table s12
# See section 5.6 of the RdSAP document for more details
# https://bregroup.com/wp-content/uploads/2019/09/RdSAP_2012_9.94-20-09-2019.pdf
return get_exposed_floor_uvalue(insulation_thickness, age_band)
# Cleans our regularly inputted insulation thickness for usage in this function
insulation_thickness = extract_insulation_thickness(insulation_thickness)
@ -492,28 +526,121 @@ def get_wall_type(
return None
def estimate_floors(floor_area, num_rooms):
def estimate_external_wall_area(num_floors, floor_height, perimeter, built_form):
"""
Simple utility funciton, which assuming a 15m squared room, estimates the number of floors in a property
:param floor_area: Gross floor area of a property
:param num_rooms: Number of rooms in a property
:return: Number of floors in a property
This method estimates the external wall area based on fundamental assumptions about the home
:param num_floors: Number of floors in the building.
:param floor_height: Height of one floor in meters.
:param perimeter: Total perimeter of the building on one floor in meters.
:param built_form: The built form of the property. This is used to determine the number of exposed walls.
:return:
"""
# Estimate total room area
total_room_area = num_rooms * 15
# Estimate the number of floors
floors = floor_area / total_room_area
# Round up to the nearest whole number
floors = round(floors)
return floors
def estimate_wall_area(num_floors, floor_height, perimeter):
wall_area_one_floor = perimeter * floor_height
total_wall_area = wall_area_one_floor * num_floors
return total_wall_area
number_exposed_walls = {
'End-Terrace': 3,
'Mid-Terrace': 2,
'Semi-Detached': 3,
'Detached': 4,
}
exposed_wall_area = total_wall_area * (number_exposed_walls[built_form] / 4)
return exposed_wall_area
def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK):
"""
# Calculate R-value (thermal resistance) using the formula: R = thickness / thermal_conductivity
# Note: The thickness should be converted to meters for the units to be consistent.
:param thickness_mm:
:param thermal_conductivity_w_mK:
:return:
"""
if thermal_conductivity_w_mK is None:
return None
r_value_m2k_w = (thickness_mm / 1000) / thermal_conductivity_w_mK
# Calculate R-value per mm
r_value_per_mm = r_value_m2k_w / thickness_mm
return r_value_per_mm
def convert_thickness_to_numeric(string_thickness, is_pitched):
"""
Roof insulation thickness could be a string like "None", "300mm+" or a numeric string.
This function will convert these strings to a number for easy usage
we handle loft insulation differently to flat roof or room in roof insulation, since for loft insulation,
we are presented with an insulation thickness, whereas for the other forms of roof, we are just told whether or not
the roof is insulated or not.
:param string_thickness: string measure of insulation thickness
:param is_pitched: boolean indicating if the roof is a pitched roof
:return: integer measure of insulation thickness
"""
if string_thickness is None:
return 0
if is_pitched:
lookup = {
"none": 0,
"below average": 50,
"average": 100,
"above average": 270
}
else:
lookup = {
"none": 0,
"below average": 100,
"average": 270,
"above average": 270
}
mapped = lookup.get(string_thickness)
if mapped is not None:
return mapped
if "+" in string_thickness:
return int(string_thickness.replace("+", ""))
return int(string_thickness)
def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float:
"""
This function will estimate the area of a pitched roof, given the floor area below the roof and the floor
height of the property.
Given limited information about the home, this is a very rough method to estimate the roof area and we
assume the the room is a gable roof.
We assume a roughly average pitch of 45 degrees
Note that both floor area and height should be in the same units. E.g. if floor area is meters squared,
floor height should be in meters
:param floor_area: area of the home's floor
:param floor_height: height of the home's floors
:return: Numerical estimate of the surface area of the top of the pitched roof
"""
# We estimate the length of the wall by just modelling the house as a square
wall_width = np.sqrt(floor_area)
# We're modelling the roof as two triangles where we know two of the three sides.
# The floor height makes up one side and half of the wall width makes up the other side
slope = np.sqrt(np.square(wall_width / 2) + np.square(floor_height))
area = 2 * (slope * wall_width)
return area

411
recommendations/tests/test_costs.py Normal file
View file

@ -0,0 +1,411 @@
from recommendations.Costs import Costs
from unittest.mock import Mock
class TestCosts:
def test_cavity_wall_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire"
}
costs = Costs(mock_property)
cwi_material = {
"description": "cwi",
"depth": 75,
"thermal_conductivity": 0.037,
"prime_cost": 5.17,
"material_cost": 5.62,
"labour_cost": 1.125,
"labour_hours": 0.065
}
cwi_results = costs.cavity_wall_insulation(
wall_area=95.9104281347967,
material=cwi_material,
)
assert cwi_results == {'total': 1027.0280465530302, 'subtotal': 855.8567054608585, 'vat': 171.1713410921717,
'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626,
'material': 539.0166061175574, 'profit': 95.09518949565093,
'labour_hours': 6.234177828761786, 'labour_cost': 94.95132385344874}
def test_loft_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire"
}
costs = Costs(mock_property)
loft_material = {
"description": "Crown Loft Roll 44 glass fibre roll",
"depth": 270,
"thermal_conductivity": 0.044,
"prime_cost": None,
"material_cost": 5.91938,
"labour_cost": 1.96,
"labour_hours": 0.11
}
loft_results = costs.loft_insulation(
floor_area=33.5,
material=loft_material,
)
assert loft_results == {'total': 414.8496486, 'subtotal': 345.70804050000004, 'vat': 69.14160810000001,
'contingency': 25.608003000000004, 'preliminaries': 25.608003000000004,
'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685,
'labour_cost': 57.7808}
def test_internal_wall_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire"
}
costs = Costs(mock_property)
iwi_non_insulation_materials = [
{'type': 'iwi_wall_demolition',
'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.',
'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27,
'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0},
{'type': 'iwi_wall_demolition',
'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim',
'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23,
'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0},
{'type': 'iwi_wall_demolition',
'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and '
'plaster',
'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85,
'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0},
{'Notes': "",
'cost_unit': "",
'depth': "",
'depth_unit': "",
'description': 'Visqueen High Performance Vapour Barrier',
'labour_cost': 0.48,
'labour_hours_per_unit': 0.02,
'link': 'SPONs',
'material_cost': 1.21,
'plant_cost': 0,
'prime_material_cost': 0.58,
'thermal_conductivity': "",
'thermal_conductivity_unit': "",
'total_cost': 1.69,
'type': 'iwi_vapour_barrier'},
{'Notes': "",
'cost_unit': "",
'depth': "",
'depth_unit': "",
'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work '
'to walls or ceilings; one coat; to plasterboard base; over 600mm wide',
'labour_cost': 6.58,
'labour_hours_per_unit': 0.25,
'link': "",
'material_cost': 0.06,
'plant_cost': 0,
'prime_material_cost': 0.0,
'thermal_conductivity': "",
'thermal_conductivity_unit': "",
'total_cost': 6.64,
'type': 'iwi_redecoration'},
{'Notes': "",
'cost_unit': "",
'depth': "",
'depth_unit': "",
'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - '
'5m high',
'labour_cost': 0.0,
'labour_hours_per_unit': 0.21,
'link': "",
'material_cost': 0.41,
'plant_cost': 0,
'prime_material_cost': "",
'thermal_conductivity': "",
'thermal_conductivity_unit': "",
'total_cost': 4.34,
'type': 'iwi_redecoration'},
{'Notes': "",
'cost_unit': "",
'depth': "",
'depth_unit': "",
'description': 'Fitting existing softwood skirting or architrave to new '
'frames; 150mm high',
'labour_cost': 4.87,
'labour_hours_per_unit': 0.01,
'link': "",
'material_cost': 4.86,
'plant_cost': 0,
'prime_material_cost': "",
'thermal_conductivity': "",
'thermal_conductivity_unit': "",
'total_cost': 4.88,
'type': 'iwi_redecoration'}
]
iwi_material = {
"type": "internal_wall_insulation",
"description": "Ecotherm Eco-Versal PIR Insulation Board",
"depth": 150,
"depth_unit": "mm",
"cost_unit": "gbp_per_m2",
"thermal_conductivity": 0.022,
"thermal_conductivity_unit": "watt_per_meter_kelvin",
"prime_material_cost": "",
"material_cost": 11.68,
"labour_cost": 3.12,
"labour_hours_per_unit": 0.18,
"plant_cost": "",
"total_cost": 14.8,
"link": "SPONs"
}
iwi_results = costs.internal_wall_insulation(
wall_area=95.9104281347967,
material=iwi_material,
non_insulation_materials=iwi_non_insulation_materials
)
assert iwi_results == {'total': 6421.5484411659245, 'subtotal': 5351.29036763827, 'vat': 1070.258073527654,
'contingency': 573.3525393898148, 'preliminaries': 382.2350262598765,
'material': 1747.488000615996, 'profit': 573.3525393898148,
'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405,
'labour_cost': 1927.1602026551818}
def test_suspended_floor_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire"
}
costs = Costs(mock_property)
sus_floor_material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll',
'depth': 140.0,
'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039,
'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0,
'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
'plant_cost': 0,
'total_cost': 13.46, 'link': 'SPONs',
'Notes': 'Spons did not contain labour costs so we use values for similar insulations. '
'We use the '
'same values as in Crown loft roll 44, since it is also an insulation roll'}
sus_floor_non_insulation_materials = [
{'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11,
'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs',
'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and '
'therefore there is no need for a skip'},
{'type': 'suspended_floor_demolition',
'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3,
'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0},
{'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0,
'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48,
'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0},
{'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside',
'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0,
'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98,
'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0},
{'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0,
'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37,
'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs',
'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; '
'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, '
'therefore we need just labour rates'}]
sus_floor_results = costs.suspended_floor_insulation(
insulation_floor_area=33.5,
material=sus_floor_material,
non_insulation_materials=sus_floor_non_insulation_materials
)
assert sus_floor_results == {
'total': 3003.366924, 'subtotal': 2502.80577, 'vat': 500.561154,
'contingency': 185.39302, 'preliminaries': 185.39302, 'material': 483.405,
'profit': 278.08952999999997, 'labour_hours': 54.940000000000005,
'labour_days': 2.289166666666667, 'labour_cost': 1370.5252
}
def test_solid_floor_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire"
}
costs = Costs(mock_property)
sol_floor_material = {
'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033,
'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0,
'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0,
'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0
}
sol_floor_non_insulation_materials = [
{'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11,
'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs',
'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and '
'therefore there is no need for a skip'},
{'type': 'solid_floor_preparation',
'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14,
'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation',
'description': 'Clean out crack to '
'form a 20mm×20mm '
'groove and fill with '
'cement: mortar mixed '
'with bonding agent',
'depth': 0, 'depth_unit': 0,
'cost_unit': 0,
'thermal_conductivity': 0,
'thermal_conductivity_unit': 0,
'prime_material_cost': 0,
'material_cost': 6.91,
'labour_cost': 18.99,
'labour_hours_per_unit': 0.61,
'plant_cost': 0.16,
'total_cost': 26.06, 'link': 0,
'Notes': 'This step is the '
'assessment and repair of '
'any damage to the concrete '
'floor such as filling '
'cracks or levelling uneven '
'areas'},
{'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0,
'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48,
'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0},
{'type': 'solid_floor_redecoration',
'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs',
'Notes': 'This is the screed layer, placed on top of the insulation'},
{'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0,
'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37,
'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs',
'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; '
'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, '
'therefore we need just labour rates'},
{'type': 'solid_floor_redecoration',
'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0,
'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12,
'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0}
]
sol_floor_results = costs.solid_floor_insulation(
insulation_floor_area=33.5,
material=sol_floor_material,
non_insulation_materials=sol_floor_non_insulation_materials
)
assert sol_floor_results == {
'total': 3962.021952, 'subtotal': 3301.68496, 'vat': 660.336992, 'contingency': 353.75196,
'preliminaries': 235.83464, 'material': 1006.3399999999999, 'profit': 353.75196, 'labour_hours': 57.285,
'labour_days': 2.386875, 'labour_cost': 1346.6464
}
def test_external_wall_insulation(self):
mock_property = Mock()
mock_property.data = {
"county": "Northamptonshire",
"property-type": "House",
"built-form": 'End-Terrace'
}
costs = Costs(mock_property)
ewi_material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022,
'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53,
'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0,
'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0}
ewi_non_insulation_materials = [
{'type': 'ewi_wall_demolition',
'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping '
'hammer; plaster to walls.',
'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2',
'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27,
'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55,
'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition',
'description': 'Stud walls: Remove wall linings '
'including battening behind; '
'plasterboard and skim',
'depth': 0, 'depth_unit': 0,
'cost_unit': 'gbp_per_m2',
'thermal_conductivity': 0,
'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0,
'labour_cost': 6.23, 'labour_hours_per_unit': 0.2,
'plant_cost': 1.25, 'total_cost': 7.48,
'link': 'SPONs', 'Notes': 0},
{'type': 'ewi_wall_demolition',
'description': 'Lathe and Plaster walls: Remove wall linings including battening '
'behind; wood lath and plaster',
'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2',
'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85,
'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94,
'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation',
'description': 'Clean and prepare surfaces, '
'one coat Keim dilution, '
'one coat primer and two coats '
'of Keim Ecosil paint; Brick or '
'block walls; over 300 mm girth',
'depth': 0, 'depth_unit': 0, 'cost_unit': 0,
'thermal_conductivity': 0,
'thermal_conductivity_unit': 0,
'prime_material_cost': 0, 'material_cost': 7.3,
'labour_cost': 5.62, 'labour_hours_per_unit': 0.3,
'plant_cost': 0, 'total_cost': 12.92,
'link': 'SPONs',
'Notes': 'This work covers the preparation and '
'priming of the wall before insulating'},
{'type': 'ewi_wall_redecoration',
'description': 'EPS insulation fixed with adhesive to SFS structure (measured '
'separately) with horizontal PVC intermediate track and vertical '
'T-spines; with glassfibre mesh reinforcement embedded in Sto '
'Armat Classic Basecoat Render and Stolit K 1.5 Decorative '
'Topcoat Render (white)',
'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0,
'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0,
'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0,
'total_cost': 69.94, 'link': 'SPONs',
'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a '
'cost of 99.17 per meter square. This includes the cost of insulation. '
'To get the costing for just the works and not the insulation, '
'we subtract the cost of EPS insulation, using Ravathem 75mm insulation '
'as an example, which costs £29.23 per meter square, giving us the cost '
'of the remaining works without insulation. This material gives us a '
'cost for basecoat, mesh application and a render finish'}]
ewi_results = costs.external_wall_insulation(
wall_area=95.9104281347967,
material=ewi_material,
non_insulation_materials=ewi_non_insulation_materials
)
assert ewi_results == {
'total': 13590.909723215433, 'subtotal': 11325.758102679527, 'vat': 2265.1516205359053,
'contingency': 808.9827216199662, 'preliminaries': 1213.4740824299492,
'material': 4020.565147410677, 'profit': 1213.4740824299492,
'labour_hours': 187.02533486285358, 'labour_days': 5.8445417144641745,
'labour_cost': 3921.5600094613983
}

8
recommendations/tests/test_data/wall_uvalue_test_cases.py
View file

@ -4,21 +4,21 @@ wall_uvalue_test_cases = [
"age_band": "A",
"is_granite_or_whinstone": False,
"is_sandstone_or_limestone": False,
"uvalue": 0.7
"uvalue": 1.3
},
{
"clean_description": "Cavity wall, as built, partial insulation",
"age_band": "F",
"is_granite_or_whinstone": False,
"is_sandstone_or_limestone": False,
"uvalue": 0.4
"uvalue": 0.85
},
{
"clean_description": "Cavity wall, as built, partial insulation",
"age_band": "F",
"age_band": "G",
"is_granite_or_whinstone": False,
"is_sandstone_or_limestone": False,
"uvalue": 0.4
"uvalue": 0.5375
},
{

58
recommendations/tests/test_fireplace_recommendations.py Normal file
View file

@ -0,0 +1,58 @@
from backend.Property import Property
from unittest.mock import Mock
from recommendations.FireplaceRecommendations import FireplaceRecommendations
class TestFirepaceRecommendations:
def test_no_fireplaces(self):
property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance.data = {
"number-open-fireplaces": 0
}
recommender = FireplaceRecommendations(
property_instance=property_instance
)
assert recommender.recommendation is None
recommender.recommend()
assert recommender.recommendation is None
def test_one_fireplace(self):
property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance.data = {
"number-open-fireplaces": 1
}
recommender = FireplaceRecommendations(
property_instance=property_instance
)
assert recommender.recommendation is None
recommender.recommend()
assert recommender.recommendation
assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
assert recommender.recommendation[0]["cost"] == 300
def test_multiple_fireplaces(self):
property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance.data = {
"number-open-fireplaces": 3
}
recommender = FireplaceRecommendations(
property_instance=property_instance
)
assert recommender.recommendation is None
recommender.recommend()
assert recommender.recommendation
assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
assert recommender.recommendation[0]["cost"] == 900

154
recommendations/tests/test_floor_recommendations.py
View file

@ -3,6 +3,7 @@ import pytest
import os
from unittest.mock import Mock
from recommendations.FloorRecommendations import FloorRecommendations
from backend.Property import Property
# with open(
# os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
@ -67,7 +68,23 @@ solid_floor_insulation_parts = [
]
parts = suspended_floor_insulation_parts + solid_floor_insulation_parts
exposed_floor_insulation_parts = [
{
"type": "exposed_floor_insulation",
"description": "Rockwool Stone Wool insulation",
"depths": [50, 100, 140],
"depth_unit": "mm",
"cost": [8, 11, 15],
"cost_unit": "gbp_sq_meter",
"r_value_per_mm": 0.026315789473684213,
"r_value_unit": "square_meter_kelvin_per_watt",
"thermal_conductivity": 0.038,
"thermal_conductivity_unit": "watt_per_meter_kelvin",
"link": "https://insulation4less.co.uk/products/rockwool-flexi-slab-all-sizes?variant=33409590853685"
},
]
parts = suspended_floor_insulation_parts + solid_floor_insulation_parts + exposed_floor_insulation_parts
class TestFloorRecommendations:
@ -98,6 +115,8 @@ class TestFloorRecommendations:
assert obj.property
def test_other_premises_below(self, input_properties):
input_properties[0].floor_area = 100
input_properties[0].number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_properties[0],
materials=parts
@ -118,6 +137,8 @@ class TestFloorRecommendations:
input_properties[2].age_band = "A"
input_properties[2].perimeter = 20
input_properties[2].wall_type = "solid brick"
input_properties[2].floor_type = "suspended"
input_properties[2].number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_properties[2],
@ -126,7 +147,7 @@ class TestFloorRecommendations:
assert recommender.estimated_u_value is None
recommender.recommend()
assert recommender.property.floor["is_suspended"]
assert recommender.estimated_u_value == 0.39
assert recommender.estimated_u_value == 0.66
assert recommender.recommendations
types = {part["type"] for x in recommender.recommendations for part in x["parts"]}
@ -139,6 +160,8 @@ class TestFloorRecommendations:
does not need floor insulation
:return:
"""
input_properties[3].floor_area = 100
input_properties[3].number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_properties[3],
materials=parts
@ -160,6 +183,8 @@ class TestFloorRecommendations:
input_properties[4].age_band = "B"
input_properties[4].perimeter = 50
input_properties[4].wall_type = "solid brick"
input_properties[4].floor_type = "solid"
input_properties[4].number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_properties[4],
@ -169,7 +194,7 @@ class TestFloorRecommendations:
recommender.recommend()
assert not recommender.property.floor["is_suspended"]
assert recommender.property.floor["is_solid"]
assert recommender.estimated_u_value == 0.71
assert recommender.estimated_u_value == 0.73
assert recommender.recommendations
types = {part["type"] for x in recommender.recommendations for part in x["parts"]}
@ -181,6 +206,8 @@ class TestFloorRecommendations:
This is another description we see when there is a property below
"""
input_properties[6].floor_area = 100
input_properties[6].number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_properties[6],
materials=parts
@ -191,3 +218,124 @@ class TestFloorRecommendations:
assert not recommender.property.floor["is_solid"]
assert recommender.estimated_u_value is None
assert not recommender.recommendations
def test_exposed_floor_no_insulation(self):
input_property = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property.floor = {
'original_description': 'To unheated space, no insulation (assumed)',
'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
'insulation_thickness': 'none'
}
input_property.age_band = "L"
input_property.set_floor_type()
input_property.data = {"floor-level": 0, "property-type": "House"}
input_property.floor_area = 100
input_property.number_of_floors = 1
recommender = FloorRecommendations(
property_instance=input_property,
materials=exposed_floor_insulation_parts
)
assert not recommender.recommendations
recommender.recommend()
# Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation
assert not len(recommender.recommendations)
assert recommender.estimated_u_value == 0.22
# Now with an older age band
input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property2.floor = {
'original_description': 'To unheated space, no insulation (assumed)',
'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
'insulation_thickness': 'none'
}
input_property2.age_band = "D"
input_property2.set_floor_type()
input_property2.data = {"floor-level": 0, "property-type": "House"}
input_property2.floor_area = 100
input_property2.number_of_floors = 1
recommender2 = FloorRecommendations(
property_instance=input_property2,
materials=exposed_floor_insulation_parts
)
assert not recommender2.recommendations
recommender2.recommend()
assert len(recommender2.recommendations) == 1
assert recommender2.recommendations[0]["new_u_value"] == 0.23
assert recommender2.recommendations[0]["starting_u_value"] == 1.2
assert recommender2.recommendations[0]["cost"] == 1500
def test_exposed_floor_below_average_insulated(self):
input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property3.floor = {
'original_description': 'To unheated space, below average insulation (assumed)',
'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
'insulation_thickness': 'below average'
}
input_property3.age_band = "C"
input_property3.set_floor_type()
input_property3.data = {"floor-level": 0, "property-type": "House"}
input_property3.floor_area = 100
input_property3.number_of_floors = 1
recommender3 = FloorRecommendations(
property_instance=input_property3,
materials=exposed_floor_insulation_parts
)
assert not recommender3.recommendations
recommender3.recommend()
assert recommender3.estimated_u_value == 0.5
assert len(recommender3.recommendations) == 1
assert recommender3.recommendations[0]["new_u_value"] == 0.22
assert recommender3.recommendations[0]["starting_u_value"] == 0.5
assert recommender3.recommendations[0]["cost"] == 1100
assert recommender3.recommendations[0]["parts"][0]["depths"] == [100]
# With average insulation, no recommendations
input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property4.floor = {
'original_description': 'To unheated space, insulated (assumed)',
'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None,
'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
'insulation_thickness': 'average'
}
input_property4.age_band = "C"
input_property4.set_floor_type()
input_property4.data = {"floor-level": 0, "property-type": "House"}
input_property4.floor_area = 100
input_property4.number_of_floors = 1
recommender4 = FloorRecommendations(
property_instance=input_property4,
materials=exposed_floor_insulation_parts
)
assert not recommender4.recommendations
recommender4.recommend()
assert recommender4.estimated_u_value is None
assert len(recommender4.recommendations) == 0

87
recommendations/tests/test_recommendation_utils.py
View file

@ -1,3 +1,4 @@
import numpy as np
import pytest
import math
from unittest.mock import MagicMock
@ -277,6 +278,22 @@ class TestRecommendationUtils:
insulation_thickness=None,
)
def test_convert_thickness_to_numeric(self):
assert recommendation_utils.convert_thickness_to_numeric("none", True) == 0
assert recommendation_utils.convert_thickness_to_numeric("below average", True) == 50
assert recommendation_utils.convert_thickness_to_numeric("average", True) == 100
assert recommendation_utils.convert_thickness_to_numeric("above average", True) == 270
assert recommendation_utils.convert_thickness_to_numeric("300+", True) == 300
assert recommendation_utils.convert_thickness_to_numeric("400+", True) == 400
assert recommendation_utils.convert_thickness_to_numeric("270", True) == 270
assert recommendation_utils.convert_thickness_to_numeric("none", False) == 0
assert recommendation_utils.convert_thickness_to_numeric("below average", False) == 100
assert recommendation_utils.convert_thickness_to_numeric("average", False) == 270
assert recommendation_utils.convert_thickness_to_numeric("above average", False) == 270
def test_estimate_perimeter_regular_inputs():
assert math.isclose(
@ -333,3 +350,73 @@ def test_park_home():
assert recommendation_utils.get_floor_u_value(
'suspended', 100, 40, 'A', 'park home', insulation_thickness="20mm"
) == 0
def test_esimtate_pitched_roof_area():
roof_area1 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=100, floor_height=2
)
assert np.isclose(roof_area1, 107.70329614269008)
# As the floor height gets bigger, the area should get bigger
roof_area2 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=100, floor_height=3
)
assert np.isclose(roof_area2, 116.61903789690601)
# As the floor area gets smaller, the area should get smaller
roof_area3 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=100, floor_height=1
)
assert np.isclose(roof_area3, 101.9803902718557)
# As the floor area decreases, area should decrease
roof_area4 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=50, floor_height=2
)
assert np.isclose(roof_area4, 57.44562646538029)
# As the floor area increases, area should increase
roof_area5 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=150, floor_height=2
)
assert np.isclose(roof_area5, 157.797338380595)
zero_roof_area = recommendation_utils.esimtate_pitched_roof_area(
floor_area=0, floor_height=1000
)
assert zero_roof_area == 0
# If the floor height zero, we don't have a traingle, it's a flat roof
flat_roof_area = recommendation_utils.esimtate_pitched_roof_area(
floor_area=1000, floor_height=0
)
assert flat_roof_area == 1000
zero_roof_area2 = recommendation_utils.esimtate_pitched_roof_area(
floor_area=0, floor_height=0
)
assert zero_roof_area2 == 0
def test_external_wall_area():
# Arrange: Define the test cases
test_cases = [
(2, 3, 40, 'End-Terrace', 180), # 3 exposed walls
(2, 3, 40, 'Mid-Terrace', 120), # 2 exposed walls
(2, 3, 40, 'Semi-Detached', 180), # 3 exposed walls
(2, 3, 40, 'Detached', 240), # 4 exposed walls
]
# Act and Assert: Run the test cases
for num_floors, floor_height, perimeter, built_form, expected in test_cases:
result = recommendation_utils.estimate_external_wall_area(num_floors, floor_height, perimeter, built_form)
assert result == expected, f"Test failed for {built_form}: Expected {expected}, got {result}"

452
recommendations/tests/test_roof_recommendations.py Normal file
View file

@ -0,0 +1,452 @@
from backend.Property import Property
from unittest.mock import Mock
from recommendations.RoofRecommendations import RoofRecommendations
loft_insulation_materials = [
{
'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation',
'depths': [270, 300], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/',
'is_active': True
}
]
loft_insulation_materials_50mm_existing = [
{
'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation',
'depths': [220, 210], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/',
'is_active': True
}
]
loft_insulation_materials_150mm_existing = [
{
'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation',
'depths': [130, 119], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/',
'is_active': True
}
]
room_roof_insulation_materials = [
{
'id': 18,
'type': 'room_roof_insulation',
'description': 'Example room roof insulation',
'depths': [50, 150, 220, 270, 300], 'depth_unit': 'mm', 'cost': [9, 10, 11, 12, 13],
'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': None, 'is_active': True
}
]
flat_roof_insulation_materials = [
{
'id': 18,
'type': 'flat_roof_insulation',
'description': 'Example flat roof insulation',
'depths': [50, 150, 220, 270, 300], 'depth_unit': 'mm', 'cost': [9, 10, 11, 12, 13],
'cost_unit': 'gbp_sq_meter',
'r_value_per_mm': 0.032727273, 'r_value_unit': 'square_meter_kelvin_per_watt',
'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
'link': None, 'is_active': True
}
]
class TestRoofRecommendations:
def test_loft_insulation_recommendation_no_insulation(self):
property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance.age_band = "F"
property_instance.floor_area = 100
property_instance.roof = {
'original_description': 'Pitched, no insulation (assumed)',
'clean_description': 'Pitched, no insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': 'none', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender = RoofRecommendations(property_instance=property_instance, materials=loft_insulation_materials)
assert not roof_recommender.recommendations
roof_recommender.recommend()
assert len(roof_recommender.recommendations)
def test_loft_insulation_recommendation_50mm_insulation(self):
property_instance2 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance2.age_band = "F"
property_instance2.floor_area = 100
property_instance2.roof = {
'original_description': 'Pitched, 50mm loft insulation (assumed)',
'clean_description': 'Pitched, 50mm loft insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': '50', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender2 = RoofRecommendations(
property_instance=property_instance2, materials=loft_insulation_materials
)
assert not roof_recommender2.recommendations
roof_recommender2.recommend()
assert len(roof_recommender2.recommendations) == 1
assert roof_recommender2.recommendations[0]["cost"] == 900
assert roof_recommender2.recommendations[0]["new_u_value"] == 0.14
assert roof_recommender2.recommendations[0]["starting_u_value"] == 0.68
property_instance3 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance3.age_band = "F"
property_instance3.floor_area = 100
property_instance3.roof = {
'original_description': 'Pitched, 50mm loft insulation (assumed)',
'clean_description': 'Pitched, 50mm loft insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': '50', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender3 = RoofRecommendations(
property_instance=property_instance3, materials=loft_insulation_materials_50mm_existing
)
assert not roof_recommender3.recommendations
roof_recommender3.recommend()
# The 220mm insulation should be selected, not the 210
assert roof_recommender3.recommendations
assert len(roof_recommender3.recommendations) == 1
assert roof_recommender3.recommendations[0]["parts"][0]["depths"] == [220]
def test_loft_insulation_recommendation_150mm_insulation(self):
property_instance4 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance4.age_band = "F"
property_instance4.floor_area = 100
property_instance4.roof = {
'original_description': 'Pitched, 150mm loft insulation (assumed)',
'clean_description': 'Pitched, 150mm loft insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': '150', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender4 = RoofRecommendations(
property_instance=property_instance4, materials=loft_insulation_materials
)
assert not roof_recommender4.recommendations
roof_recommender4.recommend()
assert len(roof_recommender4.recommendations) == 1
assert roof_recommender4.recommendations[0]["cost"] == 900
assert roof_recommender4.recommendations[0]["new_u_value"] == 0.11
assert roof_recommender4.recommendations[0]["starting_u_value"] == 0.3
property_instance5 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance5.age_band = "F"
property_instance5.floor_area = 100
property_instance5.roof = {
'original_description': 'Pitched, 150mm loft insulation (assumed)',
'clean_description': 'Pitched, 150mm loft insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': '150', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender5 = RoofRecommendations(
property_instance=property_instance5, materials=loft_insulation_materials_150mm_existing
)
assert not roof_recommender5.recommendations
roof_recommender5.recommend()
# The 130mm insulation should be selected, not the 110
assert roof_recommender5.recommendations
assert len(roof_recommender5.recommendations) == 1
assert roof_recommender5.recommendations[0]["parts"][0]["depths"] == [130]
def test_loft_insulation_recommendation_270mm_insulation(self):
# We shouldn't recommend anything in this case
property_instance6 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance6.age_band = "F"
property_instance6.floor_area = 100
property_instance6.roof = {
'original_description': 'Pitched, 270mm loft insulation (assumed)',
'clean_description': 'Pitched, 270mm loft insulation',
'thermal_transmittance': None,
'thermal_transmittance_unit': None,
'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': '270', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none'
}
roof_recommender6 = RoofRecommendations(
property_instance=property_instance6, materials=loft_insulation_materials
)
assert not roof_recommender6.recommendations
roof_recommender6.recommend()
assert len(roof_recommender6.recommendations) == 0
def test_uninsulated_room_in_roof(self):
property_instance7 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance7.age_band = "F"
property_instance7.floor_area = 100
property_instance7.roof = {
'original_description': 'Roof room(s), no insulation (assumed)',
'clean_description': 'Roof room(s), no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'
}
property_instance7.pitched_roof_area = 110
roof_recommender7 = RoofRecommendations(
property_instance=property_instance7, materials=room_roof_insulation_materials
)
assert not roof_recommender7.recommendations
roof_recommender7.recommend()
# Even though we have 3 depths, we only end with 1 due to diminishin returns
assert len(roof_recommender7.recommendations) == 1
assert roof_recommender7.recommendations[0]["parts"][0]["depths"] == [270]
assert roof_recommender7.recommendations[0]["new_u_value"] == 0.14
assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8
assert roof_recommender7.recommendations[0]["description"] == \
"Insulate your room roof with 270mm of Example room roof insulation"
def test_ceiling_insulated_room_in_roof(self):
property_instance8 = Property(id=8, address1="fake", postcode="fake", epc_client=Mock())
property_instance8.age_band = "F"
property_instance8.floor_area = 100
property_instance8.roof = {
'original_description': 'Roof room(s), ceiling insulated',
'clean_description': 'Roof room(s), ceiling insulated',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
'is_at_rafters': False,
'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': 'average'
}
property_instance8.pitched_roof_area = 110
roof_recommender8 = RoofRecommendations(
property_instance=property_instance8, materials=room_roof_insulation_materials
)
assert not roof_recommender8.recommendations
roof_recommender8.recommend()
# No recommendations in this case
assert not roof_recommender8.recommendations
def test_insulated_room_in_roof(self):
property_instance9 = Property(id=9, address1="fake", postcode="fake", epc_client=Mock())
property_instance9.age_band = "F"
property_instance9.floor_area = 100
property_instance9.roof = {
'original_description': 'Roof room(s), insulated (assumed)',
'clean_description': 'Roof room(s), insulated',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'
}
property_instance9.pitched_roof_area = 110
roof_recommender9 = RoofRecommendations(
property_instance=property_instance9, materials=room_roof_insulation_materials
)
assert not roof_recommender9.recommendations
roof_recommender9.recommend()
# No recommendations in this case
assert not roof_recommender9.recommendations
def test_limited_insulated_room_in_roof(self):
property_instance10 = Property(id=10, address1="fake", postcode="fake", epc_client=Mock())
property_instance10.age_band = "F"
property_instance10.floor_area = 100
property_instance10.roof = {
'original_description': 'Roof room(s), limited insulation (assumed)',
'clean_description': 'Roof room(s), limited insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
'insulation_thickness': 'below average'
}
property_instance10.pitched_roof_area = 110
roof_recommender10 = RoofRecommendations(
property_instance=property_instance10, materials=room_roof_insulation_materials
)
assert not roof_recommender10.recommendations
roof_recommender10.recommend()
assert len(roof_recommender10.recommendations) == 2
assert roof_recommender10.recommendations[0]["parts"][0]["depths"] == [220]
assert roof_recommender10.recommendations[1]["parts"][0]["depths"] == [270]
assert roof_recommender10.recommendations[0]["new_u_value"] == 0.16
assert roof_recommender10.recommendations[1]["new_u_value"] == 0.14
assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.8
assert roof_recommender10.recommendations[1]["starting_u_value"] == 0.8
assert roof_recommender10.recommendations[0]["description"] == \
"Insulate your room roof with 220mm of Example room roof insulation"
assert roof_recommender10.recommendations[1]["description"] == \
"Insulate your room roof with 270mm of Example room roof insulation"
def test_flat_no_insulation(self):
property_instance11 = Property(id=11, address1="fake", postcode="fake", epc_client=Mock())
property_instance11.age_band = "D"
property_instance11.floor_area = 150
property_instance11.roof = {
'original_description': 'Flat, no insulation (assumed)',
'clean_description': 'Flat, no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': False, 'is_loft': False, 'is_flat': True, 'is_thatched': False, 'is_at_rafters': False,
'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'
}
roof_recommender11 = RoofRecommendations(
property_instance=property_instance11, materials=flat_roof_insulation_materials
)
assert not roof_recommender11.recommendations
roof_recommender11.recommend()
assert len(roof_recommender11.recommendations) == 1
assert roof_recommender11.recommendations[0]["parts"][0]["depths"] == [270]
assert roof_recommender11.recommendations[0]["new_u_value"] == 0.11
assert roof_recommender11.recommendations[0]["starting_u_value"] == 2.3
assert roof_recommender11.recommendations[0]["description"] == \
"Insulate the home's flat roof with 270mm of Example flat roof insulation"
def test_flat_insulated(self):
property_instance12 = Property(id=12, address1="fake", postcode="fake", epc_client=Mock())
property_instance12.age_band = "D"
property_instance12.floor_area = 150
property_instance12.roof = {
'original_description': 'Flat, insulated (assumed)',
'clean_description': 'Flat, insulated',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': False,
'is_loft': False, 'is_flat': True, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True,
'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'
}
roof_recommender12 = RoofRecommendations(
property_instance=property_instance12, materials=flat_roof_insulation_materials
)
assert not roof_recommender12.recommendations
roof_recommender12.recommend()
assert not roof_recommender12.recommendations
def test_flat_limited_insulation(self):
property_instance13 = Property(id=12, address1="fake", postcode="fake", epc_client=Mock())
property_instance13.age_band = "D"
property_instance13.floor_area = 150
property_instance13.roof = {
'original_description': 'Flat, limited insulation (assumed)',
'clean_description': 'Flat, limited insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
'is_roof_room': False,
'is_loft': False, 'is_flat': True, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True,
'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'below average'
}
roof_recommender13 = RoofRecommendations(
property_instance=property_instance13, materials=flat_roof_insulation_materials
)
assert not roof_recommender13.recommendations
roof_recommender13.recommend()
assert len(roof_recommender13.recommendations) == 1
assert roof_recommender13.recommendations[0]["parts"][0]["depths"] == [220]
assert roof_recommender13.recommendations[0]["new_u_value"] == 0.14
assert roof_recommender13.recommendations[0]["starting_u_value"] == 2.3
assert roof_recommender13.recommendations[0]["description"] == \
"Insulate the home's flat roof with 220mm of Example flat roof insulation"
def test_property_above(self):
property_instance14 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
property_instance14.age_band = "F"
property_instance14.floor_area = 100
property_instance14.roof = {
'original_description': '(other premises above)',
'clean_description': '(other premises above)', 'thermal_transmittance': 0,
'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False,
'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True,
'insulation_thickness': None
}
roof_recommender14 = RoofRecommendations(
property_instance=property_instance14, materials=loft_insulation_materials
)
assert not roof_recommender14.recommendations
roof_recommender14.recommend()
assert not roof_recommender14.recommendations

110
recommendations/tests/test_ventilation_recommendations.py Normal file
View file

@ -0,0 +1,110 @@
from backend.Property import Property
from unittest.mock import Mock
from recommendations.VentilationRecommendations import VentilationRecommendations
ventilation_materials = [
{
'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation',
'depths': None, 'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None,
'r_value_unit': None, 'thermal_conductivity': None, 'thermal_conductivity_unit': None,
'link': None, 'is_active': True, 'estimated_cost': 1000, 'quantity': 2, 'quantity_unit': None
}
]
class TestVentilationRecommendations:
def test_natural_ventilation(self):
input_property1 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property1.data = {"mechanical-ventilation": "natural"}
recommender = VentilationRecommendations(
property_instance=input_property1,
materials=ventilation_materials
)
assert not recommender.recommendation
recommender.recommend()
assert len(recommender.recommendation) == 1
assert recommender.recommendation[0]["cost"] == 1000
assert recommender.recommendation[0]["type"] == "mechanical_ventilation"
assert len(recommender.recommendation[0]["parts"]) == 1
assert recommender.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'
assert recommender.recommendation[0]["parts"][0]["quantity"] == 2
def test_missing_ventilation(self):
input_property2 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property2.data = {"mechanical-ventilation": None}
recommender2 = VentilationRecommendations(
property_instance=input_property2,
materials=ventilation_materials
)
assert not recommender2.recommendation
recommender2.recommend()
assert len(recommender2.recommendation) == 1
assert recommender2.recommendation[0]["cost"] == 1000
assert recommender2.recommendation[0]["type"] == "mechanical_ventilation"
assert len(recommender2.recommendation[0]["parts"]) == 1
assert recommender2.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'
assert recommender2.recommendation[0]["parts"][0]["quantity"] == 2
def test_nodata_ventilation(self):
input_property3 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property3.data = {"mechanical-ventilation": "NO DATA!!"}
recommender3 = VentilationRecommendations(
property_instance=input_property3,
materials=ventilation_materials
)
assert not recommender3.recommendation
recommender3.recommend()
assert len(recommender3.recommendation) == 1
assert recommender3.recommendation[0]["cost"] == 1000
assert recommender3.recommendation[0]["type"] == "mechanical_ventilation"
assert len(recommender3.recommendation[0]["parts"]) == 1
assert recommender3.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'
assert recommender3.recommendation[0]["parts"][0]["quantity"] == 2
def test_existing_ventilation_1(self):
input_property4 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property4.data = {"mechanical-ventilation": 'mechanical, extract only'}
recommender4 = VentilationRecommendations(
property_instance=input_property4,
materials=ventilation_materials
)
assert not recommender4.recommendation
recommender4.recommend()
assert not recommender4.recommendation
assert recommender4.has_ventilaion
def test_existing_ventilation_2(self):
input_property5 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property5.data = {"mechanical-ventilation": 'mechanical, supply and extract'}
recommender5 = VentilationRecommendations(
property_instance=input_property5,
materials=ventilation_materials
)
assert not recommender5.recommendation
recommender5.recommend()
assert not recommender5.recommendation
assert recommender5.has_ventilaion

261
recommendations/tests/test_wall_recommendations.py
View file

@ -407,36 +407,8 @@ class TestWallRecommendationsBase:
wall_recommendations_instance.property.data = {"property-type": "house"}
assert wall_recommendations_instance.ewi_valid is True
def test_recommend_without_u_value(self, wall_recommendations_instance):
wall_recommendations_instance.property.walls = {
"thermal_transmittance": None,
"is_solid_brick": False,
"is_cavity_wall": False,
"insulation_thickness": "none",
"clean_description": "Solid brick, as built, no insulation",
"is_granite_or_whinstone": False,
"is_sandstone_or_limestone": False,
}
wall_recommendations_instance.property.age_band = "A"
with pytest.raises(NotImplementedError):
wall_recommendations_instance.recommend()
class TestCavityWallRecommensations:
data = {
'low-energy-fixed-light-count': '', 'address': '123 Fake Street',
'floor-height': '', 'construction-age-band': 'England and Wales: 1950-1966',
'address3': '', 'property-type': 'House', 'local-authority-label': 'Melton',
'county': 'Leicestershire', 'postcode': 'LE14 2QH',
'solar-water-heating-flag': 'N', 'constituency': 'E14000909',
'number-heated-rooms': '5', 'local-authority': 'E07000133', 'built-form': 'End-Terrace',
'address1': '1, 23 fake', 'total-floor-area': '85.0', 'environment-impact-current': '49',
'number-habitable-rooms': 3, 'address2': 'Fake', 'posttown': 'IDK',
'walls-energy-eff': 'Poor', 'current-energy-rating': 'D',
'transaction-type': 'ECO assessment', 'uprn': '999', 'current-energy-efficiency': '57',
'lodgement-date': '2019-07-10', 'lmk-key': '999', 'tenure': 'rental (private)', 'floor-level': 'NODATA!',
'walls-description': 'Cavity wall, as built, no insulation (assumed)',
}
def test_fill_empty_cavity(self):
input_property = Property(id=1, postcode="F4k3", address1="123 fake street", epc_client=Mock())
@ -465,10 +437,10 @@ class TestCavityWallRecommensations:
assert recommender.recommendations
assert recommender.estimated_u_value == 1.5
assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.25)
assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.37)
assert np.isclose(recommender.recommendations[0]["cost"], 1000)
assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.26)
assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.38)
assert np.isclose(recommender.recommendations[1]["cost"], 1250)
def test_fill_partial_filled_cavity(self):
@ -498,8 +470,233 @@ class TestCavityWallRecommensations:
assert recommender.recommendations
assert recommender.estimated_u_value == 1.3
assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.56)
assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.43)
assert np.isclose(recommender.recommendations[0]["cost"], 1000)
assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.57)
assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.45)
assert np.isclose(recommender.recommendations[1]["cost"], 1250)
def test_system_built_wall(self):
input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property2.walls = {
'original_description': 'System built, as built, no insulation (assumed)',
'clean_description': 'System built, as built, no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None,
'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,
'is_system_built': True, 'is_timber_frame': False, 'is_granite_or_whinstone': False,
'is_as_built': True, 'is_cob': False, 'is_assumed': True,
'is_sandstone_or_limestone': False, 'is_park_home': False,
'insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False
}
input_property2.age_band = "F"
input_property2.insulation_wall_area = 120
input_property2.restricted_measures = False
input_property2.data = {"property-type": "house"}
assert input_property2.walls["is_system_built"]
recommender2 = WallRecommendations(
property_instance=input_property2,
materials=internal_wall_insulation_parts + external_wall_insulation_parts
)
assert not recommender2.recommendations
recommender2.recommend()
assert recommender2.recommendations
assert len(recommender2.recommendations) == 6
assert recommender2.estimated_u_value == 1
assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.29)
assert np.isclose(recommender2.recommendations[0]["cost"], 10800)
assert recommender2.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
assert recommender2.recommendations[0]["parts"][0]["depths"] == [90]
assert np.isclose(recommender2.recommendations[5]["new_u_value"], 0.29)
assert np.isclose(recommender2.recommendations[5]["cost"], 2400)
assert recommender2.recommendations[5]["parts"][0]["type"] == "internal_wall_insulation"
assert recommender2.recommendations[5]["parts"][0]["depths"] == [20]
assert np.isclose(recommender2.recommendations[3]["new_u_value"], 0.28)
assert np.isclose(recommender2.recommendations[3]["cost"], 4800)
assert recommender2.recommendations[3]["parts"][0]["type"] == "external_wall_insulation"
assert recommender2.recommendations[3]["parts"][0]["depths"] == [40]
def test_timber_frame_wall(self):
input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property3.walls = {
'original_description': 'Timber frame, as built, no insulation (assumed)',
'clean_description': 'Timber frame, as built, no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None,
'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': True, 'is_granite_or_whinstone': False,
'is_as_built': True, 'is_cob': False, 'is_assumed': True,
'is_sandstone_or_limestone': False, 'is_park_home': False,
'insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False
}
input_property3.age_band = "B"
input_property3.insulation_wall_area = 99
input_property3.restricted_measures = False
input_property3.data = {"property-type": "house"}
assert input_property3.walls["is_timber_frame"]
recommender3 = WallRecommendations(
property_instance=input_property3,
materials=internal_wall_insulation_parts + external_wall_insulation_parts
)
assert not recommender3.recommendations
recommender3.recommend()
assert recommender3.recommendations
assert len(recommender3.recommendations) == 2
assert recommender3.estimated_u_value == 1.9
assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.26)
assert np.isclose(recommender3.recommendations[0]["cost"], 12375)
assert recommender3.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
assert recommender3.recommendations[0]["parts"][0]["depths"] == [125]
assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.26)
assert np.isclose(recommender3.recommendations[1]["cost"], 4950)
assert recommender3.recommendations[1]["parts"][0]["type"] == "external_wall_insulation"
assert recommender3.recommendations[1]["parts"][0]["depths"] == [50]
def test_granite_or_whinstone_wall(self):
input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property4.walls = {
'original_description': 'Granite or whinstone, as built, no insulation (assumed)',
'clean_description': 'Granite or whinstone, as built, no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None,
'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': True,
'is_as_built': True, 'is_cob': False, 'is_assumed': True,
'is_sandstone_or_limestone': False, 'is_park_home': False,
'insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False
}
input_property4.age_band = "A"
input_property4.insulation_wall_area = 223
input_property4.restricted_measures = False
input_property4.data = {"property-type": "Bungalow"}
assert input_property4.walls["is_granite_or_whinstone"]
recommender4 = WallRecommendations(
property_instance=input_property4,
materials=internal_wall_insulation_parts + external_wall_insulation_parts
)
assert not recommender4.recommendations
recommender4.recommend()
assert recommender4.recommendations
assert len(recommender4.recommendations) == 2
assert recommender4.estimated_u_value == 2.3
assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.27)
assert np.isclose(recommender4.recommendations[0]["cost"], 27875)
assert recommender4.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
assert recommender4.recommendations[0]["parts"][0]["depths"] == [125]
assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.27)
assert np.isclose(recommender4.recommendations[1]["cost"], 11150)
assert recommender4.recommendations[1]["parts"][0]["type"] == "external_wall_insulation"
assert recommender4.recommendations[1]["parts"][0]["depths"] == [50]
def test_cob_wall(self):
input_property5 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
input_property5.walls = {
'original_description': 'Cob, as built',
'clean_description': 'Cob, as built',
'thermal_transmittance': None, 'thermal_transmittance_unit': None,
'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False,
'is_as_built': False, 'is_cob': True, 'is_assumed': False,
'is_sandstone_or_limestone': False, 'is_park_home': False,
'insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False
}
input_property5.age_band = "E"
input_property5.insulation_wall_area = 77
input_property5.restricted_measures = False
input_property5.data = {"property-type": "Bungalow"}
assert input_property5.walls["is_cob"]
recommender5 = WallRecommendations(
property_instance=input_property5,
materials=internal_wall_insulation_parts + external_wall_insulation_parts
)
assert not recommender5.recommendations
recommender5.recommend()
assert recommender5.recommendations
assert len(recommender5.recommendations) == 9
assert recommender5.estimated_u_value == 0.8
assert np.isclose(recommender5.recommendations[0]["new_u_value"], 0.29)
assert np.isclose(recommender5.recommendations[0]["cost"], 6160)
assert recommender5.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
assert recommender5.recommendations[0]["parts"][0]["depths"] == [80]
assert np.isclose(recommender5.recommendations[3]["new_u_value"], 0.26)
assert np.isclose(recommender5.recommendations[3]["cost"], 7700)
assert recommender5.recommendations[3]["parts"][0]["type"] == "external_wall_insulation"
assert recommender5.recommendations[3]["parts"][0]["depths"] == [100]
assert np.isclose(recommender5.recommendations[6]["new_u_value"], 0.26)
assert np.isclose(recommender5.recommendations[6]["cost"], 7700)
assert recommender5.recommendations[6]["parts"][0]["type"] == "internal_wall_insulation"
assert recommender5.recommendations[6]["parts"][0]["depths"] == [100]
def test_sandstone_or_limestone_wall(self):
input_property6 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock())
input_property6.walls = {
'original_description': 'Sandstone or limestone, as built, no insulation (assumed)',
'clean_description': 'Sandstone or limestone, as built, no insulation',
'thermal_transmittance': None, 'thermal_transmittance_unit': None,
'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False,
'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False,
'is_as_built': False, 'is_cob': False, 'is_assumed': False,
'is_sandstone_or_limestone': True, 'is_park_home': False,
'insulation_thickness': 'none', 'external_insulation': False,
'internal_insulation': False
}
input_property6.age_band = "F"
input_property6.insulation_wall_area = 350
input_property6.restricted_measures = False
input_property6.data = {"property-type": "House"}
assert input_property6.walls["is_sandstone_or_limestone"]
recommender6 = WallRecommendations(
property_instance=input_property6,
materials=internal_wall_insulation_parts + external_wall_insulation_parts
)
assert not recommender6.recommendations
recommender6.recommend()
assert recommender6.recommendations
assert len(recommender6.recommendations) == 6
assert recommender6.estimated_u_value == 1
assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.29)
assert np.isclose(recommender6.recommendations[0]["cost"], 31500)
assert recommender6.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
assert recommender6.recommendations[0]["parts"][0]["depths"] == [90]
assert np.isclose(recommender6.recommendations[2]["new_u_value"], 0.28)
assert np.isclose(recommender6.recommendations[2]["cost"], 35000)
assert recommender6.recommendations[2]["parts"][0]["type"] == "external_wall_insulation"
assert recommender6.recommendations[2]["parts"][0]["depths"] == [100]
assert np.isclose(recommender6.recommendations[4]["new_u_value"], 0.28)
assert np.isclose(recommender6.recommendations[4]["cost"], 35000)
assert recommender6.recommendations[4]["parts"][0]["type"] == "internal_wall_insulation"
assert recommender6.recommendations[4]["parts"][0]["depths"] == [100]