junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #302 from Hestia-Homes/main

Browse source 
Multiple changes - but putting through to handle the aws rds certificate change
This commit is contained in:
KhalimCK 2024-05-29 11:36:14 +01:00 • committed by GitHub
commit 9e0ab81a50
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
53 changed files with 5089 additions and 271 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyNamespacePackagesService">

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>

109
backend/Property.py
View file

@ -61,7 +61,15 @@ class Property:
n_bedrooms = None
def __init__(
self, id, postcode, address, epc_record, already_installed=None, **kwargs
self,
id,
postcode,
address,
epc_record,
already_installed=None,
non_invasive_recommendations=None,
measures=None,
**kwargs
):
self.epc_record = epc_record
@ -80,6 +88,12 @@ class Property:
# cost and instead, provide a message that the measure has already been installed
self.already_installed = ast.literal_eval(already_installed['already_installed']) if already_installed else []
self.non_invasive_recommendations = (
ast.literal_eval(non_invasive_recommendations['recommendations']) if
non_invasive_recommendations else []
)
# This is a list of measures that have been recommended for the property
self.measures = ast.literal_eval(measures) if measures else None
self.uprn = epc_record.get("uprn")
self.full_sap_epc = epc_record.get("full_sap_epc")
@ -142,6 +156,8 @@ class Property:
self.current_adjusted_energy = None
self.expected_adjusted_energy = None
self.current_energy_bill = None
self.expected_energy_bill = None
self.recommendations_scoring_data = []
@ -156,12 +172,12 @@ class Property:
:return:
"""
n_bathrooms = kwargs.get("n_bathrooms", None)
if n_bathrooms is not None:
if n_bathrooms not in [None, ""]:
# We add on a small value to ensure that the number of bathrooms is rounded up, in case the value is 0.5
n_bathrooms = int(round(float(n_bathrooms) + 1e-5))
n_bedrooms = kwargs.get("n_bedrooms", None)
if n_bedrooms is not None:
if n_bedrooms not in [None, ""]:
n_bedrooms = int(round(float(n_bedrooms) + 1e-5))
return {
@ -214,6 +230,29 @@ class Property:
# self.base_difference_record.df
def simulate_all_representative_recommendations(
self, property_representative_recommendations,
):
"""
This method was put together to simulate the impact of the representative recommendations on the property
all at once, for usage within the mds report
:return:
"""
recommendation_record = self.base_difference_record.df.to_dict("records")[
0
].copy()
scoring_dict = self.create_recommendation_scoring_data(
property_id=self.id,
recommendation_record=recommendation_record,
recommendations=property_representative_recommendations,
primary_recommendation_id=self.id,
non_invasive_recommendations=self.non_invasive_recommendations,
)
return scoring_dict
def adjust_difference_record_with_recommendations(
self, property_recommendations, property_representative_recommendations
):
@ -277,6 +316,7 @@ class Property:
recommendation_record=recommendation_record,
recommendations=previous_phase_representatives + [rec],
primary_recommendation_id=rec["recommendation_id"],
non_invasive_recommendations=self.non_invasive_recommendations,
)
self.recommendations_scoring_data.append(scoring_dict)
@ -286,6 +326,7 @@ class Property:
recommendation_record,
recommendations: list,
primary_recommendation_id: int,
non_invasive_recommendations: list = None,
):
"""
This function will iterate through a list of recommendations and apply a simulation for each recommendation
@ -294,10 +335,12 @@ class Property:
:param recommendation_record: The record of the property, which will be updated
:param recommendations: The list of recommendations to apply
:param primary_recommendation_id: The id of the primary recommendation, which is used to identify the record
:param non_invasive_recommendations: The list of non-invasive recommendations
:return: The updated recommendation record
"""
output = recommendation_record.copy()
non_invasive_recommendations = [] if non_invasive_recommendations is None else non_invasive_recommendations
for col in [
"walls_insulation_thickness",
@ -310,42 +353,6 @@ class Property:
for recommendation in recommendations:
# For the list of recommendations we have, we iteratively update the output
# We update the description to indicate it's insulated
if recommendation["type"] in [
"internal_wall_insulation",
"external_wall_insulation",
"cavity_wall_insulation",
]:
# The upgrade made here is to the u-value of the walls and the description of the
# insulation thickness
output["walls_thermal_transmittance_ending"] = recommendation[
"new_u_value"
]
# Setting the insulation thickness here to above average should be tested further because we
# don't see a high volume of instances for this
output["walls_insulation_thickness_ending"] = "average"
output["walls_energy_eff_ending"] = "Good"
# Note: often when the wall is insulatied, the internal/external insulation is not noted so we should
# test the impact of using these booleans
if recommendation["type"] == "external_wall_insulation":
output["external_insulation_ending"] = True
output["internal_insulation_ending"] = False
if recommendation["type"] == "internal_wall_insulation":
output["external_insulation_ending"] = False
output["internal_insulation_ending"] = True
if recommendation["type"] == "cavity_wall_insulation":
output["is_filled_cavity_ending"] = True
else:
if output["walls_thermal_transmittance_ending"] is None:
raise ValueError("We should not have a None value for the u value")
if output["walls_insulation_thickness_ending"] is None:
output["walls_insulation_thickness_ending"] = "none"
# Update description to indicate it's insulate
if recommendation["type"] in [
"solid_floor_insulation",
@ -357,11 +364,8 @@ class Property:
"Have more than 1 floor insulation part - handle this case"
)
# output["floor_thermal_transmittance_ending"] = recommendation["new_u_value"]
# We don't really see above average for this in the training data
output["floor_insulation_thickness_ending"] = "average"
# This is rarely ever populated in the training data
# output["floor_energy_eff_ending"] = "Good"
else:
if output["floor_thermal_transmittance_ending"] is None:
raise ValueError("We should not have a None value for the u value")
@ -400,19 +404,20 @@ class Property:
400,
]
proposed_depth = int(parts[0]["depth"])
proposed_depth = recommendation["new_thickness"]
if proposed_depth not in valid_numeric_values:
# Take the nearest value for scoring
proposed_depth = min(
valid_numeric_values, key=lambda x: abs(x - proposed_depth)
)
output["roof_insulation_thickness_ending"] = str(proposed_depth)
output["roof_insulation_thickness_ending"] = str(int(proposed_depth))
if recommendation["type"] == "loft_insulation":
if proposed_depth >= 270:
output["roof_energy_eff_ending"] = "Very Good"
else:
output["roof_energy_eff_ending"] = "Good"
if output["roof_energy_eff_ending"] not in ["Good", "Very Good"]:
output["roof_energy_eff_ending"] = "Good"
else:
output["roof_energy_eff_ending"] = "Very Good"
else:
@ -432,7 +437,8 @@ class Property:
if recommendation["type"] == "windows_glazing":
output["multi_glaze_proportion_ending"] = 100
output["windows_energy_eff_ending"] = "Average"
if output["windows_energy_eff_ending"] not in ["Average", "Good", "Very Good"]:
output["windows_energy_eff_ending"] = "Average"
is_secondary_glazing = recommendation["is_secondary_glazing"]
@ -463,9 +469,12 @@ class Property:
)
if recommendation["type"] in [
"heating", "hot_water_tank_insulation", "heating_control", "secondary_heating"
"heating", "hot_water_tank_insulation", "heating_control", "secondary_heating",
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
]:
# We update the data, as defined in the recommendaton
if output["walls_insulation_thickness_ending"] is None:
output["walls_insulation_thickness_ending"] = "none"
simulation_config = recommendation["simulation_config"]
# If any entries in simulation_config are None, we will set them to "Unknown" which is the cleaning
@ -892,12 +901,16 @@ class Property:
return component_data
def set_adjusted_energy(self, current_adjusted_energy, expected_adjusted_energy):
def set_adjusted_energy(
self, current_adjusted_energy, expected_adjusted_energy, current_energy_bill, expected_energy_bill
):
"""
Stores these values for usage later
"""
self.current_adjusted_energy = current_adjusted_energy
self.expected_adjusted_energy = expected_adjusted_energy
self.current_energy_bill = current_energy_bill
self.expected_energy_bill = expected_energy_bill
def set_windows_count(self):
"""

46
backend/SearchEpc.py
View file

@ -193,33 +193,32 @@ class SearchEpc:
@classmethod
def get_house_number(cls, address: str) -> str | None:
"""
This method will use the usaddress library to parse an address and extract the house number
:return:
This method uses the usaddress library to parse an address and extract the primary house or flat number.
"""
try:
parsed = usaddress.parse(address)
parsed_house_number = [x for x in parsed if (x[1] == "AddressNumber")]
parsed_house_number = parsed_house_number[0][0] if parsed_house_number else None
if parsed_house_number is None:
# Because usaddress isn't optimal for parsing addresses with some prefixes such as 'Flat',
# we also add a custom approach
# Pattern to look for 'Flat' or 'Apartment' followed by a number, or just a number at the beginning
# Custom regex to catch a broad range of cases
pattern = r'(?i)(?:flat|apartment)\s*(\d+)|^\s*(\d+)'
match = re.search(pattern, address)
if match:
# Return the first non-None group found
return next(g for g in match.groups() if g is not None)
else:
return None
# Remove training commas
parsed_house_number = parsed_house_number.replace(",", "")
parsed = usaddress.parse(address)
# First, try to get the 'OccupancyIdentifier' if 'OccupancyType' is detected
for part, type_ in parsed:
if type_ == 'OccupancyIdentifier':
return part # This assumes the first 'OccupancyIdentifier' after 'OccupancyType' is the primary
# number
return parsed_house_number
# Fallback to 'AddressNumber' if no 'OccupancyIdentifier' is found
address_number = next((part for part, type_ in parsed if type_ == 'AddressNumber'), None)
if address_number:
return address_number.replace(",", "") # Remove any trailing commas
except Exception as e:
print(f"Error parsing address: {e}")
return None
@staticmethod
def extract_numeric_housenumber_part(house_number: str | None) -> int | None:
@ -709,8 +708,13 @@ class SearchEpc:
self.full_sap_epc = {}
# Finally, set a standardised address 1 and postcode
self.address_clean = self.ordnance_survey_client.address_os
self.postcode_clean = self.ordnance_survey_client.postcode_os
self.address_clean = (
self.ordnance_survey_client.address_os if self.ordnance_survey_client.address_os else self.address1
)
self.postcode_clean = (
self.ordnance_survey_client.postcode_os if self.ordnance_survey_client.postcode_os else
self.postcode
)
return
os_response = self.ordnance_survey_client.get_places_api()

336
backend/apis/GoogleSolarApi.py Normal file
View file

@ -0,0 +1,336 @@
from backend.Property import Property
from backend.SearchEpc import SearchEpc
from etl.epc.Record import EPCRecord
from dotenv import load_dotenv
from utils.s3 import read_dataframe_from_s3_parquet
import os
import requests
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
# This is for 6 Laura Close, Tintagel, PL34 0EB (same property that Cotswolrd energy used)
uprn = 100040099104
# This is for 353A, Hermitage Lane, ME16 9NT (one of the e.on properties)
uprn = 200000964454
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
searcher = SearchEpc(address1="", postcode="", uprn=uprn, auth_token=EPC_AUTH_TOKEN, os_api_key="")
searcher.find_property(skip_os=True)
epc_records = {
'original_epc': searcher.newest_epc.copy(),
'full_sap_epc': searcher.full_sap_epc.copy(),
'old_data': searcher.older_epcs.copy(),
}
epc = EPCRecord(
epc_records=epc_records,
run_mode="newdata",
cleaning_data=cleaning_data
)
uprn_filenames = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
)
p = Property(
id=0,
address=searcher.address_clean,
postcode=searcher.postcode_clean,
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
p.get_spatial_data(uprn_filenames)
longitude = p.spatial["longitude"]
latitude = p.spatial["latitude"]
api_key = "AIzaSyCIz8Psu5h-1txuDX0rQpUTgkvdj8yohqU"
url = 'https://solar.googleapis.com/v1/solarPotential'
params = {
'location.latitude': f'{latitude:.5f}',
'location.longitude': f'{longitude:.5f}',
'requiredQuality': "MEDIUM",
'key': api_key
}
insights_url = 'https://solar.googleapis.com/v1/buildingInsights:findClosest'
# Make the GET request to the Solar API
insights_response = requests.get(insights_url, params=params)
insights_data = insights_response.json()
solar_potential = insights_data["solarPotential"]
from pprint import pprint
pprint(solar_potential)
# This is the size of the panels used in the calculation - 400 watt
solar_potential["panelCapacityWatts"]
# Height of the panels used
solar_potential["panelHeightMeters"]
# Width of the panels used
solar_potential["panelWidthMeters"]
solar_potential["wholeRoofStats"]
# Copy of response for testing - 6 Laura Close, Tintagel, PL34 0EB
# {'name': 'buildings/ChIJ2yC6t4KEa0gRh2TIssogI7k', 'center': {'latitude': 50.667375, 'longitude': -4.7416833},
# 'imageryDate': {'year': 2021, 'month': 7, 'day': 19}, 'regionCode': 'GB', 'solarPotential': {'maxArrayPanelsCount':
# 39, 'maxArrayAreaMeters2': 76.578636, 'maxSunshineHoursPerYear': 1172.0627, 'carbonOffsetFactorKgPerMwh':
# 478.99942, 'wholeRoofStats': {'areaMeters2': 129.65686, 'sunshineQuantiles': [537, 738.3836, 805.62445, 842.6802,
# 909.8431, 972.15234, 1036.1013, 1092.051, 1135.8192, 1163.1444, 1193.6012], 'groundAreaMeters2': 112.33},
# 'roofSegmentStats': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'stats': {'areaMeters2': 44.08321,
# 'sunshineQuantiles': [614, 940.86975, 982.39124, 1057.0664, 1109.6869, 1137.5837, 1152.9211, 1163.1106, 1168.2212,
# 1170.8883, 1193.6012], 'groundAreaMeters2': 37.61}, 'center': {'latitude': 50.6673664, 'longitude':
# -4.741714099999999}, 'boundingBox': {'sw': {'latitude': 50.6673354, 'longitude': -4.741777}, 'ne': {'latitude':
# 50.6674029, 'longitude': -4.7416472}}, 'planeHeightAtCenterMeters': 93.0221}, {'pitchDegrees': 34.39779,
# 'azimuthDegrees': 31.74401, 'stats': {'areaMeters2': 44.622986, 'sunshineQuantiles': [537, 671.49774, 733.84985,
# 780.82733, 801.4026, 814.0189, 824.0077, 847.77484, 895.08295, 950.1469, 1123.3503], 'groundAreaMeters2': 36.82},
# 'center': {'latitude': 50.6673966, 'longitude': -4.7416813}, 'boundingBox': {'sw': {'latitude': 50.667361,
# 'longitude': -4.7417497}, 'ne': {'latitude': 50.6674303, 'longitude': -4.741615599999999}},
# 'planeHeightAtCenterMeters': 92.87593}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'stats': {
# 'areaMeters2': 17.074476, 'sunshineQuantiles': [644.71136, 731.0546, 782.89813, 842.7107, 908.55585, 966.6212,
# 1010.6367, 1038.2543, 1053.2788, 1090.6831, 1128.0178], 'groundAreaMeters2': 17.050001}, 'center': {'latitude':
# 50.66740850000001, 'longitude': -4.7416025}, 'boundingBox': {'sw': {'latitude': 50.6673895, 'longitude':
# -4.7416436}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415572}}, 'planeHeightAtCenterMeters':
# 90.630356}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'stats': {'areaMeters2': 13.501617,
# 'sunshineQuantiles': [749, 976.85345, 1059.0062, 1081.6173, 1097.4441, 1110.3171, 1128.2186, 1133.9421, 1142.068,
# 1148.2168, 1157.632], 'groundAreaMeters2': 12.02}, 'center': {'latitude': 50.667315699999996, 'longitude':
# -4.741675400000001}, 'boundingBox': {'sw': {'latitude': 50.667291399999996, 'longitude': -4.7417066},
# 'ne': {'latitude': 50.6673372, 'longitude': -4.741648400000001}}, 'planeHeightAtCenterMeters': 92.36334},
# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'stats': {'areaMeters2': 10.374564, 'sunshineQuantiles': [
# 617.9507, 752.2504, 847.66315, 872.0505, 881.26227, 900.9639, 933.3188, 967.4747, 1000.8129, 1038.3002, 1105.545],
# 'groundAreaMeters2': 8.83}, 'center': {'latitude': 50.6673295, 'longitude': -4.7417128}, 'boundingBox': {'sw': {
# 'latitude': 50.6673134, 'longitude': -4.7417422}, 'ne': {'latitude': 50.6673413, 'longitude': -4.7416775}},
# 'planeHeightAtCenterMeters': 92.31146}], 'solarPanelConfigs': [{'panelsCount': 4, 'yearlyEnergyDcKwh': 1867.1516,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 4,
# 'yearlyEnergyDcKwh': 1867.1515, 'segmentIndex': 0}]}, {'panelsCount': 5, 'yearlyEnergyDcKwh': 2335.0068,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 5,
# 'yearlyEnergyDcKwh': 2335.0068, 'segmentIndex': 0}]}, {'panelsCount': 6, 'yearlyEnergyDcKwh': 2799.8508,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 6,
# 'yearlyEnergyDcKwh': 2799.8508, 'segmentIndex': 0}]}, {'panelsCount': 7, 'yearlyEnergyDcKwh': 3264.6506,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 7,
# 'yearlyEnergyDcKwh': 3264.6506, 'segmentIndex': 0}]}, {'panelsCount': 8, 'yearlyEnergyDcKwh': 3726.2405,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 8,
# 'yearlyEnergyDcKwh': 3726.2405, 'segmentIndex': 0}]}, {'panelsCount': 9, 'yearlyEnergyDcKwh': 4187.721,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 9,
# 'yearlyEnergyDcKwh': 4187.721, 'segmentIndex': 0}]}, {'panelsCount': 10, 'yearlyEnergyDcKwh': 4646.094,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}]}, {'panelsCount': 11, 'yearlyEnergyDcKwh': 5103.777,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
# 'panelsCount': 1, 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}]}, {'panelsCount': 12, 'yearlyEnergyDcKwh':
# 5559.845, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 13, 'yearlyEnergyDcKwh':
# 6013.053, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 11,
# 'yearlyEnergyDcKwh': 5099.302, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 14, 'yearlyEnergyDcKwh':
# 6461.664, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 15, 'yearlyEnergyDcKwh':
# 6902.33, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
# 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 16, 'yearlyEnergyDcKwh':
# 7321.6436, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
# 'panelsCount': 1, 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 17,
# 'yearlyEnergyDcKwh': 7740.388, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331,
# 'panelsCount': 12, 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
# 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees':
# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]},
# {'panelsCount': 18, 'yearlyEnergyDcKwh': 8154.265, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 13, 'yearlyEnergyDcKwh': 5961.7896, 'segmentIndex': 0},
# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579,
# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh':
# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 19, 'yearlyEnergyDcKwh': 8566.032, 'roofSegmentSummaries': [{
# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 14, 'yearlyEnergyDcKwh': 6373.556,
# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh':
# 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 20, 'yearlyEnergyDcKwh': 8976.624,
# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 21,
# 'yearlyEnergyDcKwh': 9380.78, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331,
# 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
# 'azimuthDegrees': 301.1099, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1242.214, 'segmentIndex': 2}, {'pitchDegrees':
# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]},
# {'panelsCount': 22, 'yearlyEnergyDcKwh': 9784.078, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh': 1645.5122,
# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh':
# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 23, 'yearlyEnergyDcKwh': 10162.354, 'roofSegmentSummaries': [{
# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484,
# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh':
# 1645.5122, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount': 24, 'yearlyEnergyDcKwh':
# 10535.894, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
# 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294,
# 'azimuthDegrees': 308.42334, 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount':
# 25, 'yearlyEnergyDcKwh': 10901.273, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees':
# 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees':
# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3},
# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497,
# 'segmentIndex': 4}]}, {'panelsCount': 26, 'yearlyEnergyDcKwh': 11242.756, 'roofSegmentSummaries': [{'pitchDegrees':
# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 1, 'yearlyEnergyDcKwh': 341.4827,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 27, 'yearlyEnergyDcKwh':
# 11579.401, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 678.1277, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
# {'panelsCount': 28, 'yearlyEnergyDcKwh': 11919.106, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1017.83356,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 29, 'yearlyEnergyDcKwh':
# 12255.358, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 4, 'yearlyEnergyDcKwh': 1354.0854, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
# {'panelsCount': 30, 'yearlyEnergyDcKwh': 12586.448, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 5, 'yearlyEnergyDcKwh': 1685.1748,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 31, 'yearlyEnergyDcKwh':
# 12911.502, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 6, 'yearlyEnergyDcKwh': 2010.2289, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
# {'panelsCount': 32, 'yearlyEnergyDcKwh': 13233.139, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 7, 'yearlyEnergyDcKwh': 2331.8652,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 33, 'yearlyEnergyDcKwh':
# 13554.602, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 8, 'yearlyEnergyDcKwh': 2653.3286, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
# {'panelsCount': 34, 'yearlyEnergyDcKwh': 13893.903, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 9, 'yearlyEnergyDcKwh': 2992.6301,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 35, 'yearlyEnergyDcKwh':
# 14221.166, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 10, 'yearlyEnergyDcKwh': 3319.893, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
# {'panelsCount': 36, 'yearlyEnergyDcKwh': 14536.154, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 11, 'yearlyEnergyDcKwh': 3634.8809,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 37, 'yearlyEnergyDcKwh':
# 14850.317, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 12, 'yearlyEnergyDcKwh': 3949.0444, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775,
# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees':
# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3},
# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497,
# 'segmentIndex': 4}]}, {'panelsCount': 38, 'yearlyEnergyDcKwh': 15160.658, 'roofSegmentSummaries': [{'pitchDegrees':
# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 13, 'yearlyEnergyDcKwh': 4259.385,
# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 39, 'yearlyEnergyDcKwh':
# 15438.986, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
# 'panelsCount': 14, 'yearlyEnergyDcKwh': 4537.713, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}],
# 'panelCapacityWatts': 400, 'panelHeightMeters': 1.879, 'panelWidthMeters': 1.045, 'panelLifetimeYears': 20,
# 'buildingStats': {'areaMeters2': 138.38115, 'sunshineQuantiles': [537, 728.5604, 799.23975, 833.99713, 900.88086,
# 959.65875, 1024.2743, 1086.1285, 1132.8774, 1162.1904, 1193.6012], 'groundAreaMeters2': 117.16}, 'solarPanels': [{
# 'center': {'latitude': 50.667371499999994, 'longitude': -4.7417235}, 'orientation': 'LANDSCAPE',
# 'yearlyEnergyDcKwh': 468.5037, 'segmentIndex': 0}, {'center': {'latitude': 50.6673614, 'longitude': -4.7417023},
# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.61072, 'segmentIndex': 0}, {'center': {'latitude':
# 50.667365100000005, 'longitude': -4.7417311}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.55005,
# 'segmentIndex': 0}, {'center': {'latitude': 50.6673512, 'longitude': -4.741681000000001}, 'orientation':
# 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.48712, 'segmentIndex': 0}, {'center': {'latitude': 50.667357599999995,
# 'longitude': -4.7416734}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.8553, 'segmentIndex': 0},
# {'center': {'latitude': 50.6673779, 'longitude': -4.741715999999999}, 'orientation': 'LANDSCAPE',
# 'yearlyEnergyDcKwh': 464.84396, 'segmentIndex': 0}, {'center': {'latitude': 50.6673678, 'longitude': -4.7416947},
# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 464.79984, 'segmentIndex': 0}, {'center': {'latitude': 50.6673549,
# 'longitude': -4.7417098}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 461.58975, 'segmentIndex': 0},
# {'center': {'latitude': 50.6673816, 'longitude': -4.7417448}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
# 461.48065, 'segmentIndex': 0}, {'center': {'latitude': 50.6673881, 'longitude': -4.7417372}, 'orientation':
# 'LANDSCAPE', 'yearlyEnergyDcKwh': 458.3733, 'segmentIndex': 0}, {'center': {'latitude': 50.6673149, 'longitude':
# -4.7416768}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}, {'center': {
# 'latitude': 50.6673204, 'longitude': -4.7416867}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 456.06827,
# 'segmentIndex': 3}, {'center': {'latitude': 50.667375199999995, 'longitude': -4.7417524}, 'orientation':
# 'LANDSCAPE', 'yearlyEnergyDcKwh': 453.20776, 'segmentIndex': 0}, {'center': {'latitude': 50.667364, 'longitude':
# -4.7416659}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 448.61087, 'segmentIndex': 0}, {'center': {
# 'latitude': 50.6673094, 'longitude': -4.741666899999999}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
# 440.66626, 'segmentIndex': 3}, {'center': {'latitude': 50.667403799999995, 'longitude': -4.741588900000001},
# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'center': {'latitude':
# 50.66740850000001, 'longitude': -4.7416016999999995}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 418.74448,
# 'segmentIndex': 2}, {'center': {'latitude': 50.6673688, 'longitude': -4.7417599}, 'orientation': 'LANDSCAPE',
# 'yearlyEnergyDcKwh': 413.877, 'segmentIndex': 0}, {'center': {'latitude': 50.667348499999996, 'longitude':
# -4.7417174}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 411.76657, 'segmentIndex': 0}, {'center': {
# 'latitude': 50.6673587, 'longitude': -4.7417387}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 410.5925,
# 'segmentIndex': 0}, {'center': {'latitude': 50.6673992, 'longitude': -4.7415761}, 'orientation': 'LANDSCAPE',
# 'yearlyEnergyDcKwh': 404.15607, 'segmentIndex': 2}, {'center': {'latitude': 50.6674132, 'longitude': -4.7416145},
# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 403.29822, 'segmentIndex': 2}, {'center': {'latitude': 50.6673324,
# 'longitude': -4.7417015}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}, {'center':
# {'latitude': 50.667417799999996, 'longitude': -4.7416273}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
# 373.53967, 'segmentIndex': 2}, {'center': {'latitude': 50.667324900000004, 'longitude': -4.7417104}, 'orientation':
# 'PORTRAIT', 'yearlyEnergyDcKwh': 365.37958, 'segmentIndex': 4}, {'center': {'latitude': 50.6674043, 'longitude':
# -4.741680800000001}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 341.4827, 'segmentIndex': 1}, {'center': {
# 'latitude': 50.667392299999996, 'longitude': -4.7416919}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh':
# 336.64502, 'segmentIndex': 1}, {'center': {'latitude': 50.667397, 'longitude': -4.741704599999999}, 'orientation':
# 'PORTRAIT', 'yearlyEnergyDcKwh': 339.7059, 'segmentIndex': 1}, {'center': {'latitude': 50.6674018, 'longitude':
# -4.7417174}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 336.25195, 'segmentIndex': 1}, {'center': {'latitude':
# 50.6673875, 'longitude': -4.7416791}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 331.08936, 'segmentIndex':
# 1}, {'center': {'latitude': 50.6674065, 'longitude': -4.7417301}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh':
# 325.05405, 'segmentIndex': 1}, {'center': {'latitude': 50.6673828, 'longitude': -4.7416664}, 'orientation':
# 'PORTRAIT', 'yearlyEnergyDcKwh': 321.63647, 'segmentIndex': 1}, {'center': {'latitude': 50.667378, 'longitude':
# -4.741653599999999}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 321.46332, 'segmentIndex': 1}, {'center': {
# 'latitude': 50.667373299999994, 'longitude': -4.7416409}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 339.3016,
# 'segmentIndex': 1}, {'center': {'latitude': 50.6673853, 'longitude': -4.7416298}, 'orientation': 'PORTRAIT',
# 'yearlyEnergyDcKwh': 327.26282, 'segmentIndex': 1}, {'center': {'latitude': 50.667399499999995, 'longitude':
# -4.741668}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.9878, 'segmentIndex': 1}, {'center': {'latitude':
# 50.6673948, 'longitude': -4.7416553}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.16364, 'segmentIndex':
# 1}, {'center': {'latitude': 50.667390000000005, 'longitude': -4.7416425}, 'orientation': 'PORTRAIT',
# 'yearlyEnergyDcKwh': 310.3404, 'segmentIndex': 1}, {'center': {'latitude': 50.6674186, 'longitude': -4.7417191},
# 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 278.3281, 'segmentIndex': 1}]}, 'boundingBox': {'sw': {'latitude':
# 50.6672904, 'longitude': -4.741778}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415536}},
# 'imageryQuality': 'MEDIUM', 'imageryProcessedDate': {'year': 2024, 'month': 4, 'day': 18}}

3
backend/app/db/functions/portfolio_functions.py
View file

@ -4,7 +4,7 @@ from backend.app.db.models.portfolio import Portfolio
def aggregate_portfolio_recommendations(
session, portfolio_id: int, total_valuation_increase: float, labour_days: float
session, portfolio_id: int, total_valuation_increase: float, labour_days: float, aggregated_data: dict
):
# Aggregate multiple fields
aggregates = (
@ -27,6 +27,7 @@ def aggregate_portfolio_recommendations(
"energy_savings": aggregates.energy_savings or 0,
"co2_equivalent_savings": aggregates.co2_equivalent_savings or 0,
"energy_cost_savings": aggregates.energy_cost_savings or 0,
**aggregated_data
}
# Get the portfolio and update the fields

15
backend/app/db/models/portfolio.py
View file

@ -45,6 +45,21 @@ class Portfolio(Base):
labour_days = Column(Float)
created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
# Aggregations for summary
epc_breakdown_pre_retrofit = Column(Text)
epc_breakdown_post_retrofit = Column(Text)
n_units_to_retrofit = Column(Integer)
co2_per_unit_pre_retrofit = Column(Text)
co2_per_unit_post_retrofit = Column(Text)
energy_bill_per_unit_pre_retrofit = Column(Text)
energy_bill_per_unit_post_retrofit = Column(Text)
energy_consumption_per_unit_pre_retrofit = Column(Text)
energy_consumption_per_unit_post_retrofit = Column(Text)
valuation_improvement_per_unit = Column(Text)
cost_per_unit = Column(Text)
cost_per_co2_saved = Column(Text)
cost_per_sap_point = Column(Text)
valuation_return_on_investment = Column(Text)
class PropertyCreationStatus(enum.Enum):

490
backend/app/plan/router.py
View file

@ -1,3 +1,4 @@
import json
from datetime import datetime
from tqdm import tqdm
@ -34,6 +35,7 @@ from recommendations.optimiser.CostOptimiser import CostOptimiser
from recommendations.optimiser.GainOptimiser import GainOptimiser
from recommendations.optimiser.optimiser_functions import prepare_input_measures
from recommendations.Recommendations import Recommendations
from recommendations.Mds import Mds
from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3
from backend.ml_models.Valuation import PropertyValuation
@ -51,12 +53,170 @@ def patch_epc(patch, epc_records):
"""
for patch_variable, patch_value in patch.items():
if patch_variable in ["address", "postcode"]:
continue
if patch_value == "":
continue
if patch_variable in epc_records["original_epc"]:
epc_records["original_epc"][patch_variable] = patch_value
return epc_records
def extract_portfolio_aggregation_data(
input_properties, total_valuation_increase, recommendations, new_epc_bands, property_value_increase_ranges
):
# We aggregate a number of metrics for the portfolio:
# 1) A breakdown of the number of properties in each EPC band
# a) before retrofit
# b) after retrofit
# 2) Number of units
# 3) Co2/unit
# a) before retrofit
# b) after retrofit
# 4) Energy bill/unit
# a) before retrofit
# b) after retrofit
# 5) Average valuation improvement/unit
# 6) Total cost
# 7) Cost per unit
# 8) £ per CO2 saved
# 9) £ per SAP point
# We need to construct the underlyind data for this
# Helper function to reformat the EPC data
def reformat_epc_data(epc_counts):
# Define all possible EPC bands in the required order
epc_bands = ["G", "F", "E", "D", "C", "B", "A"]
# Create the formatted data list by checking each band in the order
formatted_data = []
for band in epc_bands:
# Get the count from the dictionary, defaulting to 0 if not present
count = epc_counts.get(band, 0)
# Append the formatted dictionary to the list
formatted_data.append({"name": band, band: count})
return formatted_data
n_units = len(input_properties)
agg_data = []
for p in input_properties:
# Get the recommendations for the property - we include all properties, even ones without recommendations
property_recommendations = recommendations.get(p.id, [])
# Get just the default recommendations
default_recommendations = [r for r in property_recommendations if r["default"]]
has_recommendations = len(default_recommendations) > 0
# We can now calculate multiple outputs based on default recommendations
carbon_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations])
pre_retrofit_co2 = p.data["co2-emissions-current"]
post_retrofit_co2 = pre_retrofit_co2 - carbon_savings
pre_retrofit_energy_bill = p.current_energy_bill
post_retrofit_energy_bill = p.current_energy_bill - sum(
[r["energy_cost_savings"] for r in default_recommendations]
)
pre_retrofit_energy_consumption = p.current_adjusted_energy
post_retrofit_energy_consumption = p.current_adjusted_energy - sum(
[r["adjusted_heat_demand"] for r in default_recommendations]
)
# Add up energy savings
cost = sum([r["total"] for r in default_recommendations])
sap_point_improvement = sum([r["sap_points"] for r in default_recommendations])
lower_bound_valuation_uplift = (
property_value_increase_ranges[p.id]["lower_bound_increased_value"] -
property_value_increase_ranges[p.id]["current_value"]
)
upper_bound_valuation_uplift = (
property_value_increase_ranges[p.id]["upper_bound_increased_value"] -
property_value_increase_ranges[p.id]["current_value"]
)
agg_data.append({
"pre_retrofit_epc": p.data["current-energy-rating"],
"post_retrofit_epc": new_epc_bands[p.id],
"pre_retrofit_co2": pre_retrofit_co2,
"post_retrofit_co2": post_retrofit_co2,
"pre_retrofit_energy_bill": pre_retrofit_energy_bill,
"post_retrofit_energy_bill": post_retrofit_energy_bill,
"pre_retrofit_energy_consumption": pre_retrofit_energy_consumption,
"post_retrofit_energy_consumption": post_retrofit_energy_consumption,
"cost": cost,
"sap_point_improvement": sap_point_improvement,
"lower_bound_valuation_uplift": lower_bound_valuation_uplift,
"upper_bound_valuation_uplift": upper_bound_valuation_uplift,
"has_recommendations": has_recommendations
})
agg_data = pd.DataFrame(agg_data)
n_units_to_retrofit = agg_data["has_recommendations"].sum()
valuation_improvement_lower_bound_per_unit = (
agg_data["lower_bound_valuation_uplift"].mean()
)
valuation_improvement_upper_bound_per_unit = (
agg_data["upper_bound_valuation_uplift"].mean()
)
total_carbon_saved = agg_data["pre_retrofit_co2"].sum() - agg_data["post_retrofit_co2"].sum()
total_sap_points = agg_data["sap_point_improvement"].sum()
def format_money(amount):
return f"£{amount:,.0f}"
valuation_improvment_per_unit = str(
format_money(
total_valuation_increase / n_units) + (f" ({format_money(valuation_improvement_lower_bound_per_unit)} - "
f"{format_money(valuation_improvement_upper_bound_per_unit)})")
)
valuation_return_on_investment = str(
str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) +
f" ("
f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - "
f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})"
)
aggregation_data = {
"epc_breakdown_pre_retrofit": json.dumps(
reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict())
),
"epc_breakdown_post_retrofit": json.dumps(
reformat_epc_data(agg_data["post_retrofit_epc"].value_counts().to_dict())
),
"number_of_properties": int(n_units),
"n_units_to_retrofit": int(n_units_to_retrofit),
"co2_per_unit_pre_retrofit": str(round(agg_data["pre_retrofit_co2"].mean(), 2)) + "t",
"co2_per_unit_post_retrofit": str(round(agg_data["post_retrofit_co2"].mean(), 2)) + "t",
"energy_bill_per_unit_pre_retrofit": format_money(agg_data["pre_retrofit_energy_bill"].mean()),
"energy_bill_per_unit_post_retrofit": format_money(agg_data["post_retrofit_energy_bill"].mean()),
"energy_consumption_per_unit_pre_retrofit": str(
round(agg_data["pre_retrofit_energy_consumption"].mean())) + "kWh",
"energy_consumption_per_unit_post_retrofit": str(
round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh",
"valuation_improvement_per_unit": valuation_improvment_per_unit,
"cost_per_unit": format_money(agg_data["cost"].mean()),
"cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved),
"cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points),
"valuation_return_on_investment": valuation_return_on_investment,
# TODO: Could we add 10yr carbon credits value?
}
return aggregation_data
router = APIRouter(
prefix="/plan",
tags=["plan"],
@ -91,6 +251,12 @@ async def trigger_plan(body: PlanTriggerRequest):
bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.already_installed_file_path
)
non_invasive_recommendations = []
if body.non_invasive_recommendations_file_path:
non_invasive_recommendations = read_csv_from_s3(
bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.non_invasive_recommendations_file_path
)
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet",
)
@ -107,9 +273,12 @@ async def trigger_plan(body: PlanTriggerRequest):
postcode=config["postcode"],
uprn=uprn,
auth_token=get_settings().EPC_AUTH_TOKEN,
os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY
os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY,
)
epc_searcher.find_property()
epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None)
epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None)
# For the moment, our OS API access is unavailable, so we skip and interpolate
epc_searcher.find_property(skip_os=True)
# Create a record in db
property_id, is_new = create_property(
session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn
@ -146,6 +315,12 @@ async def trigger_plan(body: PlanTriggerRequest):
x for x in already_installed if
(x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
), {})
property_non_invasive_recommendations = next((
x for x in non_invasive_recommendations if
(x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
), {})
input_properties.append(
Property(
id=property_id,
@ -153,6 +328,7 @@ async def trigger_plan(body: PlanTriggerRequest):
postcode=epc_searcher.postcode_clean,
epc_record=prepared_epc,
already_installed=property_already_installed,
non_invasive_recommendations=property_non_invasive_recommendations,
**Property.extract_kwargs(config)
)
)
@ -205,6 +381,7 @@ async def trigger_plan(body: PlanTriggerRequest):
logger.info("Preparing data for scoring in sap change api")
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
recommendations_scoring_data = recommendations_scoring_data.drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
@ -243,7 +420,13 @@ async def trigger_plan(body: PlanTriggerRequest):
property_instance = [p for p in input_properties if p.id == property_id][0]
recommendations_with_impact, current_adjusted_energy, expected_adjusted_energy = (
(
recommendations_with_impact,
current_adjusted_energy,
expected_adjusted_energy,
current_energy_bill,
expected_energy_bill
) = (
Recommendations.calculate_recommendation_impact(
property_instance=property_instance,
all_predictions=all_predictions,
@ -254,7 +437,9 @@ async def trigger_plan(body: PlanTriggerRequest):
# Store the resulting adjusted energy in the property instance
property_instance.set_adjusted_energy(
current_adjusted_energy=current_adjusted_energy,
expected_adjusted_energy=expected_adjusted_energy
expected_adjusted_energy=expected_adjusted_energy,
current_energy_bill=current_energy_bill,
expected_energy_bill=expected_energy_bill
)
input_measures = prepare_input_measures(recommendations_with_impact, body.goal)
@ -316,6 +501,8 @@ async def trigger_plan(body: PlanTriggerRequest):
logger.info("Uploading recommendations to the database")
property_valuation_increases = []
session.commit()
new_epc_bands = {}
property_value_increase_ranges = {}
for i in range(0, len(input_properties), BATCH_SIZE):
try:
# Take a slice of the input_properties list to make a batch
@ -327,8 +514,10 @@ async def trigger_plan(body: PlanTriggerRequest):
total_sap_points = sum([r["sap_points"] for r in default_recommendations])
new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points
new_epc = sap_to_epc(new_sap_points)
new_epc_bands[p.id] = new_epc
valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc)
property_value_increase_ranges[p.id] = valuations
# Your existing operations
property_details_epc = p.get_property_details_epc(
@ -392,11 +581,20 @@ async def trigger_plan(body: PlanTriggerRequest):
[sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()]
))
aggregated_data = extract_portfolio_aggregation_data(
input_properties=input_properties,
total_valuation_increase=total_valuation_increase,
recommendations=recommendations,
new_epc_bands=new_epc_bands,
property_value_increase_ranges=property_value_increase_ranges
)
aggregate_portfolio_recommendations(
session,
portfolio_id=body.portfolio_id,
total_valuation_increase=total_valuation_increase,
labour_days=labour_days
labour_days=labour_days,
aggregated_data=aggregated_data
)
# Commit final changes
@ -421,3 +619,285 @@ async def trigger_plan(body: PlanTriggerRequest):
session.close()
return Response(status_code=200)
@router.post("/mds")
async def build_mds(body: PlanTriggerRequest):
# TODO: This is a placeholder location for the MDS endpoint, which this is being assembled
logger.info("Connecting to db")
session = sessionmaker(bind=db_engine)()
created_at = datetime.now().isoformat()
try:
session.begin()
logger.info("Getting the inputs")
plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path)
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet",
)
input_properties = []
for property_id, config in tqdm(enumerate(plan_input), total=len(plan_input)):
# We validate each record in the file. If the record is NOT valid, we need to handle this accordingly
uprn = config.get("uprn", None)
uprn = None if uprn == "" else uprn
if uprn:
uprn = int(float(uprn))
epc_searcher = SearchEpc(
address1=config["address"],
postcode=config["postcode"],
uprn=uprn,
auth_token=get_settings().EPC_AUTH_TOKEN,
os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY,
)
epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None)
epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None)
# For the moment, our OS API access is unavailable, so we skip and interpolate
epc_searcher.find_property(skip_os=True)
if config["address"] == "35b High Street":
print("Performing temporary patch")
epc_searcher.newest_epc["uprn"] = 10002911892
epc_searcher.full_sap_epc["uprn"] = 10002911892
# Create a record in db
# TODO: If we productionise the creation of this mds report, we will need to store this in the db
# property_id, is_new = create_property(
# session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn
# )
# if not is_new:
# continue
#
# create_property_targets(
# session,
# property_id=property_id,
# portfolio_id=body.portfolio_id,
# epc_target=body.goal_value,
# heat_demand_target=None
# )
epc_records = {
'original_epc': epc_searcher.newest_epc.copy(),
'full_sap_epc': epc_searcher.full_sap_epc.copy(),
'old_data': epc_searcher.older_epcs.copy(),
}
# patch = next((
# x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
# ), {})
# epc_records = patch_epc(patch, epc_records)
prepared_epc = EPCRecord(
epc_records=epc_records,
run_mode="newdata",
cleaning_data=cleaning_data
)
# property_already_installed = next((
# x for x in already_installed if
# (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
# ), {})
#
# property_non_invasive_recommendations = next((
# x for x in non_invasive_recommendations if
# (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
# ), {})
measures = config["measures"] if "measures" in config else None
input_properties.append(
Property(
id=property_id,
address=epc_searcher.address_clean,
postcode=epc_searcher.postcode_clean,
epc_record=prepared_epc,
# already_installed=property_already_installed,
# non_invasive_recommendations=property_non_invasive_recommendations,
measures=measures,
**Property.extract_kwargs(config)
)
)
logger.info("Reading in materials and cleaned datasets")
materials = get_materials(session)
cleaned = get_cleaned()
uprn_filenames = read_dataframe_from_s3_parquet(
bucket_name=get_settings().DATA_BUCKET, file_key="spatial/filename_meta.parquet"
)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket=get_settings().DATA_BUCKET)
logger.info("Getting spatial data")
for p in tqdm(input_properties):
p.get_spatial_data(uprn_filenames)
logger.info("Getting components and epc recommendations")
recommendations_scoring_data = []
representative_recommendations = {}
for p in tqdm(input_properties):
p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
mds = Mds(property_instance=p, materials=materials)
property_representative_recommendations, errors = mds.build()
if errors:
logger.info("Errors occurred during MDS build")
representative_recommendations[p.id] = property_representative_recommendations
# Build the scoring data
p.create_base_difference_epc_record(cleaned_lookup=cleaned)
recommendations_scoring_data.append(
p.simulate_all_representative_recommendations(property_representative_recommendations)
)
logger.info("Preparing data for scoring in sap change api")
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
recommendations_scoring_data = recommendations_scoring_data.drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at)
all_predictions = {
"sap_change_predictions": pd.DataFrame(),
"heat_demand_predictions": pd.DataFrame(),
"carbon_change_predictions": pd.DataFrame()
}
to_loop_over = range(0, recommendations_scoring_data.shape[0], SCORING_BATCH_SIZE)
for chunk in tqdm(to_loop_over, total=len(to_loop_over)):
predictions_dict = model_api.predict_all(
df=recommendations_scoring_data.iloc[chunk:chunk + SCORING_BATCH_SIZE],
bucket=get_settings().DATA_BUCKET,
prediction_buckets={
"sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET,
"heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET,
"carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET
}
)
# Append the predictions to the predictions dictionary
for key, scored in predictions_dict.items():
all_predictions[key] = pd.concat([all_predictions[key], scored])
# We now produce a table of results for the mds report
# TODO: TEMP
for p in plan_input:
if p["uprn"]:
p["uprn"] = str(int(float(p["uprn"])))
results = []
for p in input_properties:
measures = p.measures
property_recommendations = [r['type'] for r in representative_recommendations[p.id]]
# TODO: Check high heat retention storage heaters - looks like it's excluded controls!
sap_prediction = all_predictions["sap_change_predictions"][
all_predictions["sap_change_predictions"]["property_id"] == str(p.id)
]
heat_demand_prediction = all_predictions["heat_demand_predictions"][
all_predictions["heat_demand_predictions"]["property_id"] == str(p.id)
]
carbon_prediction = all_predictions["carbon_change_predictions"][
all_predictions["carbon_change_predictions"]["property_id"] == str(p.id)
]
# Get a before and after for SAP, heat demand, CO2 and also calculate energy bill and energy savings
sap_before = int(p.data["current-energy-efficiency"])
sap_after = sap_prediction["predictions"].values[0] if measures else sap_before
epc_before = p.data["current-energy-rating"]
epc_after = sap_to_epc(sap_after) if measures else epc_before
heat_demand_before = p.data["energy-consumption-current"]
heat_demand_after = heat_demand_prediction["predictions"].values[0] if measures else heat_demand_before
carbon_before = p.data["co2-emissions-current"]
carbon_after = carbon_prediction["predictions"].values[0] if measures else carbon_before
# Estimate bill savings
from backend.ml_models.AnnualBillSavings import AnnualBillSavings
current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
epc_energy_consumption=heat_demand_before * p.floor_area,
current_epc_rating=epc_before,
)
# TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are
# actually implemented
expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
epc_energy_consumption=heat_demand_after * p.floor_area,
current_epc_rating=epc_before,
)
# TODO: We should determine if the home is gas & electricity or just electricity
current_energy_bill = AnnualBillSavings.calculate_annual_bill(
current_adjusted_energy,
)
expected_energy_bill = AnnualBillSavings.calculate_annual_bill(
expected_adjusted_energy,
)
bill_savings = current_energy_bill - expected_energy_bill
energy_savings = current_adjusted_energy - expected_adjusted_energy
config = [c for c in plan_input if c["uprn"] == str(p.uprn)]
if not config:
config = {"address": None, "postcode": None}
else:
config = config[0]
to_append = {
"config_address": config["address"],
"config_postcode": config["postcode"],
"address": p.address,
"postcode": p.postcode,
"measures": measures,
"property_recommendations": property_recommendations,
"year_of_epc": p.data['lodgement-date'],
"sap_before": sap_before,
"sap_after": sap_after,
"epc_before": epc_before,
"epc_after": epc_after,
"heat_demand_before": heat_demand_before,
"heat_demand_after": heat_demand_after,
"carbon_before": carbon_before,
"carbon_after": carbon_after,
"bill_savings": bill_savings,
"energy_savings": energy_savings,
}
results.append(to_append)
results = pd.DataFrame(results)
results["sap_uplift"] = results["sap_after"] - results["sap_before"]
except IntegrityError:
logger.error("Database integrity error occurred", exc_info=True)
session.rollback()
return Response(status_code=500, content="Database integrity error.")
except OperationalError:
logger.error("Database operational error occurred", exc_info=True)
session.rollback()
return Response(status_code=500, content="Database operational error.")
except ValueError:
logger.error("Value error - possibly due to malformed data", exc_info=True)
session.rollback()
return Response(status_code=400, content="Bad request: malformed data.")
except Exception as e: # General exception handling
logger.error(f"An error occurred: {e}")
session.rollback()
return Response(status_code=500, content="An unexpected error occurred.")
finally:
session.close()

1
backend/app/plan/schemas.py
View file

@ -11,6 +11,7 @@ class PlanTriggerRequest(BaseModel):
trigger_file_path: str
already_installed_file_path: Optional[str] = None
patches_file_path: Optional[str] = None
non_invasive_recommendations_file_path: Optional[str] = None
exclusions: Optional[conlist(str, min_items=1)] = None
# Pre-defined list of possibilities for exclusions

9
backend/ml_models/AnnualBillSavings.py
View file

@ -43,15 +43,20 @@ class AnnualBillSavings:
return cls.ELECTRICITY_PRICE_CAP * kwh
@classmethod
def calculate_annual_bill(cls, kwh):
def calculate_annual_bill(cls, kwh, mains_gas=True):
"""
This method will estimate the total annual bill for a property
It assumed gas & electricity are used
:param kwh: The total kwh consumption
:param mains_gas: Whether the property uses mains gas
:return: An estimate for annual bill
"""
return cls.PRICE_FACTOR * kwh + (cls.DAILY_STANDARD_CHARGE_GAS + cls.DAILY_STANDARD_CHARGE_ELECTRICITY * 365)
if mains_gas:
return cls.PRICE_FACTOR * kwh + (
cls.DAILY_STANDARD_CHARGE_GAS + cls.DAILY_STANDARD_CHARGE_ELECTRICITY * 365)
return cls.ELECTRICITY_PRICE_CAP * kwh + (cls.DAILY_STANDARD_CHARGE_ELECTRICITY * 365)
@classmethod
def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating):

62
backend/ml_models/Valuation.py
View file

@ -63,6 +63,33 @@ class PropertyValuation:
90093693: 279_000, # Based on Zoopla
90055152: 149_000, # Based on Zoopla
90028499: 238_000, # Based on Zoopla
# IMMO Dudley Pilot 2- search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
90039318: 177_000, # Based on Zoopla
90038384: 170_000, # Based on Zoopla
90105380: 185_000, # Based on Zoopla
90124001: 165_000, # Based on Zoopla
90013980: 148_000, # Based on Zoopla
90087154: 184_000, # Based on Zoopla
90046817: 167_000, # Based on Zoopla
# Goldman Sachs Pilot for inrto - search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
100070358888: 153_000, # Based on Zoopla
10090436544: 282_000, # Based on Zoopla
100070365751: 177_000, # Based on Zoopla
10095952767: 168_000, # Based on Zoopla
100070520130: 177_000, # Based on Zoopla
100070333957: 185_000, # Based on Zoopla
100070543258: 211_000, # Based on Zoopla
# Vander Elliot Pilot - search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
41018850: 104_000, # Based on Zoopla
38237316: 74_000, # Based on Zoopla
38237317: 74_000, # Based on Zoopla
41052320: 70_000, # Based on Zoopla
41052321: 70_000, # Based on Zoopla
41052322: 38_000, # Based on Zoopla
41222759: 38_000, # Based on Zoopla
41222760: 46_000, # Based on Zoopla
41222761: 270_000, # Based on Zoopla
41212534: 38_000, # Based on Zoopla
}
# We base our valuation uplifts on a number of sources
@ -100,6 +127,33 @@ class PropertyValuation:
# {"start": "D", "end": "A", "increase_percentage": 0.017},
]
# Found here: https://www.rightmove.co.uk/news/articles/property-news/green-premium-epc-ratings/
# F -> C is + 15%
# E -> C is +7%
# D -> C is +3%
RIGHTMOVE_MAPPING = [
{"start": "G", "end": "C", "increase_percentage": 0.15},
{"start": "G", "end": "B", "increase_percentage": 0.15},
{"start": "G", "end": "A", "increase_percentage": 0.15},
{"start": "F", "end": "C", "increase_percentage": 0.15},
{"start": "F", "end": "B", "increase_percentage": 0.15},
{"start": "F", "end": "A", "increase_percentage": 0.15},
{"start": "E", "end": "C", "increase_percentage": 0.07},
{"start": "E", "end": "B", "increase_percentage": 0.07},
{"start": "E", "end": "A", "increase_percentage": 0.07},
{"start": "D", "end": "C", "increase_percentage": 0.03},
{"start": "D", "end": "B", "increase_percentage": 0.03},
{"start": "D", "end": "A", "increase_percentage": 0.03},
]
# Additional sources:
# https://superhomes.org.uk/wp-content/uploads/2024/05/The-Impact-of-Retrofit-on-Residential-Property-Market
# -Values-7-rotated-1.pdf
EPC_BANDS = ["G", "F", "E", "D", "C", "B", "A"]
@classmethod
@ -151,14 +205,18 @@ class PropertyValuation:
msm_increase, lloyds_increase = cls.get_increase(epc_band_range)
# We now use the knight frank and nationwide data to get further valuation evidence, if we have it
# We now use the knight frank, nationwide and Rightmove data to get further valuation evidence, if we have it
kf_increase = [x for x in cls.KNIGHT_FRANK_MAPPING if x["start"] == current_epc and x["end"] == target_epc]
nw_increase = [x for x in cls.NATIONWIDE_MAPPING if x["start"] == current_epc and x["end"] == target_epc]
rm_increase = [x for x in cls.RIGHTMOVE_MAPPING if x["start"] == current_epc and x["end"] == target_epc]
kf_increase = kf_increase[0]["increase_percentage"] if kf_increase else None
nw_increase = nw_increase[0]["increase_percentage"] if nw_increase else None
rm_increase = rm_increase[0]["increase_percentage"] if rm_increase else None
all_increases = [x for x in [msm_increase, lloyds_increase, kf_increase, nw_increase] if x is not None]
all_increases = [
x for x in [msm_increase, lloyds_increase, kf_increase, nw_increase, rm_increase] if x is not None
]
max_increase = max(all_increases)
min_increase = min(all_increases)

13
backend/ml_models/api.py
View file

@ -99,6 +99,13 @@ class ModelApi:
# depending on how you want to handle errors in your application
return None
@staticmethod
def extract_phase(recommendation_id):
if 'phase=' in recommendation_id:
return int(recommendation_id.split('phase=')[1][0])
else:
return None
def predict_all(self, df, bucket, prediction_buckets) -> dict:
"""
@ -135,9 +142,11 @@ class ModelApi:
# To grab the phase, we pull the integer after "phase=" in the recommendation_id. We can do this with a
# string split on phase= and then grab the second element of the resulting list. We could also use a
# regular expression to do this but we use the string split method here, for safety.
predictions_df['phase'] = predictions_df['recommendation_id'].str.split('phase=').str[1].str[0]
# We may not always have a phase to split on, so we need to handle this case. We can do this by using the
# str[1] method to grab the second element of the resulting list. We then grab the first character of this
# string to get the phase. We then convert this to an integer.
# Convert back to int
predictions_df['phase'] = predictions_df['phase'].astype(int)
predictions_df['phase'] = predictions_df['recommendation_id'].apply(self.extract_phase)
predictions[model_prefix] = predictions_df

46
etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py
View file

@ -21,6 +21,8 @@ class AirSourceHeatPumpEfficiency:
def create_dataset(self):
logger.info("Creating solar photo supply dataset")
all_counts = []
for dir in tqdm(self.file_directories):
filepath = dir / "certificates.csv"
df = pd.read_csv(filepath, low_memory=False)
@ -44,9 +46,15 @@ class AirSourceHeatPumpEfficiency:
df = df[
df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False)
]
# Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
df = df[~pd.isnull(df[col])]
# Get the columns we're interested in
df = df[
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
@ -60,6 +68,8 @@ class AirSourceHeatPumpEfficiency:
counts = df.groupby(
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
@ -71,8 +81,34 @@ class AirSourceHeatPumpEfficiency:
]
).size().reset_index(name="count")
# Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
df = df[~pd.isnull(df[col])]
# Take newest LODGEMENT_DATE per UPRN
df = df.sort_values(by="LODGEMENT_DATE", ascending=False).drop_duplicates(subset=["UPRN"])
all_counts.append(counts)
all_counts = pd.concat(all_counts)
all_counts_agg = all_counts.groupby(
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
"MAINHEATC_ENERGY_EFF",
"MAIN_FUEL",
"HOTWATER_DESCRIPTION",
"HOT_WATER_ENERGY_EFF",
"MAINS_GAS_FLAG"
]
)["count"].sum().reset_index()
all_counts_agg.groupby("PROPERTY_TYPE")["count"].sum()
# In houses, 68% of the cases where we see air source heat pumps are in detached and semi-detached houses
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "House"]["BUILT_FORM"].value_counts(normalize=True)
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Flat"]["BUILT_FORM"].value_counts()
# In Bungalows, 74% of cases where we see air source heat pumps are in detached and semi-detached houses
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Bungalow"]["BUILT_FORM"].value_counts(normalize=True)
# TODO: Research options for mid and end-terrace houses
# TODO: Research the options for flats - we see them appear in flats, but practically speaking, how does the
# install process work?

271
etl/customers/eon/pilot_asset_list.py Normal file
View file

@ -0,0 +1,271 @@
import time
import pandas as pd
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from dotenv import load_dotenv
import os
from tqdm import tqdm
from utils.s3 import save_csv_to_s3
# Read in the .env file in backend
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
# Stored in my notes
ORDNANCE_SURVEY_API_KEY = ""
PORTFOLIO_ID = 80
USER_ID = 8
def extract_mds_measures(config):
measures = []
if not pd.isnull(config["EWI (Trad Const)"]):
measures.append({"external_wall_insulation": "EWI (Trad Const)"})
if not pd.isnull(config["EWI (Non Trad Const)"]):
measures.append({"external_wall_insulation": "EWI (Non Trad Const)"})
if not pd.isnull(config["CWI"]):
measures.append({"cavity_wall_insulation": "CWI"})
if not pd.isnull(config["LI"]):
measures.append({"loft_insulation": "LI"})
if not pd.isnull(config["Party Wall Insu"]):
measures.append({"party_wall_insulation": "Party Wall Insu"})
if not pd.isnull(config["IWI (POA - Prov Sum Only)"]):
measures.append({"internal_wall_insulation": "IWI (POA - Prov Sum Only)"})
if not pd.isnull(config["U/F Insu (Manual install)"]):
measures.append({"suspended_floor_insulation": "U/F Insu (Manual install)"})
if not pd.isnull(config["U/F insu (Qbot)"]):
measures.append({"suspended_floor_insulation": "U/F insu (Qbot)"})
if not pd.isnull(config["Solid floor insl (Out of scope - Prov sum only)"]):
measures.append({"solid_floor_insulation": "Solid floor insl (Out of scope - Prov sum only)"})
if not pd.isnull(config["ASHP Htg"]):
measures.append({"air_source_heat_pump": "ASHP Htg"})
if not pd.isnull(config["GSHP Htg"]):
measures.append({"ground_source_heat_pump": "GSHP Htg"})
if not pd.isnull(config["Shared ground loops"]):
measures.append({"shared_ground_loops": "Shared ground loops"})
if not pd.isnull(config["Communal heat networks"]):
measures.append({"communal_heat_networks": "Communal heat networks"})
if not pd.isnull(config["District heating networks"]):
measures.append({"district_heating_networks": "District heating networks"})
if not pd.isnull(config["Elec Storage Htrs (Out of scope -Prov sum only)"]):
measures.append({"electric_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)"})
if not pd.isnull(config["Low Energy Bulbs"]):
measures.append({"low_energy_lighting": "Low Energy Bulbs"})
if not pd.isnull(config["Cyl Insulation"]):
measures.append({"cylinder_insulation": "Cyl Insulation"})
if not pd.isnull(config["Smart controls"]):
measures.append({"smart_controls": "Smart controls"})
if not pd.isnull(config["Zone controls"]):
measures.append({"zone_controls": "Zone controls"})
if not pd.isnull(config["Upgrade TRV's"]):
measures.append({"trvs": "Upgrade TRV's"})
if not pd.isnull(config["Solar PV"]):
measures.append({"solar_pv": "Solar PV"})
if not pd.isnull(config["Solar Thermal"]):
measures.append({"solar_thermal": "Solar Thermal"})
if not pd.isnull(config["Double Glazing (POA - Prov sum only)"]):
measures.append({"double_glazing": "Double Glazing (POA - Prov sum only)"})
if not pd.isnull(config["Draught Proofing"]):
measures.append({"draught_proofing": "Draught Proofing"})
if not pd.isnull(config["Ventilation upgrade"]):
measures.append({"mechanical_ventilation": "Ventilation upgrade"})
if not pd.isnull(config["Gas Boiler Replacement"]):
measures.append({"gas_boiler": "Gas Boiler Replacement"})
if not pd.isnull(config["Flat roof (Out of scope - prov sum only)"]):
measures.append({"flat_roof_insulation": "Flat roof (Out of scope - prov sum only)"})
if not pd.isnull(config["RIR (POA - Prov sum only)"]):
measures.append({"room_in_roof_insulation": "RIR (POA - Prov sum only)"})
if not pd.isnull(config["EV Charging"]):
measures.append({"ev_charging": "EV Charging"})
if not pd.isnull(config["Battery"]):
measures.append({"battery": "Battery"})
return measures
def parse_property_type(config):
# This should come from the ordnance survey api eventually
# array(['Detached', 'Semi-detached', 'Bungalow', 'Mid Terrace',
# 'End Terrace', 'Top Flat', 'Mid Flat',
# 'Low rise flat (1-2 storey)', nan], dtype=object)
if config["Address"] == "Flat Central Garage":
return {"property_type": "Bungalow", "built_form": "Mid-Terrace"}
if pd.isnull(config["Property Type"]):
return {"property_type": None, "built_form": None}
lookup = {
"Detached": {"property_type": "House", "built_form": "Detached"},
"Semi-detached": {"property_type": "House", "built_form": "Semi-detached"},
"Bungalow": {"property_type": "Bungalow", "built_form": "Detached"},
"Mid Terrace": {"property_type": "House", "built_form": "Mid-Terrace"},
"End Terrace": {"property_type": "House", "built_form": "End-Terrace"},
"Top Flat": {"property_type": "Flat", "built_form": None},
"Mid Flat": {"property_type": "Flat", "built_form": None},
"Low rise flat (1-2 storey)": {"property_type": "Flat", "built_form": None},
}
return lookup[config["Property Type"]]
def app():
"""
Create the initial asset list for the E.ON pilot
:return:
"""
raw_asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/E.ON/sample SHDF Information MDS Template Vr3.0.xlsx",
header_row=11,
drop_all_na=False
)
# Keep just the columns we need
raw_asset_list_base = raw_asset_list[
[
"Address", "Postcode", "No Bedrooms"
]
].copy().rename(
columns={
"Address": "address",
"Postcode": "postcode",
"No Bedrooms": "n_bedrooms"
}
)
# For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free
# trial with Ordnance Survey with my personal account as a temporary solution.
# Let's just pull the full EPC data for this
asset_list_with_uprn = []
for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]):
if row <= 104:
continue
time.sleep(1.1)
searcher = SearchEpc(
address1=property_meta["address"],
postcode=property_meta["postcode"],
auth_token=EPC_AUTH_TOKEN,
os_api_key=ORDNANCE_SURVEY_API_KEY,
full_address=", ".join([property_meta["address"], property_meta["postcode"]])
)
# Let's just find the UPRN
searcher.ordnance_survey_client.get_places_api()
uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"]
# searcher.find_property(skip_os=False)
asset_list_with_uprn.append(
{
**property_meta,
"uprn": uprn,
}
)
# Store this as a backup
# import pandas as pd
# asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn)
# asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False)
# Read in
# asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records")
# Store the asset list and create the portfolio payload
asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn)
asset_list_with_uprn_df["uprn"] = asset_list_with_uprn_df["uprn"].astype(str).astype(int)
# We now determine which measures we need for each property
finalised_asset_list = []
for i, config in raw_asset_list.iterrows():
asset_config = asset_list_with_uprn_df[
(asset_list_with_uprn_df["address"] == config["Address"]) &
(asset_list_with_uprn_df["postcode"] == config["Postcode"])
]
if asset_config.shape[0] != 1:
raise ValueError("Could not find a unique match for the property")
measures = extract_mds_measures(config)
# Get the property type
pt = parse_property_type(config)
if config["Address"] in [
"28 Hermitage Lane",
"35a High Street",
"35b High Street",
"Flat Over 20 Holborough Road",
"Flat above 7 Malling Road"
]:
print(config["Address"])
uprn = None
else:
uprn = asset_config["uprn"].values[0]
finalised_asset_list.append(
{
"address": config["Address"],
"postcode": config["Postcode"],
"uprn": uprn,
"n_bedrooms": config["No Bedrooms"],
"measures": measures,
**pt
}
)
finalised_asset_list = pd.DataFrame(finalised_asset_list)
# Store the asset list in s3
filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
save_csv_to_s3(
dataframe=finalised_asset_list,
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# EPC C portoflio
body = {
"portfolio_id": str(PORTFOLIO_ID),
"housing_type": "Social",
"goal": "Increase EPC",
"goal_value": "C",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": "",
"non_invasive_recommendations_file_path": "",
"budget": None,
}
print(body)

3
etl/customers/gla_croydon_demo/asset_list.py
View file

@ -34,9 +34,6 @@ def app():
low_memory=False
)
z = epc_data.groupby(["WALLS_DESCRIPTION", "WALLS_ENERGY_EFF"]).size().reset_index(name="count")
z = z[z["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"]
# Filter on entries where we have a UPRN
epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]

63
etl/customers/goldman/asset_list.py Normal file
View file

@ -0,0 +1,63 @@
import pandas as pd
from utils.s3 import read_excel_from_s3
from utils.s3 import save_csv_to_s3
PORTFOLIO_ID = 75
USER_ID = 8
def app():
asset_list = [
{
"address": "19 Emily Gardens",
"postcode": "B16 0ED",
},
{
"address": "Flat 6 41 Bradford Street",
"postcode": "B5 6HX",
},
{
"address": "197 FIELD LANE",
"postcode": "B32 4HL",
},
{
"address": "FLAT 4 108 SUMMER ROAD",
"postcode": "B23 6DY",
},
{
"address": "1, St. Benedicts Road",
"postcode": "B10 9DP",
},
{
"address": "29 COOKSEY LANE",
"postcode": "B44 9QL",
},
{
"address": "40 TRITTIFORD ROAD",
"postcode": "B13 0HG",
}
]
asset_list = pd.DataFrame(asset_list)
# Store the asset list in s3
filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
save_csv_to_s3(
dataframe=asset_list,
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# EPC C portoflio
body = {
"portfolio_id": str(PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "B",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": "",
"non_invasive_recommendations_file_path": "",
"budget": None,
}
print(body)

25
etl/customers/goldman/epc_f_g_properties.py Normal file
View file

@ -0,0 +1,25 @@
import pandas as pd
def app():
"""
Pulling the list of EPC G & F properties in Birmingham for Goldman Sachs
"""
epc_data = pd.read_csv(
"local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv",
low_memory=False
)
epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
epc_data["UPRN"] = epc_data["UPRN"].astype(int).astype(str)
# Get the newest EPC for each UPRN. We use LODGEMENT_DATE as a proxy for this
epc_data["LODGEMENT_DATETIME"] = pd.to_datetime(epc_data["LODGEMENT_DATETIME"], format='mixed')
epc_data = epc_data.sort_values("LODGEMENT_DATETIME", ascending=False).drop_duplicates("UPRN")
# Get G & F properties
epc_data = epc_data[epc_data["CURRENT_ENERGY_RATING"].isin(["G", "F"])]
# Save as an excel
epc_data.to_excel("Birmingham EPC F & G Properties.xlsx", index=False)

492
etl/customers/goldman/property_ownership.py Normal file
View file

@ -0,0 +1,492 @@
import re
import pandas as pd
from tqdm import tqdm
import Levenshtein
from backend.SearchEpc import SearchEpc
# Average value of a property in the midlands in 2024 was £238,000. Since these are EPC F & G properties, we assume
# £207,000 since they trade at a discount. This is based on the rightmove study where moving from an EPC F/G -> C has a
# +15% impact on valuation and D -> C has a +3% impact on valuation.
# The mode EPC rating is D, so we associate the £238k valuation with an EPC D property
# Therefore value_of_F * 1.15 = value_of_D * 1.03
# Therefore value_of_F = value_of_D * 1.03/1.15 = 238k * (1.03/1.15) = 213165
PROPERTY_VALUE_ESTIMATE = 213_165
def aggregate_matches(matching_lookup, company_ownership, properties):
df = matching_lookup.merge(
company_ownership, how="left", on="Title Number"
).merge(
properties[["UPRN", "LOCAL_AUTHORITY_LABEL"]], how="left", on="UPRN"
)
counts = (
df.groupby(["Company Registration No. (1)", "LOCAL_AUTHORITY_LABEL"])["UPRN"]
.count()
.reset_index(name="number_of_properties")
)
counts = counts.sort_values("number_of_properties", ascending=False)
pivot_counts = counts.pivot_table(
index=["Company Registration No. (1)"], # Rows: companies and proprietors
columns="LOCAL_AUTHORITY_LABEL", # Columns: each local authority
values="number_of_properties", # The counts of properties
fill_value=0 # Fill missing values with 0 (where there are no properties owned)
).reset_index()
total_counts = (
df.groupby(["Company Registration No. (1)"])["UPRN"]
.count()
.reset_index(name="total_number_of_properties")
)
# We have cases where the same company registration number results in the same company name, so we produce a best
# name per company registration number
best_names = (
df.groupby(["Company Registration No. (1)"])["Proprietor Name (1)"]
.first()
.reset_index()
)
total_counts = best_names.merge(
total_counts, how="left", on=["Company Registration No. (1)"]
)
pivot_counts = pivot_counts.merge(
total_counts, how="left", on=["Company Registration No. (1)"]
)
pivot_counts = pivot_counts.sort_values("total_number_of_properties", ascending=False)
pivot_counts["approx_value"] = PROPERTY_VALUE_ESTIMATE * pivot_counts["total_number_of_properties"]
pivot_counts["cumulative_value"] = pivot_counts["approx_value"].cumsum()
return pivot_counts
def find_f_g_properties(paths):
data = []
for path in tqdm(paths):
epc_data = pd.read_csv(path, low_memory=False)
epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
epc_data["UPRN"] = epc_data["UPRN"].astype(int).astype(str)
# Get the newest EPC for each UPRN. We use LODGEMENT_DATE as a proxy for this
epc_data["LODGEMENT_DATETIME"] = pd.to_datetime(epc_data["LODGEMENT_DATETIME"], format='mixed')
epc_data = epc_data.sort_values("LODGEMENT_DATETIME", ascending=False).drop_duplicates("UPRN")
# Get G & F properties
epc_data = epc_data[epc_data["CURRENT_ENERGY_RATING"].isin(["G", "F"])]
data.append(epc_data)
data = pd.concat(data)
# Save as an excel
data.to_excel("EPC F & G Properties.xlsx", index=False)
def remove_text_in_brackets(address: str) -> str:
"""
Removes any text within parentheses, including the parentheses themselves.
Parameters:
- address (str): The address string to clean.
Returns:
- str: The cleaned address with text in parentheses removed.
"""
# Regex to find and remove content in parentheses
cleaned_address = re.sub(r'\s*\([^)]*\)', '', address)
return cleaned_address
def extract_numeric_part(house_number: str) -> str:
"""
Extracts only the numeric part from a house number that may contain letters.
Parameters:
- house_number (str): The house number string possibly containing letters.
Returns:
- str: The numeric part of the house number.
"""
# Use regular expression to replace all non-digit characters with nothing
numeric_part = re.sub(r'\D', '', house_number)
return numeric_part
def levenstein_match(matching_string, df, address_col):
match_to = df[address_col].tolist()
# Strip out punctuation and spaces
match_to = [re.sub(r'[^\w\s]', '', x) for x in match_to]
match_to = [x.replace(" ", "") for x in match_to]
# Perform matching between full key and match_to
distances = [Levenshtein.distance(matching_string, s) for s in match_to]
best_match_index = distances.index(min(distances))
# We might want to consider a threshold for the distance, however for the momeny,
# we don't consider this for the moment
df = df.iloc[best_match_index:best_match_index + 1]
return df
def extract_range_from_house_number(house_number_range: str):
"""
Detects if the house number includes a numeric range (formatted as 'x-y') and extracts all values within this range.
Non-numeric strings containing hyphens are ignored.
Parameters:
- house_number_range (str): The house number string that might contain a range.
Returns:
- list of str: A list of all numbers within the range if it is a range; otherwise, returns None.
"""
if not house_number_range:
return None
if '-' in house_number_range:
parts = house_number_range.split('-')
if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
# Both parts are numeric, so it's a valid range
start, end = map(int, parts) # Convert parts to integers
return [str(x) for x in range(start, end + 1)]
else:
# Not a valid numeric range
return None
else:
# No hyphen present or not a range
return None
def is_in_range(row, house_no):
""" Check if the house number is within the range provided in the row. """
if row and any(house_no == num for num in row):
return True
return False
def remove_duplicate_matches(matching_lookup, properties, company_ownership):
duplicated_titles = matching_lookup[matching_lookup["Title Number"].duplicated()]["Title Number"].unique()
to_drop = []
for dupe_title in duplicated_titles:
dupe_data = matching_lookup[matching_lookup["Title Number"] == dupe_title].copy()
matched_addresses = dupe_data.merge(
properties[["UPRN", "ADDRESS"]].rename(columns={"ADDRESS": "epc_address"}),
how="left", on="UPRN"
).merge(
company_ownership[["Title Number", "Property Address"]],
how="left", on="Title Number"
)
# We perform levenstein to get the best match
best_match = levenstein_match(
matching_string=matched_addresses["Property Address"].values[0],
df=matched_addresses,
address_col="epc_address"
)
matches_to_drop = matched_addresses[
~matched_addresses["UPRN"].isin(best_match["UPRN"].values)
]
to_drop.append(
matches_to_drop[["UPRN", "Title Number"]].copy()
)
to_drop = pd.concat(to_drop)
if not to_drop.empty:
merged = pd.merge(matching_lookup, to_drop, on=['UPRN', 'Title Number'], how='left', indicator=True)
merged = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
return merged
return matching_lookup
def remove_duplicate_uprn_matches(matching_lookup, properties, company_ownership):
dupe_uprns = matching_lookup[matching_lookup["UPRN"].duplicated()]["UPRN"].unique().tolist()
to_drop = []
for dupe_uprn in dupe_uprns:
dupe_data = matching_lookup[matching_lookup["UPRN"] == dupe_uprn].copy()
matched_addresses = dupe_data.merge(
properties[["UPRN", "ADDRESS"]].rename(columns={"ADDRESS": "epc_address"}),
how="left", on="UPRN"
).merge(
company_ownership[["Title Number", "Property Address"]],
how="left", on="Title Number"
)
# We perform levenstein to get the best match
best_match = levenstein_match(
matching_string=matched_addresses["Property Address"].values[0],
df=matched_addresses,
address_col="epc_address"
)
matches_to_drop = matched_addresses[
~matched_addresses["Title Number"].isin(best_match["Title Number"].values)
]
to_drop.append(
matches_to_drop[["UPRN", "Title Number"]].copy()
)
to_drop = pd.concat(to_drop)
if not to_drop.empty:
merged = pd.merge(matching_lookup, to_drop, on=['UPRN', 'Title Number'], how='left', indicator=True)
merged = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
return merged
return matching_lookup
def app():
"""
This script is for scoping property ownership for EPC F & G rated properties in Birmingam, for Goldman Sachs
"""
# paths = [
# "local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000031-Wolverhampton/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000026-Coventry/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000016-Leicester/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000015-Derby/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000021-Stoke-on-Trent/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000018-Nottingham/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000154-Northampton/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000061-North-Northamptonshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000062-West-Northamptonshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000152-East-Northamptonshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000155-South-Northamptonshire/certificates.csv",
# #
# "local_data/all-domestic-certificates/domestic-E08000027-Dudley/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000029-Solihull/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000234-Bromsgrove/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000030-Walsall/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000028-Sandwell/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000019-Herefordshire-County-of/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000020-Telford-and-Wrekin/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000218-North-Warwickshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000222-Warwick/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000237-Worcester/certificates.csv",
# # East midlands
# "local_data/all-domestic-certificates/domestic-E07000035-Derbyshire-Dales/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000038-North-East-Derbyshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000039-South-Derbyshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000012-North-East-Lincolnshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000013-North-Lincolnshire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000138-Lincoln/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E07000134-North-West-Leicestershire/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E06000017-Rutland/certificates.csv",
# ]
# paths = list(set(paths))
# find_f_g_properties(paths)
properties = pd.read_excel("EPC F & G Properties.xlsx")
company_ownership = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_04.csv")
company_ownership["is_overseas"] = False
overseas_company_ownership = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/OCOD_FULL_2024_04 2.csv")
overseas_company_ownership["is_overseas"] = True
company_ownership = pd.concat([company_ownership, overseas_company_ownership])
# FIlter on relevant postcodes
company_ownership = company_ownership[
company_ownership["Postcode"].str.lower().isin(properties["POSTCODE"].str.lower().unique())]
# Now we filter properties the other way around
properties = properties[properties["POSTCODE"].str.lower().isin(company_ownership["Postcode"].str.lower().unique())]
# We end up with 7.4k entires on a postcode match, however we need to now do a direct address match
# Take just private rentals
properties = properties[
properties["TENURE"].isin(["rental (private)", "Rented (private)", "owner-occupied", "Owner-occupied"])
]
# We have some duplicated on UPRN
# Take the newest UPRN
properties = properties.sort_values("LODGEMENT_DATE", ascending=False).drop_duplicates("UPRN")
# Remove entries where the address begins with the term "land adjoining", or other records that don't reference the
# the property itself
starting_terms = [
"land adjoining", "land on the", "land to the rear of", "land and buildings on the",
"garage adjoining", "car park adjoining", "the land adjoining", "land and buildings adjoining",
"all royal mines"
]
for starting_term in starting_terms:
company_ownership = company_ownership[
~company_ownership["Property Address"].str.lower().str.startswith(starting_term)
]
freehold_matching_lookup = [] # 634
leasehold_matching_lookup = [] # 86
shared_leasehold_match = []
shared_freehold_match = []
for _, address in tqdm(properties.iterrows(), total=len(properties)):
match_type = "exact"
filtered = company_ownership[
company_ownership["Postcode"].str.lower() == address["POSTCODE"].lower()
].copy()
# Remove postcode and remove trailing commas
filtered["house_number"] = (
filtered["Property Address"]
.apply(remove_text_in_brackets)
.apply(SearchEpc.get_house_number)
.str.lower()
.str.replace(",", "")
)
house_no = SearchEpc.get_house_number(address["ADDRESS1"])
if house_no is not None:
house_no = house_no.replace(",", "")
if house_no is None:
# It's hard for us to get a reliable match
# filtered = filtered[filtered["Property Address"].str.contains(address["ADDRESS1"])]
# if filtered.shape[0] > 1:
# raise Exception("No valid - maybe we should do levenstein?")
continue
else:
if house_no not in filtered["house_number"].values:
# If this happens, we check house_number for a x-y range of addresses
filtered["house_number_range"] = filtered["house_number"].apply(extract_range_from_house_number)
# If we have found a house number range, we check if the house number is in the range and if not,
# we drop the row
filtered['is_in_range'] = filtered['house_number_range'].apply(lambda x: is_in_range(x, house_no))
if filtered['is_in_range'].any():
# If house_no is found in any range, keep only rows where it is in range
filtered = filtered[filtered['is_in_range']]
else:
# If house_no is not found in any range, filter out rows where 'house_number_range' is not None
filtered = filtered[filtered['house_number_range'].isnull()]
# Strip out letters from house_no and house_number
house_no = extract_numeric_part(house_no)
filtered["house_number"] = filtered["house_number"].astype(str).apply(extract_numeric_part)
match_type = "approximate"
filtered = filtered[filtered["house_number"] == house_no]
if filtered.empty:
continue
filtered_freehold = filtered[filtered["Tenure"] == "Freehold"]
filtered_leasehold = filtered[filtered["Tenure"] == "Leasehold"]
if filtered_freehold.shape[0] > 1:
matched = filtered_leasehold[["Title Number"]].copy()
matched.insert(0, "UPRN", address["UPRN"])
shared_freehold_match.append(matched)
elif not filtered_freehold.empty:
freehold_matching_lookup.append(
{
"UPRN": address["UPRN"],
"Title Number": filtered_freehold["Title Number"].values[0],
"match_type": match_type,
}
)
if filtered_leasehold.shape[0] > 1:
matched = filtered_leasehold[["Title Number"]].copy()
matched.insert(0, "UPRN", address["UPRN"])
shared_leasehold_match.append(matched)
elif not filtered_leasehold.empty:
leasehold_matching_lookup.append(
{
"UPRN": address["UPRN"],
"Title Number": filtered_leasehold["Title Number"].values[0],
"match_type": match_type,
}
)
freehold_matching_lookup = pd.DataFrame(freehold_matching_lookup)
leasehold_matching_lookup = pd.DataFrame(leasehold_matching_lookup)
shared_leasehold_match = pd.concat(shared_leasehold_match)
shared_freehold_match = pd.concat(shared_freehold_match)
# freehold_matching_lookup.to_excel("freehold_matching_lookup_new.xlsx")
# leasehold_matching_lookup.to_excel("leasehold_matching_lookup_new.xlsx")
# shared_leasehold_match.to_excel("shared_leasehold_match_new.xlsx")
# shared_freehold_match.to_excel("shared_freehold_match_new.xlsx")
# The approximate matches aren't very good
freehold_matching_lookup = freehold_matching_lookup[freehold_matching_lookup["match_type"] == "exact"]
leasehold_matching_lookup = leasehold_matching_lookup[leasehold_matching_lookup["match_type"] == "exact"]
# Combine
combined_matching_lookup = pd.concat([freehold_matching_lookup, leasehold_matching_lookup])
# Remove duplicates
combined_matching_lookup = remove_duplicate_matches(combined_matching_lookup, properties, company_ownership)
# We also have duplicates at a UPRN level
combined_matching_lookup = remove_duplicate_uprn_matches(combined_matching_lookup, properties, company_ownership)
# There are some cases where we have duplicates
# freehold_matching_lookup = remove_duplicate_matches(freehold_matching_lookup, properties, company_ownership)
# leasehold_matching_lookup = remove_duplicate_matches(leasehold_matching_lookup, properties, company_ownership)
matched_addresses = combined_matching_lookup.merge(
properties[["UPRN", "ADDRESS", "CURRENT_ENERGY_EFFICIENCY", "CURRENT_ENERGY_RATING"]].rename(
columns={"ADDRESS": "epc_address"}),
how="left", on="UPRN"
).merge(
company_ownership[["Title Number", "Property Address", "Company Registration No. (1)", "Proprietor Name (1)"]],
how="left", on="Title Number"
)
# shared_freehold_match = pd.DataFrame(shared_freehold_match)
# Strore these files
# freehold_matching_lookup.to_excel("freehold_matching_lookup.xlsx")
# leasehold_matching_lookup.to_excel("leasehold_matching_lookup.xlsx")
# shared_leasehold_match.to_excel("shared_leasehold_match.xlsx")
# shared_freehold_match.to_excel("shared_freehold_match.xlsx")
# read the files
# freehold_matching_lookup = pd.read_excel("freehold_matching_lookup.xlsx")
# leasehold_matching_lookup = pd.read_excel("leasehold_matching_lookup.xlsx")
# shared_leasehold_match = pd.read_excel("shared_leasehold_match.xlsx")
freehold_aggregate = aggregate_matches(freehold_matching_lookup, company_ownership, properties)
leasehold_aggregate = aggregate_matches(leasehold_matching_lookup, company_ownership, properties)
combined_aggregate = aggregate_matches(
combined_matching_lookup, company_ownership, properties
)
investment_20m = combined_aggregate[combined_aggregate["cumulative_value"] <= 20_500_000]
investment_50m = combined_aggregate[combined_aggregate["cumulative_value"] <= 51_000_000]
investment_20m_properties = matched_addresses[
matched_addresses["Company Registration No. (1)"].isin(investment_20m["Company Registration No. (1)"])
]
investment_50m_properties = matched_addresses[
matched_addresses["Company Registration No. (1)"].isin(investment_50m["Company Registration No. (1)"])
]
portfolio_epc_data_50m = properties[properties["UPRN"].isin(investment_50m_properties["UPRN"])]
portfolio_epc_data_20m = properties[properties["UPRN"].isin(investment_20m_properties["UPRN"])]
investment_20m_properties.to_excel("investment_20m_properties 28th May.xlsx", index=False)
investment_50m_properties.to_excel("investment_50m_properties 28th May.xlsx", index=False)
# Store the EPC data
portfolio_epc_data_50m.to_excel("portfolio_epc_data_50m 28th May.xlsx", index=False)
portfolio_epc_data_20m.to_excel("portfolio_epc_data_20m 28th May.xlsx", index=False)
def company_aggregation():
company_ownership = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_04.csv")
aggregation = (
company_ownership
.groupby(["Proprietor Name (1)", "Company Registration No. (1)"])
["Property Address"]
.count()
.reset_index(name="Number of Properties")
)
aggregation = aggregation.sort_values("Number of Properties", ascending=False)
aggregation.to_excel("Company ownership aggregation.xlsx")

98
etl/customers/guiness/route_march.py Normal file
View file

@ -0,0 +1,98 @@
import os
import pandas as pd
from tqdm import tqdm
from dotenv import load_dotenv
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from epc_api.client import EpcClient
from utils.s3 import save_csv_to_s3
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
def app():
"""
This app is satisying an adhoc request to retrieve EPC data for properties owned by Guiness, to help plan the
route march
These properties were provided to us by Ecosurv
:return:
"""
asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/guiness/TGP CW Properties PV.xlsx",
header_row=0
)
epc_data = []
for _, guiness_property in tqdm(asset_list.iterrows(), total=len(asset_list)):
searcher = SearchEpc(
address1=str(guiness_property["Address"]),
postcode=guiness_property["POSTCODES"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
continue
epc = {
"asset_list_address": guiness_property["Address"],
"asset_list_postcode": guiness_property["POSTCODES"],
**searcher.newest_epc.copy()
}
epc_data.append(epc)
epc_df = pd.DataFrame(epc_data)
# Retrieve just the data we need
epc_df = epc_df[
[
"asset_list_address",
"asset_list_postcode",
"uprn",
"property-type",
"built-form",
"inspection-date",
"current-energy-rating",
"current-energy-efficiency",
"roof-description",
"walls-description",
"transaction-type"
]
]
asset_list = asset_list.merge(
epc_df, how="left", left_on=["Address", "POSTCODES"], right_on=["asset_list_address", "asset_list_postcode"]
)
# De-dupe on the address and postcode, since 137 Badger Avenue was duplicated
asset_list = asset_list.drop_duplicates(subset=["Address", "POSTCODES"])
asset_list = asset_list.drop(columns=["asset_list_address", "asset_list_postcode"])
# Rename the columns
asset_list = asset_list.rename(columns={
"property-type": "Property Type",
"built-form": "Archetype",
"inspection-date": "Last EPC Inspection Date",
"current-energy-rating": "Last survey EPC Rating",
"current-energy-efficiency": "Last survey SAP Score",
"roof-description": "Roof Construction",
"walls-description": "Wall Construction",
"transaction-type": "Last EPC Reason"
})
# Store as an excel
filename = "Guiness EPC data.xlsx"
asset_list.to_excel(filename, index=False)

30
etl/customers/immo/pilot/asset_list.py
View file

@ -21,6 +21,7 @@ council_tax_bands = pd.DataFrame(council_tax_bands)
# This is information we need to override on the EPC itself, for instance if a new survey has been conducted and
# that has not reached the API
# For 53 Bromley, the non-invasives found the walls to be partially filled
patches = [
{
'address': '6 Beech Road', 'postcode': 'DY1 4BP',
@ -42,7 +43,11 @@ patches = [
'energy-consumption-current': '491',
'co2-emissions-current': '5.0',
'potential-energy-efficiency': '87'
}
},
{
'address': '53 Bromley', 'postcode': 'DY5 4PJ',
'walls-description': 'Cavity wall, partial insulation (assumed)',
},
]
# This is information that is found as a result of the non-invasives, that mean that certain measures
@ -56,6 +61,19 @@ already_installed = [
}
]
non_invasive_recommendations = [
{'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'recommendations': []},
{'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'recommendations': ['cavity_extract_and_refill']},
{'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'recommendations': ['cavity_extract_and_refill']},
{'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'recommendations': ['cavity_extract_and_refill']},
{'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'recommendations': ['cavity_surveyed_as_filled_is_partial']},
{'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'recommendations': ['cavity_extract_and_refill']},
{'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'recommendations': ['cavity_extract_and_refill']},
{'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'recommendations': ['cavity_extract_and_refill']},
{'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'recommendations': []},
{'address': '5 Oaklands', 'postcode': 'B62 0JA', 'recommendations': ['cavity_extract_and_refill']},
]
def app():
raw_asset_list = read_excel_from_s3(
@ -102,6 +120,14 @@ def app():
file_name=patches_filename
)
# Store non-invasive recommendations in S3
non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
save_csv_to_s3(
dataframe=pd.DataFrame(non_invasive_recommendations),
bucket_name="retrofit-plan-inputs-dev",
file_name=non_invasive_recommendations_filename
)
# EPC C portoflio
body = {
"portfolio_id": str(PORTFOLIO_ID),
@ -111,6 +137,7 @@ def app():
"trigger_file_path": filename,
"already_installed_file_path": already_installed_filename,
"patches_file_path": patches_filename,
"non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
"budget": None,
}
print(body)
@ -124,6 +151,7 @@ def app():
"trigger_file_path": filename,
"already_installed_file_path": already_installed_filename,
"patches_file_path": patches_filename,
"non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
"budget": None,
}
print(body)

152
etl/customers/immo/pilot/asset_list_2.py Normal file
View file

@ -0,0 +1,152 @@
import pandas as pd
from utils.s3 import read_excel_from_s3
from utils.s3 import save_csv_to_s3
USER_ID = 8
PORTFOLIO_ID = 72
# For
patches = [
{
'address': '116 Parkes Hall Road',
'postcode': 'DY1 3RJ',
'uprn': '90046817',
'walls-description': 'Cavity wall, filled cavity',
'walls-energy-eff': 'Average',
'roof-description': 'Pitched, 270 mm loft insulation',
'roof-energy-eff': 'Good',
'windows-description': 'Fully double glazed',
'windows-energy-eff': 'Good',
'mainheat-description': 'Boiler and radiators, mains gas',
'mainheat-energy-eff': 'Good',
'mainheatcont-description': 'Programmer, room thermostat and TRVs',
'mainheatc-energy-eff': 'Good',
'lighting-description': 'Low energy lighting in 27% of fixed outlets',
'lighting-energy-eff': 'Average',
'floor-description': 'Solid, no insulation (assumed)',
'secondheat-description': 'None',
'current-energy-efficiency': '73',
'current-energy-rating': 'C',
'energy-consumption-current': '184',
'co2-emissions-current': '2.4',
'potential-energy-efficiency': '88',
'total-floor-area': '73',
'construction-age-band': 'England and Wales: 1930-1949',
'property-type': 'House',
'built-form': 'Mid-Terrace',
}
]
# This is information that is found as a result of the non-invasives, that mean that certain measures
# have been installed already. To reflect this in the front end, it is included in the recommendation, however
# the cost is removed and instead, a message is presented saying that the measure is already installed.
already_installed = [
{
'address': '28 Sangwin Road', 'postcode': 'WV14 9EQ', "already_installed": ["loft_insulation"]
},
{
'address': '51 Hillwood Road', 'postcode': 'B62 8NQ', "already_installed": ["loft_insulation"]
},
{
'address': '47 Watsons Close', 'postcode': 'DY2 7HL', "already_installed": ["loft_insulation"]
},
{
'address': '44 Hatfield Road',
'postcode': 'DY9 7LW',
"already_installed": ["loft_insulation", "cavity_wall_insulation"]
}
]
non_invasive_recommendations = []
def app():
raw_asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/Immo/Dudley Asset List - Hestia - pilot2.xlsx",
header_row=0
)
raw_asset_list = raw_asset_list[raw_asset_list["in_pilot"]].copy()
# Extract address and postcode
raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0]
raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip()
# We're provided with number of bathrooms and number of bedrooms.
# THe UPRNs are not the official ones
asset_list = raw_asset_list.rename(
columns={
"No. of Beds": "n_bedrooms",
"No. of WC's": "n_bathrooms",
'Property Type': 'property_type',
'Architype': 'built_form'
}
)
# Remap the values
asset_list["built_form"] = asset_list["built_form"].map({
"SEMI DETACHED": "Semi-Detached",
"MID TERRACE": "Mid-Terrace",
"END TERRACE": "End-Terrace",
})
# Store the asset list in s3
filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
save_csv_to_s3(
dataframe=asset_list,
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# Store overrides in s3
already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json"
save_csv_to_s3(
dataframe=pd.DataFrame(already_installed),
bucket_name="retrofit-plan-inputs-dev",
file_name=already_installed_filename
)
# Store patches in s3
patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json"
save_csv_to_s3(
dataframe=pd.DataFrame(patches),
bucket_name="retrofit-plan-inputs-dev",
file_name=patches_filename
)
# Store non-invasive recommendations in S3
non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
save_csv_to_s3(
dataframe=pd.DataFrame(non_invasive_recommendations),
bucket_name="retrofit-plan-inputs-dev",
file_name=non_invasive_recommendations_filename
)
# EPC C portoflio
body = {
"portfolio_id": str(PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "C",
"trigger_file_path": filename,
"already_installed_file_path": already_installed_filename,
"patches_file_path": patches_filename,
"non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
"budget": None,
}
print(body)
# EPC B portoflio
body = {
"portfolio_id": str(PORTFOLIO_ID + 1),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "B",
"trigger_file_path": filename,
"already_installed_file_path": already_installed_filename,
"patches_file_path": patches_filename,
"non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
"budget": None,
}
print(body)

134
etl/customers/livewest/route_march.py Normal file
View file

@ -0,0 +1,134 @@
import os
import pandas as pd
from tqdm import tqdm
from dotenv import load_dotenv
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from epc_api.client import EpcClient
from utils.s3 import save_csv_to_s3
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
def route_march_may_2024():
"""
This code pulls supplementary data for a route march that is expected to happen in May 2024. This code
was authored on the 30th April 2024.
"""
asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/Livewest/Livewest proposed route march Apr-May 2024.xlsx",
header_row=0
)
epc_data = []
for _, unit in tqdm(asset_list.iterrows(), total=len(asset_list)):
lst = [unit["NO"], unit["ADDRESS 1"], unit["ADDRESS 2"], unit["ADDRESS 3"], unit["POSTCODE"]]
lst = [str(x).strip() for x in lst if not pd.isnull(x)]
full_address = ", ".join(lst)
searcher = SearchEpc(
address1=str(unit["NO"]),
postcode=unit["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
# We try with a different address 1
add1 = str(unit["NO"]).lower()
add1 = (
add1
.replace("flat", "")
.replace("ft", "")
.replace("t", "").strip()
)
searcher = SearchEpc(
address1=add1,
postcode=unit["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
continue
epc = {
"asset_list_house_no": unit["NO"],
"asset_list_address1": unit["ADDRESS 1"],
"asset_list_postcode": unit["POSTCODE"],
**searcher.newest_epc.copy()
}
epc_data.append(epc)
epc_df = pd.DataFrame(epc_data)
#
# Retrieve just the data we need
epc_df = epc_df[
[
"asset_list_house_no",
"asset_list_address1",
"asset_list_postcode",
"uprn",
"address",
"property-type",
"built-form",
"inspection-date",
"current-energy-rating",
"current-energy-efficiency",
"roof-description",
"walls-description",
"transaction-type"
]
].rename(columns={"address": "Matched EPC Address"})
asset_list = asset_list.merge(
epc_df,
how="left",
left_on=["NO", "ADDRESS 1", "POSTCODE"],
right_on=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"]
)
asset_list = asset_list.drop_duplicates(subset=["NO", "ADDRESS 1", "POSTCODE"])
asset_list = asset_list.drop(columns=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"])
# Rename the columns
asset_list = asset_list.rename(columns={
"property-type": "Property Type",
"built-form": "Archetype",
"inspection-date": "Last EPC Inspection Date",
"current-energy-rating": "Last survey EPC Rating",
"current-energy-efficiency": "Last survey SAP Score",
"roof-description": "Roof Construction",
"walls-description": "Wall Construction",
"transaction-type": "Last EPC Reason"
})
# Store as an excel
filename = "Livewest EPC data.xlsx"
asset_list.to_excel(filename, index=False)

300
etl/customers/places_for_people/route_march.py Normal file
View file

@ -0,0 +1,300 @@
import os
import time
import pandas as pd
from tqdm import tqdm
from dotenv import load_dotenv
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from epc_api.client import EpcClient
from utils.s3 import save_csv_to_s3
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
def app():
"""
This app is satisying an adhoc request to retrieve EPC data for properties owned by Guiness, to help plan the
route march
These properties were provided to us by Ecosurv
:return:
"""
asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/Places For People/PFP ROUTE MARCH PHASE 1.xlsx",
header_row=1
)
epc_data = []
for _, pfp_property in tqdm(asset_list.iterrows(), total=len(asset_list)):
lst = [
pfp_property["ADDRESS"],
pfp_property["ADDRESS.1"],
# pfp_property["ADDRESS.2"],
pfp_property["POSTCODE"]
]
lst = [str(x).strip() for x in lst if not pd.isnull(x)]
full_address = ", ".join(lst)
searcher = SearchEpc(
address1=str(pfp_property["ADDRESS"]),
postcode=pfp_property["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
# We try with a different address 1
add1 = str(pfp_property["ADDRESS"]).lower()
add1 = add1.replace("ft", "").replace("t", "").strip()
searcher = SearchEpc(
address1=add1,
postcode=pfp_property["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
continue
epc = {
"asset_list_address": pfp_property["ADDRESS"],
"asset_list_address1": pfp_property["ADDRESS.1"],
"asset_list_postcode": pfp_property["POSTCODE"],
**searcher.newest_epc.copy()
}
epc_data.append(epc)
epc_df = pd.DataFrame(epc_data)
# 702
# Retrieve just the data we need
epc_df = epc_df[
[
"asset_list_address",
"asset_list_address1",
"asset_list_postcode",
"uprn",
"address",
"property-type",
"built-form",
"inspection-date",
"current-energy-rating",
"current-energy-efficiency",
"roof-description",
"walls-description",
"transaction-type"
]
].rename(columns={"address": "Matched EPC Address"})
asset_list = asset_list.merge(
epc_df,
how="left",
left_on=["ADDRESS", "ADDRESS.1", "POSTCODE"],
right_on=["asset_list_address", "asset_list_address1", "asset_list_postcode"]
)
# De-dupe on the address and postcode, since 137 Badger Avenue was duplicated
asset_list = asset_list.drop_duplicates(subset=["ADDRESS", "ADDRESS.1", "POSTCODE"])
asset_list = asset_list.drop(columns=["asset_list_address", "asset_list_address1", "asset_list_postcode"])
# Rename the columns
asset_list = asset_list.rename(columns={
"property-type": "Property Type",
"built-form": "Archetype",
"inspection-date": "Last EPC Inspection Date",
"current-energy-rating": "Last survey EPC Rating",
"current-energy-efficiency": "Last survey SAP Score",
"roof-description": "Roof Construction",
"walls-description": "Wall Construction",
"transaction-type": "Last EPC Reason"
})
# Store as an excel
filename = "Places For People EPC data.xlsx"
asset_list.to_excel(filename, index=False)
# TODO: TEMP
# This script takes in a a list of properties
# Will be postcode and address
import requests
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from tqdm import tqdm
SEARCH_POSTCODE_URL = ("https://find-energy-certificate.service.gov.uk/find-a-certificate/search-by-postcode?postcode"
"={postcode_input}")
BASE_ENERGY_URL = "https://find-energy-certificate.service.gov.uk"
def retrieve_find_my_epc_data(postcode: str, address: str):
"""
For a post code and address, we pull out all the required data from the find my epc website
"""
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/111.0.0.0 Safari/537.36'}
postcode_input = postcode.replace(" ", "+")
postcode_search = SEARCH_POSTCODE_URL.format(postcode_input=postcode_input)
postcode_response = requests.get(postcode_search, headers=headers)
postcode_res = BeautifulSoup(postcode_response.text)
address_links_full = postcode_res.findAll('a', {'class': 'govuk-link', 'rel': 'nofollow'})
address_links = {element.text.lstrip().rstrip(): BASE_ENERGY_URL + element['href'] for element in
address_links_full}
# TODO: to check the logic works for all cases but seems to be good
index_of_address = [key.lower().startswith(address) for key in list(address_links.keys())]
chosen_epc = address_links[list(address_links.keys())[np.where(index_of_address)[0][0]]]
epc_certificate = chosen_epc.split('/')[-1]
address_response = requests.get(chosen_epc, headers=headers)
address_res = BeautifulSoup(address_response.text)
# print("## Energy rating - current and potential")
ratings = address_res.find('desc', {'id': 'svg-desc'}).text
# print('### Current EPC rating')
current_rating = ratings.split(".")[0]
# print("##### " + current_rating)
# print('### Potential EPC rating')
potential_rating = ratings.split(".")[1]
# print("##### " + potential_rating)
new_property_df = pd.DataFrame(
{'address': [address],
'epc_certificate': [epc_certificate],
'current_epc_rating': [current_rating.split(' ')[-6]],
'current_epc_efficiency': [current_rating.split(' ')[-1]],
'potential_epc_rating': [potential_rating.split(' ')[-6]],
"potential_epc_efficiency": [potential_rating.split(' ')[-1]]}
)
# print("Find assessor")
assessor_block = address_res.find('div', {'class': 'epc-contact-assessor'})
assessor_fields = assessor_block.find_all('dd', {"class": 'govuk-summary-list__value govuk-!-width-one-half'})
assessor_name = assessor_fields[0].text.strip()
assessor_number = assessor_fields[1].text.strip()
assessor_email = assessor_fields[2].text.strip()
new_property_df['assessor_name'] = assessor_name
new_property_df['assessor_number'] = assessor_number
new_property_df['assessor_email'] = assessor_email
return new_property_df
# print('### Changes that can be made:')
# improvements = address_res.find('div', {"class": "govuk-body printable-area epb-recommended-improvements"})
# if improvements is None:
# print("No changes suggested")
# else:
# changes = improvements.find_all('h3')
# changes_impact = improvements.find_all('dl', {"class": 'govuk-summary-list'})
# for element in zip(changes, changes_impact):
# improvement_header = element[0].text
# print("#### " + improvement_header)
# improvement_text = element[1].text
# print(improvement_text)
# col_name = improvement_header.split(":")[1]
# cost = element[1].find('dd', {"class": "govuk-summary-list__value"}).text.lstrip().rstrip()
# impact = element[1].find('text', {"class": "govuk-!-font-weight-bold"}).text.split(" ")
# impact_num = impact[0]
# impact_cat = impact[1]
# print(cost)
# new_property_df[col_name] = True
# # cost_column = col_name + '-cost'
# # new_property_df.assign(cost_column=cost)
# new_property_df[col_name + '-cost'] = cost
# new_property_df[col_name + '-impact_num'] = impact_num
# new_property_df[col_name + '-impact_cat'] = impact_cat
# data = pd.concat([data, new_property_df])
# data.to_csv('./portfolio.csv')
def main():
"""
Main pipeline function to take in a predefined list of properties and extract names of contractors
"""
# Load in list of properties
addresses_df = pd.read_excel("/Users/khalimconn-kowlessar/Downloads/Places For People EPC data.xlsx")
addresses_df["uprn"] = addresses_df["uprn"].astype("Int64").astype(str)
# 1256
find_my_epc_data_list = []
for i, row in tqdm(addresses_df.iterrows(), total=addresses_df.shape[0]):
if pd.isnull(row['Matched EPC Address']):
continue
# 10 second break every 50 iterations
if (i % 50 == 0) and (i != 0):
time.sleep(10)
time.sleep(1)
if row['Matched EPC Address'] == "6 CHURCHWOOD, CHURCH STREET, CRAMLINGTON":
address_data = retrieve_find_my_epc_data(
postcode=row['POSTCODE'],
address=" ".join([str(row["ADDRESS"]), row["ADDRESS.1"]]).lower()
)
else:
address_data = retrieve_find_my_epc_data(
postcode=row['POSTCODE'],
address=", ".join(row['Matched EPC Address'].split(", ")[:-1]).lower()
)
address_data.insert(0, "uprn", row["uprn"])
find_my_epc_data_list.append(address_data)
find_my_epc_data = pd.concat(find_my_epc_data_list)
find_my_epc_data.to_csv('find_my_epc_data.csv')
find_my_epc_data = find_my_epc_data.drop_duplicates("uprn")
# Match back to addresses
addresses_df2 = addresses_df.merge(
find_my_epc_data,
how="left",
on="uprn"
)
addresses_df2.to_excel("Places For People EPC data with surveyor.xlsx", index=False)
if __name__ == "__main__":
main()

0
etl/customers/vander_elliot/__init__.py Normal file
View file

23
etl/customers/vander_elliot/initial_scoping.py Normal file
View file

@ -0,0 +1,23 @@
import pandas as pd
from utils.s3 import save_csv_to_s3
def app():
# Check how many properties there are at EPC F/G in Birmingham
epc_data = pd.read_csv(
"local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv",
low_memory=False
)
# Filter on entries where we have a UPRN
epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
# Get the newest EPC for each UPRN. We use LODGEMENT_DATE as a proxy for this
epc_data["LODGEMENT_DATE"] = pd.to_datetime(epc_data["LODGEMENT_DATE"])
epc_data = epc_data.sort_values("LODGEMENT_DATE", ascending=False).drop_duplicates("UPRN")
epc_data = epc_data[epc_data["CURRENT_ENERGY_RATING"].isin(["F", "G"])]
one_years_ago = pd.Timestamp.now() - pd.DateOffset(days=int(1 * 365))
epc_data = epc_data[epc_data["LODGEMENT_DATE"] >= one_years_ago]

105
etl/customers/vander_elliot/pilot.py Normal file
View file

@ -0,0 +1,105 @@
import pandas as pd
from utils.s3 import save_csv_to_s3
EPC_C_PORTFOLIO_ID = 78
EPC_B_PORTFOLIO_ID = 79
USER_ID = 8
def app():
"""
This code sets up the asset list for the 9 property portfolio for the pilot
:return:
"""
asset_list = [
{
"address": "79 Clare Road",
"postcode": "L20 9LZ",
"uprn": 41018850, # 3 bedroom property
},
{
"address": "Flat 1, 29 Bedford Road",
"postcode": "L4 5PS",
"uprn": 38237316 # Single dewlling converted into two flats
},
{
"address": "Flat 2, 29 Bedford Road",
"postcode": "L4 5PS",
"uprn": 38237317 # Single dewlling converted into two flats
},
# 7 Flats above a domestic unit
{
"address": "Flat 1, 2 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41052320
},
{
"address": "Flat 2, 2 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41052321,
},
{
"address": "Flat 3, 2 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41052322,
},
{
"address": "Flat 4, 2 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41222759,
},
{
"address": "Flat 1, 4 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41222760,
},
{
"address": "Flat 2, 4 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41222761,
},
{
"address": "Flat 3, 4 Linacre Lane",
"postcode": "L20 5AH",
"uprn": 41212534,
},
]
asset_list = pd.DataFrame(asset_list)
# Store the asset list in s3
filename = f"{USER_ID}/{EPC_C_PORTFOLIO_ID}/pilot.csv"
save_csv_to_s3(
dataframe=asset_list,
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# EPC C portoflio
body = {
"portfolio_id": str(EPC_C_PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "C",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": "",
"non_invasive_recommendations_file_path": "",
"budget": None,
}
print(body)
# EPC B portoflio
body = {
"portfolio_id": str(EPC_B_PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "B",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": "",
"non_invasive_recommendations_file_path": "",
"budget": None,
}
print(body)

56
etl/customers/vander_elliot/single_property_pilot.py Normal file
View file

@ -0,0 +1,56 @@
import pandas as pd
from utils.s3 import read_excel_from_s3
from utils.s3 import save_csv_to_s3
PORTFOLIO_ID = 77
USER_ID = 8
patches = [
{
"address": "79 Perryn Road",
"postcode": "W3 7LT",
"roof-description": "Pitched, no insulation (assumed)"
}
]
def app():
asset_list = [
{
'uprn': 12103117,
"address": "79 Perryn Road",
"postcode": "W3 7LT",
},
]
asset_list = pd.DataFrame(asset_list)
# Store the asset list in s3
filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
save_csv_to_s3(
dataframe=asset_list,
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# Store patches in s3
patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json"
save_csv_to_s3(
dataframe=pd.DataFrame(patches),
bucket_name="retrofit-plan-inputs-dev",
file_name=patches_filename
)
body = {
"portfolio_id": str(PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increase EPC",
"goal_value": "B",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": patches_filename,
"non_invasive_recommendations_file_path": "",
"budget": None,
}
print(body)

204
etl/eligibility/ha_15_32/ha_analysis_batch_3.py
View file

@ -24,9 +24,13 @@ from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
from etl.epc.DataProcessor import EPCDataProcessor
from datetime import datetime
import inspect
src_file_path = inspect.getfile(lambda: None)
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"
DATA_FOLDER = Path(__file__).parent / "local_data" / "ha_data"
ENV_FILE = Path(src_file_path).parent / "etl" / "eligibility" / "ha_15_32" / ".env"
DATA_FOLDER = Path(src_file_path).parent / "local_data" / "ha_data"
logger = setup_logger()
load_dotenv(ENV_FILE)
@ -6127,6 +6131,9 @@ def classify_loft(x):
def fml_analysis(loader):
# In the case of the optimistic scenario, we assume that the at-risk pipeline is still viable, just at a lower rate
optimistic_scenario_rate = 1500
assumed_ciga_pass_rate = 0.731
has_bruh = [
"HA7", "HA14", "HA25", "HA39", "HA16", "HA28", "HA13",
@ -6224,6 +6231,7 @@ def fml_analysis(loader):
if fuck_this.shape[0] != before_merge_shape:
raise Exception("SOMETHING WENT WRONG")
# Automated archetype check
if any(fuck_this["ECO Eligibility"].str.contains("subject to archetype")):
# We perform the archetype test. If the property is a house, we it needs to be detached, semi-detached
# or end terrace. If it's a bungalow, it must be attached
@ -6319,6 +6327,7 @@ def fml_analysis(loader):
]
# Characterise no CIGA check needed
# !!!!!!!!!!!! AT RISK !!!!!!!!!!!!
ciga_check_passed = had_survey[had_survey["ECO Eligibility"] == "eco4 - passed ciga"]
# These should be treated the same as one that have passed their ciga checks, from a detection perspective
ciga_check_passed_eligible = ciga_check_passed[
@ -6392,6 +6401,12 @@ def fml_analysis(loader):
identified_as_gbis_looks_like_eco4
)
# This is the work that is at risk
eco4_work_at_risk = (
passed_ciga_expectation +
ciga_check_expectation
)
no_ciga_check_needed_actually_gbis = no_ciga_check_needed_eligible_gbis.shape[0]
gbis_qualified = gbis_qualified.shape[0]
@ -6490,11 +6505,13 @@ def fml_analysis(loader):
# "Of which sold": sales_since_nov,
"EPC verified ECO4 Eligible - Remaining": int(total_eco4_expectation),
"EPC verified GBIS Eligibile - Remaining": int(total_gbis_expectation),
# At risk work
"Work at risk due to audits": eco4_work_at_risk
}
)
results_df = pd.DataFrame(results)
results_df.to_csv("analysis - revised.csv")
results_df.to_csv("analysis - revised - audit update.csv")
# results_df["Delta vs November"] = 100 * (
# results_df["Of which ECO4 Eligible - Remaining"] - results_df["Original ECO4 Estimate - Remaining"]
@ -6509,7 +6526,7 @@ def create_final_report():
This function will produce the final output for the HA analysis
:return:
"""
epc_validated_results = pd.read_csv("analysis - revised.csv")
epc_validated_results = pd.read_csv("analysis - revised - audit update.csv")
pipeline_results = pd.read_csv("pipeline_remaining_raw.csv")
####################################
@ -6593,12 +6610,14 @@ def create_final_report():
[
"HA Name",
"EPC verified ECO4 Eligible - Remaining",
"EPC verified GBIS Eligibile - Remaining"
"EPC verified GBIS Eligibile - Remaining",
"Work at risk due to audits"
]
].copy().rename(
columns={
"EPC verified ECO4 Eligible - Remaining": "# ECO4 remaining - From EPC Database (post CIGA)",
"EPC verified GBIS Eligibile - Remaining": "# GBIS remaining - From EPC Database (post CIGA)",
"Work at risk due to audits": "ECO4 remaining work at risk due to Audits",
}
)
@ -6623,7 +6642,8 @@ def create_final_report():
'# ECO4 remaining - All HA Summary',
'# ECO4 remaining - Postcode list (pre CIGA)',
'# ECO4 remaining - Postcode list (post CIGA)',
'# ECO4 remaining - From EPC Database (post CIGA)'
'# ECO4 remaining - From EPC Database (post CIGA)',
'ECO4 remaining work at risk due to Audits'
]:
revenue[col] = revenue[col] * 1710
@ -6688,8 +6708,8 @@ def create_final_report():
# "# GBIS remaining - Postcode list (post CIGA)"]]
# Store final outputs
volumes.to_csv("HA Analysis Final - volumes.csv")
revenue.to_csv("HA Analysis Final - revenue.csv")
volumes.to_csv("HA Analysis - Audit Update - volumes.csv")
revenue.to_csv("HA Analysis - Audit Update - revenue.csv")
def identify_eco_works(loader):
@ -7203,84 +7223,96 @@ def app():
loader.load()
loader.ha_facts_and_figures()
# import pickle
# with open("ha_analysis_data_temp.pkl", "wb") as f:
# pickle.dump(loader, f)
# import pickle
# with open("ha_analysis_data_temp.pkl", "rb") as f:
# loader = pickle.load(f)
forecast_remaining_sales(loader)
# Functions to produce the final output lol...
# fml_data_pull(loader) # If we need to pull EPC data
fml_analysis(loader)
create_final_report()
# Adhoc - for HA16, get the properties that still need a CIGA check
asset_list_ha16 = loader.data["HA16"]["asset_list"].copy()
ha_16_need_ciga = asset_list_ha16[
asset_list_ha16["ECO Eligibility"].str.contains("subject to ciga")
]
completed_cigas = loader.data["HA16"]["ciga_list"].copy()
# Store the results
ha_16_need_ciga.to_csv("ha16_need_ciga.csv")
completed_cigas.to_csv("ha16_completed_cigas.csv")
# Adhoc - look at the current pipeline and identify how many dormant, CIGA dependent properties there are for
# live projects
# Read excel
orderbook_filepath = "local_data/ha_data/Warmfront HA client order book overview_20240129.xlsx"
orderbook_workbook = openpyxl.load_workbook(orderbook_filepath)
orderbook_sheet = orderbook_workbook["Contractual Info"]
orderbook_colnames = [cell.value for cell in orderbook_sheet[1]]
rows = []
for row in orderbook_sheet.iter_rows(min_row=2, values_only=False):
row_data = [cell.value for cell in row] # This will get you the cell values
rows.append(row_data)
orderbook = pd.DataFrame(rows, columns=orderbook_colnames)
live_orderbook = orderbook[orderbook["Live, New, or Historic?"] == "LIVE"].copy()
live_orderbook['Redacted HA'] = live_orderbook['Redacted HA'].str.replace(" ", "")
dormant_properties = []
missed_has = []
for _, customer in live_orderbook.iterrows():
if customer['Redacted HA'] not in loader.data.keys():
missed_has.append(customer['Redacted HA'])
continue
asset_list = loader.data[customer['Redacted HA']]["asset_list"].copy()
survey_list = loader.data[customer['Redacted HA']]["survey_list"].copy()
# Remove sold
if not survey_list.empty:
survey_list = survey_list[~pd.isnull(survey_list["asset_list_row_id"])]
asset_list = asset_list.merge(
survey_list[["asset_list_row_id", "installation_status"]],
how="left",
on="asset_list_row_id"
)
# Anything that has an installation has gone to installation, and therefore is not remaining
asset_list = asset_list[pd.isnull(asset_list["installation_status"])]
asset_list = asset_list.drop(columns=["installation_status"])
# We pull out the properties that need a CIGA check
need_ciga = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to ciga)"]
need_archetype = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to archetype)"]
need_ciga_and_archetype = asset_list[
asset_list["ECO Eligibility"] == "eco4 (subject to ciga) (subject to archetype)"
]
dormant_properties.append(
{
"HA Name": customer['Redacted HA'],
"Need CIGA": need_ciga.shape[0],
"Need Archetype": need_archetype.shape[0],
"Need CIGA and Archetype": need_ciga_and_archetype.shape[0]
}
)
dormant_properties = pd.DataFrame(dormant_properties)
totals = dormant_properties.sum()
totals["HA Name"] = "Total"
dormant_properties = pd.concat([dormant_properties, totals.to_frame().T])
dormant_properties.to_csv("dormant_properties.csv")
loader.december_figures["ECO4 remaining"].sum()
december_figures = loader.december_figures.copy()
december_figures["ECO4 remaining"] = np.where(
december_figures["ECO4 remaining"] < 0,
0,
december_figures["ECO4 remaining"]
)
december_figures["ECO4 remaining"].sum()
# asset_list_ha16 = loader.data["HA16"]["asset_list"].copy()
# ha_16_need_ciga = asset_list_ha16[
# asset_list_ha16["ECO Eligibility"].str.contains("subject to ciga")
# ]
# completed_cigas = loader.data["HA16"]["ciga_list"].copy()
# # Store the results
# ha_16_need_ciga.to_csv("ha16_need_ciga.csv")
# completed_cigas.to_csv("ha16_completed_cigas.csv")
#
# # Adhoc - look at the current pipeline and identify how many dormant, CIGA dependent properties there are for
# # live projects
#
# # Read excel
# orderbook_filepath = "local_data/ha_data/Warmfront HA client order book overview_20240129.xlsx"
# orderbook_workbook = openpyxl.load_workbook(orderbook_filepath)
# orderbook_sheet = orderbook_workbook["Contractual Info"]
# orderbook_colnames = [cell.value for cell in orderbook_sheet[1]]
#
# rows = []
# for row in orderbook_sheet.iter_rows(min_row=2, values_only=False):
# row_data = [cell.value for cell in row] # This will get you the cell values
# rows.append(row_data)
#
# orderbook = pd.DataFrame(rows, columns=orderbook_colnames)
# live_orderbook = orderbook[orderbook["Live, New, or Historic?"] == "LIVE"].copy()
# live_orderbook['Redacted HA'] = live_orderbook['Redacted HA'].str.replace(" ", "")
#
# dormant_properties = []
# missed_has = []
# for _, customer in live_orderbook.iterrows():
# if customer['Redacted HA'] not in loader.data.keys():
# missed_has.append(customer['Redacted HA'])
# continue
# asset_list = loader.data[customer['Redacted HA']]["asset_list"].copy()
# survey_list = loader.data[customer['Redacted HA']]["survey_list"].copy()
# # Remove sold
# if not survey_list.empty:
# survey_list = survey_list[~pd.isnull(survey_list["asset_list_row_id"])]
# asset_list = asset_list.merge(
# survey_list[["asset_list_row_id", "installation_status"]],
# how="left",
# on="asset_list_row_id"
# )
# # Anything that has an installation has gone to installation, and therefore is not remaining
# asset_list = asset_list[pd.isnull(asset_list["installation_status"])]
# asset_list = asset_list.drop(columns=["installation_status"])
#
# # We pull out the properties that need a CIGA check
# need_ciga = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to ciga)"]
# need_archetype = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to archetype)"]
# need_ciga_and_archetype = asset_list[
# asset_list["ECO Eligibility"] == "eco4 (subject to ciga) (subject to archetype)"
# ]
#
# dormant_properties.append(
# {
# "HA Name": customer['Redacted HA'],
# "Need CIGA": need_ciga.shape[0],
# "Need Archetype": need_archetype.shape[0],
# "Need CIGA and Archetype": need_ciga_and_archetype.shape[0]
# }
# )
#
# dormant_properties = pd.DataFrame(dormant_properties)
# totals = dormant_properties.sum()
# totals["HA Name"] = "Total"
#
# dormant_properties = pd.concat([dormant_properties, totals.to_frame().T])
# dormant_properties.to_csv("dormant_properties.csv")
#
# loader.december_figures["ECO4 remaining"].sum()
# december_figures = loader.december_figures.copy()
# december_figures["ECO4 remaining"] = np.where(
# december_figures["ECO4 remaining"] < 0,
# 0,
# december_figures["ECO4 remaining"]
# )
# december_figures["ECO4 remaining"].sum()

25
etl/epc/Dataset.py
View file

@ -203,11 +203,11 @@ class TrainingDataset(BaseDataset):
common_cols = [[col + "_starting", col + "_ending"] for col in common_cols]
self.df = self.df.loc[
:,
no_suffix_cols
+ only_ending_cols
+ [col for cols in common_cols for col in cols],
]
:,
no_suffix_cols
+ only_ending_cols
+ [col for cols in common_cols for col in cols],
]
def _remove_abnormal_change_in_floor_area(self):
"""
@ -509,7 +509,7 @@ class TrainingDataset(BaseDataset):
expanded_df["is_sandstone_or_limestone"]
== expanded_df["is_sandstone_or_limestone_ending"]
)
]
]
elif component == "floor":
expanded_df = expanded_df[
(expanded_df["is_suspended"] == expanded_df["is_suspended_ending"])
@ -526,7 +526,7 @@ class TrainingDataset(BaseDataset):
expanded_df["is_to_external_air"]
== expanded_df["is_to_external_air_ending"]
)
]
]
elif component == "roof":
expanded_df = expanded_df[
(expanded_df["is_pitched"] == expanded_df["is_pitched_ending"])
@ -539,7 +539,7 @@ class TrainingDataset(BaseDataset):
expanded_df["has_dwelling_above"]
== expanded_df["has_dwelling_above_ending"]
)
]
]
return expanded_df
@ -567,13 +567,12 @@ class TrainingDataset(BaseDataset):
"is_system_built_ending",
"is_timber_frame_ending",
"is_granite_or_whinstone_ending",
"is_as_built_ending",
# "is_as_built_ending",
"is_cob_ending",
"is_assumed_ending",
"is_sandstone_or_limestone_ending",
# Re remove the is_assumed columns
"is_assumed",
"is_assumed_ending",
# "is_assumed",
# "is_assumed_ending",
],
"floor": [
"original_description",
@ -698,6 +697,8 @@ class TrainingDataset(BaseDataset):
# Rename columns to component specific names, if they have not been dropped
expanded_df = expanded_df.rename(
columns={
"is_assumed": f"{component}_is_assumed",
"is_assumed_ending": f"{component}_is_assumed_ending",
"insulation_thickness": f"{component}_insulation_thickness",
"insulation_thickness_ending": f"{component}_insulation_thickness_ending",
"thermal_transmittance": f"{component}_thermal_transmittance",

2
etl/epc/Record.py
View file

@ -191,7 +191,7 @@ class EPCRecord:
This method will clean the records using the data processor
"""
epc_data_processor = EPCDataProcessor(
data=self.epc_record_as_dataframe("prepared_epc"),
data=self.epc_record_as_dataframe("prepared_epc").copy(),
run_mode="newdata",
cleaning_averages=self.cleaning_data,
)

21
etl/epc_clean/app.py
View file

@ -2,24 +2,27 @@ from tqdm import tqdm
import os
import pandas as pd
import msgpack
import inspect
from etl.epc_clean.EpcClean import EpcClean
from etl.epc.settings import EARLIEST_EPC_DATE
from pathlib import Path
from utils.s3 import save_data_to_s3
src_file_path = inspect.getfile(lambda: None)
LAND_REGISTRY_PATHS = [
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-monthly-update-new-version.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2022 (1).csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2021.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2020.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2019.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2018.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part1.csv",
os.path.abspath(os.path.dirname(__file__)) + "/model_data/local_data/pp-2017-part2.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-monthly-update-new-version.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2022 (1).csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2021.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2020.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2019.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2018.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2017-part1.csv",
os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2017-part2.csv",
]
EPC_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates"
EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates"
ENVIRONMENT = os.getenv("ENVIRONMENT", "dev")

9
etl/epc_clean/epc_attributes/HotWaterAttributes.py
View file

@ -116,7 +116,14 @@ class HotWaterAttributes(Definitions):
"instantaneous at "
"point of use, "
"waste water heat "
"recovery"
"recovery",
"ogçör brif system, adfer gwres d+¦r gwastraff": "from main system, waste water heat recovery",
"twymwr tanddwr, tarriff safonol, adfer gwres d+¦r gwastraff": "electric immersion, standard tariff, waste "
"water heat recovery",
"ogçör brif system, dim thermostat ar y silindr, adfer gwres nwyon ffliw": "from main system, no cylinder "
"thermostat, flue gas heat recovery",
"ogçör brif system, gydag ynnigçör haul, adfer gwres nwyon ffliw": "from main system, plus solar, flue gas "
"heat recovery",
}
def __init__(self, description: str):

3
etl/epc_clean/epc_attributes/MainheatAttributes.py
View file

@ -56,6 +56,9 @@ class MainHeatAttributes(Definitions):
"bwyler a gwres dan y llawr, lpg": "boiler and underfloor heating, lpg",
"bwyler a gwres dan y llawr, trydan": "boiler and underfloor heating, electric",
"boiler and radiators, nwy prif gyflenwad, mains gas": "boiler and radiators, mains gas",
"bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage "
"heaters",
"pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric",
}
REMAP = {

3
etl/epc_clean/epc_attributes/MainheatControlAttributes.py
View file

@ -111,7 +111,8 @@ class MainheatControlAttributes(Definitions):
't+-ól un gyfradd, trvs': 'single rate heating, trvs',
't+ól un gyfradd, rhaglennydd a trvs': 'single rate heating, programmer, trvs',
't+ól un gyfradd, trvs': 'single rate heating, trvs',
'trvs a falf osgoi': 'trvs and bypass'
'trvs a falf osgoi': 'trvs and bypass',
'rheolaeth celect': 'celect-type control',
}
def __init__(self, description: str):

1
etl/epc_clean/epc_attributes/WindowAttributes.py
View file

@ -30,6 +30,7 @@ class WindowAttributes(Definitions):
"gwydrau eilaidd llawn": "full secondary glazing",
"gwydrau eilaidd mwyaf": "mostly secondary glazing",
"gwydrau eilaidd rhannol": "partial secondary glazing",
"gwydrau lluosog ym mhobman": "multiple glazing throughout",
}
def __init__(self, description: str):

2
etl/epc_clean/epc_attributes/attribute_utils.py
View file

@ -24,7 +24,7 @@ def extract_thermal_transmittance(result: dict, description: str) -> Tuple[
if match:
result['thermal_transmittance'] = float(match.group(1))
result['thermal_transmittance_unit'] = match.group(3)
result['thermal_transmittance_unit'] = "w/m-¦k" # We standardise the unit
# Remove the match from the description
description = re.sub(THERMAL_TRANSMITTANCE_STR, "", description)
else:

19
etl/non_intrusive_surveys/photos/README.md Normal file
View file

@ -0,0 +1,19 @@
# Non Intrusive Surveys - photo upload
This folder contains photos taken during non-intrusive surveys. Photos are stored in folders named after the survey ID.
## Getting started
Install the required packages by running the following command:
```bash
pip install -r requirements.txt
```
## Usage
The main application is found in the app.py file. To run the application, use the following command:
```bash
python app.py
```

149
etl/non_intrusive_surveys/photos/app.py Normal file
View file

@ -0,0 +1,149 @@
import boto3
import os
from PIL import Image
from pathlib import Path
from dotenv import load_dotenv
# Inputs
ENV_FILEPATH = "etl/non_intrusive_surveys/photos/.env"
PHOTO_DIRECTORY = "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data"
FOLDER_UPRN_LOOKUP = {
"91 Osprey Drive DY1 2JS": 90048026,
"195 Ashenhurst Rd DY1 2JB": 90051858,
"6 Beech Rd DY1 4BP": 90055152,
"53 Bromley DY5 4PJ": 90060989,
"5 Oaklands B62 0JA": 90028499,
"47 Fairfield Rd DY8 5UJ": 90077535,
"150 Huntingtree Rd B63 4HP": 90093693,
"27 Milton Rd DY1 2JB": 90106884,
"21 Wells Rd DY5 3TB": 90022227,
"8 Corporation Rd DY2 7PX": 90070461
}
load_dotenv(ENV_FILEPATH)
CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME = os.getenv("CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME", None)
CDN_BUCKET_NAME = os.getenv("CDN_BUCKET_NAME", None)
def list_subdirectories(directory_path):
"""
List all subdirectories within a given directory.
:param directory_path: Path to the directory.
:return: A list of paths to the subdirectories.
"""
directory = Path(directory_path)
subdirectories = [subdir for subdir in directory.iterdir() if subdir.is_dir()]
return subdirectories
def list_files_in_directory(directory_path, file_extension=".jpg"):
"""
List all files with a specific extension within a given directory and its subdirectories.
:param directory_path: Path to the directory to scan.
:param file_extension: File extension to filter by.
:return: A list of paths to the files.
"""
# Convert the directory path to a Path object if it's not already one
directory = Path(directory_path) if not isinstance(directory_path, Path) else directory_path
# List all files of the specified type in the directory and subdirectories
file_list = [file for file in directory.rglob(f'*{file_extension}')]
return file_list
def create_images(input_path, uprn):
# Define the base directory path
base_directory = f"non_intrusive_photos/{uprn}"
print(f"Creating directory: {base_directory}") # Debug: print the directory to be created
# Need to create local directory if it doesn't exist
os.makedirs(base_directory, exist_ok=True)
# Define output paths
thumbnail_path = os.path.join(base_directory, "thumbnail.jpg")
full_hd_path = os.path.join(base_directory, "1080p.jpg")
webp_path = os.path.join(base_directory, "webp.webp") # Save as WebP format
# Load the image
with Image.open(input_path) as img:
# Create a thumbnail
thumbnail = img.copy()
thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS)
thumbnail.save(thumbnail_path, 'JPEG', quality=85)
# Create a 1080p version
full_hd = img.copy()
full_hd.thumbnail((1920, 1080), Image.Resampling.LANCZOS)
full_hd.save(full_hd_path, 'JPEG', quality=90)
# Convert to WebP for better compression
webp = img.copy()
webp.save(webp_path, 'WEBP', quality=90)
# Return paths to the processed images
return thumbnail_path, full_hd_path, webp_path
def upload_to_s3(bucket_name, file_path, object_name):
s3_client = boto3.client('s3')
s3_client.upload_file(file_path, bucket_name, object_name)
print(f"Uploaded {object_name} to S3 bucket {bucket_name}")
def upload_photos_to_s3(bucket_name, photo_paths):
# Upload each photo
for path in photo_paths:
object_name = path.split('/')[-1] # Assuming the path format is folder/filename
upload_to_s3(bucket_name, path, object_name)
def generate_cdn_url(distribution_domain, object_name):
return f"https://{distribution_domain}/{object_name}"
def process_and_upload_images(uprn, input_image_path, bucket_name, distribution_domain):
# Create images
thumbnail, full_hd, original = create_images(input_image_path, uprn=str(uprn))
# Upload images
upload_photos_to_s3(bucket_name, photo_paths=[thumbnail, full_hd, original])
# Generate CDN links
cdn_links = [generate_cdn_url(distribution_domain, path.split('/')[-1]) for path in [thumbnail, full_hd, original]]
# Delete local files
for path in [thumbnail, full_hd, original]:
os.remove(path)
return cdn_links
def app():
"""
This application is tasked with uploading the photos, recorded during the non-invasive surveys, to s3 and the
database.
To begin with, this app will simply read the files from the local machine, however we will come up with a more
efficient way to do this in the future.
:return:
"""
# List all files in the directory using pathlib
property_directories = list_subdirectories(PHOTO_DIRECTORY)
# For each property, we want to list all of the photos in the directory
for property_dir in property_directories:
photo_files = list_files_in_directory(property_dir)
uprn = FOLDER_UPRN_LOOKUP[property_dir.name]
# We now want to convert each file, and upload it to s3
for photo_filepath in photo_files:
process_and_upload_images(
uprn=uprn,
input_image_path=photo_filepath,
bucket_name=CDN_BUCKET_NAME,
distribution_domain=CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME
)

3
etl/non_intrusive_surveys/photos/requirements.txt Normal file
View file

@ -0,0 +1,3 @@
Pillow
boto3
python-dotenv

4
infrastructure/terraform/main.tf
View file

@ -81,6 +81,8 @@ resource "aws_db_instance" "default" {
# We will look to change this in the future but as we are pre-MVP at the time of setting this, we don't
# have major security demand and don't want to set this up now
publicly_accessible = true
# Specify the CA certificate with the default RDS CA certificate
ca_cert_identifier = "rds-ca-rsa2048-g1"
}
# Set up the bucket that recieve the csv uploads of epc to be retrofit
@ -147,7 +149,7 @@ module "route53" {
source = "./modules/route53"
domain_name = var.domain_name
api_url_prefix = var.api_url_prefix
providers = {
providers = {
aws.aws_use1 = aws.aws_use1
}
}

74
recommendations/Costs.py
View file

@ -37,6 +37,25 @@ MCS_SOLAR_PV_COST_DATA = {
"average_cost_per_kwh-Northern Ireland": 2126.09,
}
# This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average,
# to be conservative
MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = {
"Outer London": 13220,
"Inner London": 13220,
"South East England": 13547,
"South West England": 12776,
"East of England": 12585,
"East Midlands": 12239,
"West Midlands": 13182,
"North East England": 11829,
"North West England": 11714,
"Yorkshire and the Humber": 11919,
"Wales": 13701,
"Scotland": 12586,
"Northern Ireland": 12000, # There are hardly any air source heat pump installs going on in Northern Ireland
}
BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500
# This is based on quotes from installers
BATTERY_COST = 3500
@ -67,18 +86,12 @@ LOW_CARBON_COMBI_BOILER = 2200
# https://www.greenmatch.co.uk/boilers/35kw-boiler
# https://www.greenmatch.co.uk/boilers/40kw-boiler
# These are exclusive of installation costs
COMBI_BOILER_COSTS = {
CONDENSING_BOILER_COSTS = {
"30kw": 1550,
"35kw": 1610,
"40kw": 1625
}
CONVENTIONAL_BOILER_COSTS = {
"30kw": 1117,
"35kw": 1546,
"40kw": 1776
}
# Assumes 3 hours to remove each heater (including re-decorating)
ROOM_HEATER_REMOVAL_COST = 120
ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3
@ -91,6 +104,10 @@ DOUBLE_RADIATOR_COST = 300
FLUE_COST = 600
PIPEWORK_COST = 750 # Min cost is £500
# This is the cost per meter squared for cavity extraction
# https://www.checkatrade.com/blog/cost-guides/cavity-wall-insulation-removal-cost/
CAVITY_EXTRACTION_COST = 21.5
class Costs:
"""
@ -173,7 +190,7 @@ class Costs:
if not self.labour_adjustment_factor:
raise ValueError("Labour adjustment factor not found")
def cavity_wall_insulation(self, wall_area, material):
def cavity_wall_insulation(self, wall_area, material, is_extraction_and_refill=False):
"""
Calculates the total cost for cavity wall insulation based on material and labor costs,
including contingency, preliminaries, profit, and VAT.
@ -208,6 +225,13 @@ class Costs:
# Assume a team of 2
labour_days = (labour_hours / 8) / 2
if is_extraction_and_refill:
# bump up the cost of the work
total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area
# Additional 2 days work
labour_hours = labour_hours + (2 * 8)
labour_days = labour_days + 2
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
@ -602,12 +626,10 @@ class Costs:
preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
else:
preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
elif self.property.data["property-type"] == "Maisonette":
elif self.property.data["property-type"] in ["Maisonette", "Flat"]:
preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
elif self.property.data["property-type"] == "Bungalow":
preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
else:
raise ValueError("Unsupported property type - haven't handled flats")
demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"]
preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"]
@ -1168,7 +1190,7 @@ class Costs:
estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators)
return round(estimated_radiators)
def boiler(self, is_combi, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms):
def boiler(self, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms):
"""
Based on a basic estimate of median value £2600 to install a low carbon combi boiler
First time central heating vosts can als be found here:
@ -1176,7 +1198,7 @@ class Costs:
:return:
"""
unit_cost = COMBI_BOILER_COSTS[size] if is_combi else CONVENTIONAL_BOILER_COSTS[size]
unit_cost = CONDENSING_BOILER_COSTS[size]
# The unit cost is the cost without VAT
# We now need to estimate the cost of the works
labour_days = 2
@ -1235,3 +1257,29 @@ class Costs:
"labour_hours": labour_hours,
"labour_days": labour_days,
}
def air_source_heat_pump(self):
"""
Based on the region and type of property, this function will produce a cost estimation for an air source heat
pump. This cost will include the boiler upgrade scheme grant
"""
# This is the average cost of a project, we'll add some additional contingency
regional_cost = MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA[self.region]
total_cost = regional_cost * (1 + self.CONTINGENCY) - BOILER_UPGRADE_SCHEME_ASHP_VALUE
subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
vat = total_cost - subtotal_before_vat
# We assume 3 days installation
labour_days = 3
labour_hours = labour_days * 8
return {
"total": total_cost,
"subtotal": subtotal_before_vat,
"vat": vat,
"labour_hours": labour_hours,
"labour_days": labour_days,
}

3
recommendations/HeatingControlRecommender.py
View file

@ -35,6 +35,9 @@ class HeatingControlRecommender:
return
if heating_description in ["Air source heat pump, radiators, electric"]:
self.recommend_time_temperature_zone_controls()
def recommend_room_heaters_electric_controls(self):
"""
If the home has Room heaters, electric, we start by identifying potential heating controls that could

189
recommendations/HeatingRecommender.py
View file

@ -1,6 +1,4 @@
import pandas as pd
from recommendations.Costs import Costs
from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE
from recommendations.recommendation_utils import check_simulation_difference, override_costs
from backend.Property import Property
from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
@ -15,15 +13,24 @@ class HeatingRecommender:
self.property = property_instance
self.costs = Costs(self.property)
self.recommendations = []
self.heating_recommendations = []
self.heating_control_recommendations = []
def recommend(self, phase=0):
def recommend(self, has_cavity_or_loft_recommendations, phase=0):
"""
Produces heating recommendations
:param has_cavity_or_loft_recommendations: boolean indicating if we have produced a cavity or loft insulation
recommendation. If there are cavity or loft recommendations, the property would need to complete those measures
before being able to get the boiler upgrade scheme benefits. The messaging in the front end would be to
:param phase: indicates the phase of the retrofit programme
"""
# TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace
# the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this
# in the Costs class, stored as SYSTEM_FLUSH_COST
self.recommendations = []
self.heating_recommendations = []
self.heating_control_recommendations = []
# This first iteration of the recommender will provide very basic recommendation
# We recommend heating controls based on the main heating system
@ -79,8 +86,124 @@ class HeatingRecommender:
phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters
)
# We recommend air source heat pumps
# Heat pumps are suitable for all property types:
# https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/
# Just seems least probable for flats, so we'll allow houses and bungalows
# In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions
# and either allow or prevent the recommendation of an air source heat pump
suitable_property_type = self.property.data["property-type"] in ["House", "Bungalow"]
has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"]
if suitable_property_type and not has_air_source_heat_pump:
self.recommend_air_source_heat_pump(
phase=phase, has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations
)
return
def recommend_air_source_heat_pump(self, phase, has_cavity_or_loft_recommendations, _return=False):
"""
This method will implement the recommendation for an air source heat pump
This is ultimately an overhaul to the heating system and so is recommended as an alternative to other
heating system recommendations
:return:
"""
controls_recommender = HeatingControlRecommender(self.property)
controls_recommender.recommend(heating_description="Air source heat pump, radiators, electric")
ashp_costs = self.costs.air_source_heat_pump()
# We add the costs of the heating controls, onto each key in the costs dictionary
if controls_recommender.recommendation:
for key in ashp_costs:
ashp_costs[key] += controls_recommender.recommendation[0][key]
already_installed = "air_source_heat_pump" in self.property.already_installed
if already_installed:
ashp_costs = override_costs(ashp_costs)
description = "The property already has an air source heat pump, no further action needed."
else:
if controls_recommender.recommendation:
description = ("Install an air source heat pump, and upgrade heating controls to Smart Thermostats, "
"room sensors and smart radiator valves (time & temperature zone control).")
else:
description = "Install an air source heat pump."
# If the property does not have existing cavity and loft insulation, we include a note that the cost
# includes the boiler upgrade scheme and that the cavity and loft need to be treated, to ensure access
# to the funding
if has_cavity_or_loft_recommendations:
description = description + (f" The cost includes the £"
f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant. "
f"You must ensure that the property has an insulated cavity and "
f"270mm+ loft insulation to qualify for the grant")
else:
description = description + (f" The cost includes the £"
f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant")
simulation_config = {
"mainheat_energy_eff_ending": "Good",
"hot_water_energy_eff_ending": "Good"
}
# Installation of a boiler improves the hot water system so we need to reflect this in
# the outcome of the recommendation
heating_ending_config = MainHeatAttributes("Air source heat pump, radiators, electric").process()
hotwater_ending_config = HotWaterAttributes("From main system").process()
# If the property does not currently have electric main fuel, we'll simulate the change
fuel_ending_config = {}
if self.property.main_fuel["fuel_type"] != "electricity":
fuel_ending_config = MainFuelAttributes("electricity (not community)").process()
# Check the simulation differences
heating_simulation_config = check_simulation_difference(
new_config=heating_ending_config, old_config=self.property.main_heating
)
hotwater_simulation_config = check_simulation_difference(
new_config=hotwater_ending_config, old_config=self.property.hotwater
)
fuel_simulation_config = check_simulation_difference(
new_config=fuel_ending_config, old_config=self.property.main_fuel
)
simulation_config = {
**simulation_config,
**heating_simulation_config,
**hotwater_simulation_config,
**fuel_simulation_config,
}
if controls_recommender.recommendation:
# We should have just the single recommendation for heat controls, which is time
# and temperature zone controls
if len(controls_recommender.recommendation) != 1:
raise NotImplementedError("More than one heat controls recommendation for air source heat pump")
simulation_config = {
**simulation_config,
**controls_recommender.recommendation[0]["simulation_config"]
}
ashp_recommendation = {
"phase": phase,
"parts": [
# TODO
],
"type": "heating",
"description": description,
"starting_u_value": None,
"new_u_value": None,
"sap_points": None,
"already_installed": already_installed,
"simulation_config": simulation_config,
**ashp_costs
}
if _return:
return [ashp_recommendation]
self.heating_recommendations.append(ashp_recommendation)
@staticmethod
def check_simulation_difference(old_config, new_config):
"""
@ -144,7 +267,7 @@ class HeatingRecommender:
recommendation_description = f"{description} and {controls_description}"
already_installed = "cavity_wall_insulation" in self.property.already_installed
already_installed = "heating_controls" in self.property.already_installed
if already_installed:
total_costs = override_costs(total_costs)
recommendation_description = "Heating system has already been upgraded, no further action needed."
@ -191,7 +314,7 @@ class HeatingRecommender:
return output
def recommend_hhr_storage_heaters(self, phase, system_change, heating_controls_only):
def recommend_hhr_storage_heaters(self, phase, system_change, heating_controls_only, _return=False):
"""
We will recommend upgrading to a high heat retention storage system, if the current system is not already
high heat retention storage
@ -200,6 +323,8 @@ class HeatingRecommender:
:param system_change: Indicates if we are recommending a different type of heating system, compared to the
current system
:param heating_controls_only: Indicates if we should include a recommendation for just heating controls
:param _return: Indicates if we should return the recommendations, rather than appending them to the
recommendations list
:return:
"""
@ -253,8 +378,10 @@ class HeatingRecommender:
heating_controls_only=heating_controls_only,
system_change=system_change
)
if _return:
return recommendations
self.recommendations.extend(recommendations)
self.heating_recommendations.extend(recommendations)
@staticmethod
def estimate_boiler_size(property_type, built_form, floor_area, floor_height, num_heated_rooms):
@ -312,7 +439,15 @@ class HeatingRecommender:
simulation_config = {}
boiler_costs = {}
boiler_recommendation = {}
if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]:
has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]
has_inefficient_mains_water = (
self.property.hotwater["clean_description"] in ["From main system"] and
self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"]
)
if has_inefficient_space_heating or has_inefficient_mains_water:
boiler_size = self.estimate_boiler_size(
property_type=self.property.data["property-type"],
built_form=self.property.data["built-form"],
@ -321,22 +456,12 @@ class HeatingRecommender:
num_heated_rooms=self.property.data["number-heated-rooms"],
)
# We recommend a combi boiler under the following conditions
# 1) If there are 4 or fewer rooms (we don't use heqted rooms because none of the rooms could be
# heated if there is no existing heating system).
# 2) There 1 or fewer bathrooms
# Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple
# bathrooms
is_combi = (
(self.property.number_of_rooms <= 4) and
(self.property.n_bathrooms in [None, 0, 1])
)
if is_combi:
description = "Upgrade to a new combi boiler"
else:
description = "Upgrade to a new gas condensing boiler"
description = "Upgrade to a new condensing boiler"
simulation_config = {"mainheat_energy_eff_ending": "Good"}
simulation_config = {
"mainheat_energy_eff_ending": "Good",
"hot_water_energy_eff_ending": "Good"
}
if system_change:
# Installation of a boiler improves the hot water system so we need to reflect this in
# the outcome of the recommendation
@ -359,11 +484,9 @@ class HeatingRecommender:
**heating_simulation_config,
**hotwater_simulation_config,
**fuel_simulation_config,
"hot_water_energy_eff_ending": "Good"
}
boiler_costs = self.costs.boiler(
is_combi=is_combi,
size=f"{boiler_size}kw",
exising_room_heaters=exising_room_heaters,
system_change=system_change,
@ -397,9 +520,13 @@ class HeatingRecommender:
controls_recommender.recommend(heating_description="Boiler and radiators, mains gas")
# We may have 2 recommendations from the heating controls
if not controls_recommender.recommendation:
if not controls_recommender.recommendation and not boiler_recommendation:
return
if not system_change and len(boiler_recommendation):
# If there is not a system change, we add the boiler recommendation at point.
self.heating_recommendations.extend([boiler_recommendation])
if system_change:
# We combine the heating and controls recommendations, in the case of a system change
combined_recommendations = []
@ -416,12 +543,12 @@ class HeatingRecommender:
combined_recommendations.extend(combined_recommendation)
# Overwrite the existing boiler recommendation
self.recommendations.extend(combined_recommendations)
self.heating_recommendations.extend(combined_recommendations)
else:
# We increment the recommendation phase, since the heating controls are separate from the boiler upgrade
# but we'll only upgrade if we have a heating recommendation
has_heating_recommendation = any(
recommendation["type"] == "heating" for recommendation in self.recommendations
rec["type"] == "heating" for rec in self.heating_recommendations
)
if has_heating_recommendation:
recommendation_phase += 1
@ -430,6 +557,6 @@ class HeatingRecommender:
for recommendation in controls_recommender.recommendation:
recommendation["phase"] = recommendation_phase
self.recommendations.extend(controls_recommender.recommendation)
self.heating_control_recommendations.extend(controls_recommender.recommendation)
return

173
recommendations/Mds.py Normal file
View file

@ -0,0 +1,173 @@
from backend.Property import Property
from recommendations.FloorRecommendations import FloorRecommendations
from recommendations.WallRecommendations import WallRecommendations
from recommendations.RoofRecommendations import RoofRecommendations
from recommendations.VentilationRecommendations import VentilationRecommendations
from recommendations.FireplaceRecommendations import FireplaceRecommendations
from recommendations.LightingRecommendations import LightingRecommendations
from recommendations.SolarPvRecommendations import SolarPvRecommendations
from recommendations.WindowsRecommendations import WindowsRecommendations
from recommendations.HeatingRecommender import HeatingRecommender
from recommendations.HotwaterRecommendations import HotwaterRecommendations
from recommendations.SecondaryHeating import SecondaryHeating
from recommendations.Recommendations import Recommendations
class Mds:
"""
Handles the contruction of the MDS report
"""
def __init__(self, property_instance: Property, materials):
self.property_instance = property_instance
self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials)
self.wall_recommender = WallRecommendations(property_instance=property_instance, materials=materials)
self.roof_recommender = RoofRecommendations(property_instance=property_instance, materials=materials)
self.ventilation_recomender = VentilationRecommendations(
property_instance=property_instance, materials=materials
)
self.fireplace_recommender = FireplaceRecommendations(property_instance=property_instance)
self.lighting_recommender = LightingRecommendations(property_instance=property_instance, materials=materials)
self.windows_recommender = WindowsRecommendations(property_instance=property_instance, materials=materials)
self.solar_recommender = SolarPvRecommendations(property_instance=property_instance)
self.heating_recommender = HeatingRecommender(property_instance=property_instance)
self.hotwater_recommender = HotwaterRecommendations(property_instance=property_instance)
self.secondary_heating_recommender = SecondaryHeating(property_instance=property_instance)
def build(self):
if self.property_instance.measures is None:
raise NotImplementedError("No measures in the property - implement me")
measures = self.property_instance.measures
measure_config_list = [list(m.keys())[0] for m in measures]
not_implemented_measures = [
"party_wall_insulation",
"ground_source_heat_pump",
"shared_ground_loops",
"communal_heat_networks",
"district_heating_networks",
"solar_thermal",
"draught_proofing",
"ev_charging",
"battery",
]
# Check if we have a not implemented measure
if any([m in not_implemented_measures for m in measure_config_list]):
raise NotImplementedError("Not implemented measure in the property - implement me")
mds_recommendations = []
errors = []
# TODO: Could use a decarator to reduce the boilerplate code - insert_recommendation_id and then the append
if "external_wall_insulation" in measure_config_list:
recs = self.wall_recommender.mds_recommend_ewi(phase=0)
if not recs:
raise Exception("No recommendations for external wall insulation")
recs = self.insert_recommendation_id(recs, measures, "external_wall_insulation")
mds_recommendations.append(recs)
if "cavity_wall_insulation" in measure_config_list:
recs = self.wall_recommender.mds_recommend_cavity_wall_insulation(phase=0)
recs = self.insert_recommendation_id(recs, measures, "cavity_wall_insulation")
mds_recommendations.append(recs)
if "loft_insulation" in measure_config_list:
# Check if the roof is suitable for loft insulation
if self.property_instance.roof['is_roof_room']:
errors.append("Roof is a room")
else:
recs = self.roof_recommender.mds_loft_insulation(phase=0)
if not recs:
raise Exception("No recommendations for loft insulation")
recs = self.insert_recommendation_id(recs, measures, "loft_insulation")
mds_recommendations.append(recs)
if "internal_wall_insulation" in measure_config_list:
raise Exception("check me out 4")
self.wall_recommender.recommend(phase=0)
if "suspended_floor_insulation" in measure_config_list:
raise Exception("check me out 5")
self.floor_recommender.recommend(phase=0)
if "solid_floor_insulation" in measure_config_list:
raise Exception("check me out 6")
self.floor_recommender.recommend(phase=0)
if "air_source_heat_pump" in measure_config_list:
recs = self.heating_recommender.recommend_air_source_heat_pump(
phase=0, has_cavity_or_loft_recommendations=False, _return=True
)
recs = self.insert_recommendation_id(recs, measures, "air_source_heat_pump")
mds_recommendations.append(recs)
if "electric_storage_heaters" in measure_config_list:
recs = self.heating_recommender.recommend_hhr_storage_heaters(
phase=0, system_change=True, heating_controls_only=False, _return=True
)
recs = self.insert_recommendation_id(recs, measures, "electric_storage_heaters")
mds_recommendations.append(recs)
if "low_energy_lighting" in measure_config_list:
raise Exception("check me out 9")
self.lighting_recommender.recommend(phase=0)
if "cylinder_insulation" in measure_config_list:
raise Exception("check me out 10")
self.hotwater_recommender.recommend(phase=0)
if "smart_controls" in measure_config_list:
raise Exception("check me out 11")
self.heating_recommender.recommend(phase=0)
if "zone_controls" in measure_config_list:
raise Exception("check me out 12")
self.heating_recommender.recommend(phase=0)
if "trvs" in measure_config_list:
raise Exception("check me out 13")
self.heating_recommender.recommend(phase=0)
if "solar_pv" in measure_config_list:
recs = self.solar_recommender.mds_recommend(phase=0, solar_pv_percentage=0.5)
recs = self.insert_recommendation_id(recs, measures, "solar_pv")
mds_recommendations.append(recs)
if "double_glazing" in measure_config_list:
raise Exception("check me out 15")
self.windows_recommender.recommend(phase=0)
if "mechanical_ventilation" in measure_config_list:
raise Exception("check me out 16")
self.ventilation_recomender.recommend(phase=0)
if "gas_boiler" in measure_config_list:
raise Exception("check me out 17")
self.heating_recommender.recommend(phase=0)
if "flat_roof_insulation" in measure_config_list:
raise Exception("check me out 18")
self.roof_recommender.recommend(phase=0)
if "room_in_roof_insulation" in measure_config_list:
raise Exception("check me out 19")
self.roof_recommender.recommend(phase=0)
property_representative_recommendations = Recommendations.create_representative_recommendations(
mds_recommendations, non_invasive_recommendations=[]
)
return property_representative_recommendations, errors
@staticmethod
def insert_recommendation_id(recommendations, measures, measure_name):
# Insert the recommendation identifier into this recommendation
measure_config = [m for m in measures if measure_name in m][0]
for r in recommendations:
r["recommendation_id"] = list(measure_config.values())[0]
return recommendations

79
recommendations/Recommendations.py
View file

@ -109,13 +109,51 @@ class Recommendations:
# Heating and Electical systems
if "heating" not in self.exclusions:
self.heating_recommender.recommend(phase=phase)
if self.heating_recommender.recommendations:
property_recommendations.append(self.heating_recommender.recommendations)
cavity_or_loft_recommendations = [
r for r in self.wall_recomender.recommendations + self.roof_recommender.recommendations
if r["type"] in ["cavity_wall_insulation", "loft_insulation"]
]
has_cavity_or_loft_recommendations = len(cavity_or_loft_recommendations) > 0
self.heating_recommender.recommend(
phase=phase, has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations
)
if (
self.heating_recommender.heating_recommendations or
self.heating_recommender.heating_control_recommendations
):
# We split into first and second phase recommendations
first_phase_recommendations = [
r for r in (
self.heating_recommender.heating_recommendations +
self.heating_recommender.heating_control_recommendations
)
if r["phase"] == phase
]
second_phase_recommendations = [
r for r in (
self.heating_recommender.heating_recommendations +
self.heating_recommender.heating_control_recommendations
)
if r["phase"] == phase + 1
]
if first_phase_recommendations:
property_recommendations.append(first_phase_recommendations)
if second_phase_recommendations:
property_recommendations.append(second_phase_recommendations)
# We check if we have distinct heating and heating controls recommendations
# If so, we increment by 2 (one of the heating system, one for the heating controls)
# otherwise we incremenet by 1
max_used_phase = max([rec["phase"] for rec in self.heating_recommender.recommendations])
max_used_phase = max(
[rec["phase"] for rec in
self.heating_recommender.heating_recommendations +
self.heating_recommender.heating_control_recommendations]
)
amount_to_increment = max_used_phase - phase + 1
phase += amount_to_increment
@ -149,12 +187,14 @@ class Recommendations:
property_recommendations = self.insert_temp_recommendation_id(property_recommendations)
# We also need to create the representative recommendations for each recommendation type
property_representative_recommendations = self.create_representative_recommendations(property_recommendations)
property_representative_recommendations = self.create_representative_recommendations(
property_recommendations, non_invasive_recommendations=self.property_instance.non_invasive_recommendations
)
return property_recommendations, property_representative_recommendations
@staticmethod
def create_representative_recommendations(property_recommendations):
def create_representative_recommendations(property_recommendations, non_invasive_recommendations):
"""
This method will create a representative recommendation for each recommendation type
In order to create a representative recommendation, we choose the recommendation that has:
@ -169,6 +209,13 @@ class Recommendations:
for recommendations_by_type in property_recommendations:
# If the property was initially surveyed as filled, but the cavity was only partially filled, we don't
# want to include the cavity wall insulation recommendation in the defaults
# if (recommendations_by_type[0].get("type") == "cavity_wall_insulation") and (
# "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations
# ):
# continue
if recommendations_by_type[0].get("type") == "mechanical_ventilation":
continue
@ -238,13 +285,13 @@ class Recommendations:
property_sap_predictions = all_predictions["sap_change_predictions"][
all_predictions["sap_change_predictions"]["property_id"] == str(property_instance.id)
]
].copy()
property_heat_predictions = all_predictions["heat_demand_predictions"][
all_predictions["heat_demand_predictions"]["property_id"] == str(property_instance.id)
]
].copy()
property_carbon_predictions = all_predictions["carbon_change_predictions"][
all_predictions["carbon_change_predictions"]["property_id"] == str(property_instance.id)
]
].copy()
property_recommendations = recommendations[property_instance.id].copy()
@ -272,6 +319,8 @@ class Recommendations:
current_epc_rating=property_instance.data["current-energy-rating"],
)
# TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are
# actually implemented
expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
epc_energy_consumption=expected_heat_demand,
current_epc_rating=property_instance.data["current-energy-rating"],
@ -281,6 +330,10 @@ class Recommendations:
current_adjusted_energy - expected_adjusted_energy
)
# TODO: We should determine if the home is gas & electricity or just electricity
current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy)
expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy)
for recommendations_by_type in property_recommendations:
for rec in recommendations_by_type:
@ -355,4 +408,10 @@ class Recommendations:
rec["heat_demand"] is None) or (rec["energy_cost_savings"] is None):
raise ValueError("sap points, co2 or heat demand is missing")
return property_recommendations, current_adjusted_energy, expected_adjusted_energy
return (
property_recommendations,
current_adjusted_energy,
expected_adjusted_energy,
current_energy_bill,
expected_energy_bill
)

23
recommendations/RoofRecommendations.py
View file

@ -54,6 +54,26 @@ class RoofRecommendations:
]
]
def mds_loft_insulation(self, phase):
"""
For usages within the mds report
:param phase:
:return:
"""
self.recommendations = []
insulation_thickness = convert_thickness_to_numeric(
self.property.roof["insulation_thickness"],
self.property.roof["is_pitched"],
self.property.roof["is_flat"]
)
u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band})
self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
return self.recommendations
def recommend(self, phase):
if self.property.roof["has_dwelling_above"]:
@ -210,6 +230,7 @@ class RoofRecommendations:
already_installed = "loft_insulation" in self.property.already_installed
if already_installed:
cost_result = override_costs(cost_result)
new_thickness = insulation_thickness + material["depth"]
elif material["type"] == "flat_roof_insulation":
cost_result = self.costs.flat_roof_insulation(
floor_area=self.property.insulation_floor_area,
@ -219,6 +240,7 @@ class RoofRecommendations:
already_installed = "flat_roof_insulation" in self.property.already_installed
if already_installed:
cost_result = override_costs(cost_result)
new_thickness = None
else:
raise ValueError("Invalid material type")
@ -239,6 +261,7 @@ class RoofRecommendations:
"new_u_value": new_u_value,
"sap_points": None,
"already_installed": already_installed,
"new_thickness": new_thickness,
**cost_result
}
)

52
recommendations/SolarPvRecommendations.py
View file

@ -35,6 +35,46 @@ class SolarPvRecommendations:
return trimmed_list
def mds_recommend(self, phase=None, solar_pv_percentage=0.5):
# For specific usage within the mds report
solar_pv_roof_area = self.property.get_solar_pv_roof_area(solar_pv_percentage)
number_solar_panels = np.floor(solar_pv_roof_area / self.SOLAR_PANEL_AREA)
solar_panel_wattage = number_solar_panels * self.SOLAR_PANEL_WATTAGE
solar_panel_wattage = np.clip(
a=solar_panel_wattage, a_min=self.MIN_SYSTEM_WATTAGE, a_max=self.MAX_SYSTEM_WATTAGE
)
# We now have a property which is potentially suitable for solar PV
roof_coverage_percent = round(solar_pv_percentage * 100)
# Given the wattage, we estimate the cost of the solar PV system. This is based on the MCS database
# of solar PV installations
cost_result = self.costs.solar_pv(wattage=solar_panel_wattage, has_battery=False)
kw = np.floor(solar_panel_wattage / 100) / 10
description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p"
f"anel system on {round(roof_coverage_percent)}% the roof.")
return [
{
"phase": phase,
"parts": [],
"type": "solar_pv",
"description": description,
"starting_u_value": None,
"new_u_value": None,
"sap_points": None,
"already_installed": False,
**cost_result,
# This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale
# back up here
"photo_supply": roof_coverage_percent,
"has_battery": False
}
]
def recommend(self, phase):
"""
We check if a property is potentially suitable for solar PV based on the following criteria:
@ -44,7 +84,7 @@ class SolarPvRecommendations:
:return:
"""
is_valid_property_type = self.property.data["property-type"] in ["House", "Bungalow"]
is_valid_property_type = self.property.data["property-type"] in ["House", "Bungalow", "Maisonette"]
is_valid_roof_type = (
self.property.roof["is_flat"] or self.property.roof["is_pitched"] or self.property.roof["is_roof_room"]
)
@ -56,14 +96,18 @@ class SolarPvRecommendations:
if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv:
return
solar_pv_percentage = self.property.solar_pv_percentage
# We round up to the neaest 10%
solar_pv_percentage = np.ceil(solar_pv_percentage * 10) / 10
# For the solar recommendations, we produce the following scenarios:
# 1) Solar panels only, we present a high, medium and low coverage
# 2) With and without battery
roof_coverage_scenarios = [
self.property.solar_pv_percentage - 0.1, self.property.solar_pv_percentage,
solar_pv_percentage - 0.1, solar_pv_percentage,
]
if self.property.solar_pv_percentage <= 0.4:
roof_coverage_scenarios.append(self.property.solar_pv_percentage + 0.1)
if solar_pv_percentage <= 0.4:
roof_coverage_scenarios.append(solar_pv_percentage + 0.1)
# We make sure we haven't gone too low or high - we allow no more than 60% coverage
roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 0.6]
# If we only have two scenarios, we add a coverage scenario 10% less than the smallest

150
recommendations/WallRecommendations.py
View file

@ -6,9 +6,10 @@ import pandas as pd
from datatypes.enums import QuantityUnits
from backend.Property import Property
from BaseUtility import Definitions
from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes
from recommendations.recommendation_utils import (
r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
get_recommended_part, get_wall_u_value, override_costs
get_recommended_part, get_wall_u_value, override_costs, check_simulation_difference
)
from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
from recommendations.Costs import Costs
@ -53,6 +54,26 @@ class WallRecommendations(Definitions):
# threshold
NEW_BUILD_INSULATED = 0.75
# These are the ending descriptions we consider for walls with external insulation
EXTERNALLY_INSULATED_WALL_DESCRIPTIONS = {
"solid_brick": "Solid brick, with external insulation",
"cob": "Cob, with external insulation",
"system_built": "System built, with external insulation",
"granite_or_whinstone": 'Granite or whinstone, with external insulation',
"sandstone_or_limestone": 'Sandstone or limestone, with external insulation',
"timber_frame": "Timber frame, with external insulation"
}
# These are the ending descriptions we consider for walls with internal insulation
INTERNALLY_INSULATED_WALL_DESCRIPTIONS = {
"solid_brick": "Solid brick, with internal insulation",
"cob": "Cob, with internal insulation",
"system_built": "System built, with internal insulation",
"granite_or_whinstone": 'Granite or whinstone, with internal insulation',
"sandstone_or_limestone": 'Sandstone or limestone, with internal insulation',
"timber_frame": "Timber frame, with internal insulation"
}
def __init__(
self,
property_instance: Property,
@ -103,6 +124,47 @@ class WallRecommendations(Definitions):
return True
def mds_recommend_cavity_wall_insulation(self, phase=None):
# Function specifically for cavity wall insulation, for usage in the mds report
self.recommendations = []
insulation_thickness = self.property.walls["insulation_thickness"]
u_value = get_wall_u_value(
clean_description=self.property.walls["clean_description"],
age_band=self.property.age_band,
is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
)
# Test filling cavity
self.find_cavity_insulation(u_value, insulation_thickness, phase)
return self.recommendations
def mds_recommend_ewi(self, phase=None):
# Function specifically for external wall insulation, for usage in the mds report
self.recommendations = []
u_value = self.property.walls["thermal_transmittance"]
if u_value is None:
u_value = get_wall_u_value(
clean_description=self.property.walls["clean_description"],
age_band=self.property.age_band,
is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
)
# EWI
ewi_recommendations = self._find_insulation(
u_value=u_value,
insulation_materials=pd.DataFrame(self.external_wall_insulation_materials),
non_insulation_materials=self.external_wall_non_insulation_materials,
phase=phase
)
return ewi_recommendations
def recommend(self, phase=0):
# if building built after 1990 + we're able to identify U-value +
# U-value less than 0.18 and if in or close to a conversation area,
@ -113,7 +175,9 @@ class WallRecommendations(Definitions):
insulation_thickness = self.property.walls["insulation_thickness"]
# We check if the wall is already insulated and if so, we exit
if (insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]:
if ((insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]) and (
"cavity_extract_and_refill" not in self.property.non_invasive_recommendations
):
return
if u_value:
@ -216,15 +280,41 @@ class WallRecommendations(Definitions):
if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE:
lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
is_extraction_and_refill = "cavity_extract_and_refill" in self.property.non_invasive_recommendations
cost_result = self.costs.cavity_wall_insulation(
wall_area=self.property.insulation_wall_area,
material=material.to_dict(),
is_extraction_and_refill=is_extraction_and_refill
)
already_installed = "cavity_wall_insulation" in self.property.already_installed
if already_installed:
cost_result = override_costs(cost_result)
if is_extraction_and_refill:
description = f"Extract and refill cavity wall insulation with {material['description']}"
else:
description = self._make_description(material)
# updated the new u-value with the best possible our installers have
new_u_value = max(0.31, new_u_value)
wall_ending_config = WallAttributes("Cavity wall, filled cavity").process()
simulation_config = {}
if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
simulation_config = {
"walls_energy_eff_ending": "Good",
"walls_thermal_transmittance_ending": new_u_value
}
walls_simulation_config = check_simulation_difference(
new_config=wall_ending_config, old_config=self.property.walls, prefix="walls_"
)
simulation_config = {**simulation_config, **walls_simulation_config}
recommendations.append(
{
"phase": phase,
@ -237,17 +327,44 @@ class WallRecommendations(Definitions):
)
],
"type": "cavity_wall_insulation",
"description": self._make_description(material),
"description": description,
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": None,
"already_installed": already_installed,
"simulation_config": simulation_config,
**cost_result
}
)
self.recommendations = recommendations
def get_internal_external_wall_description(self, description_map, new_u_value):
if self.property.walls["is_solid_brick"]:
return description_map["solid_brick"]
if self.property.walls["is_cob"]:
return description_map["cob"]
if self.property.walls["is_system_built"]:
return description_map["system_built"]
if self.property.walls["is_granite_or_whinstone"]:
return description_map["granite_or_whinstone"]
if self.property.walls["is_sandstone_or_limestone"]:
return description_map["sandstone_or_limestone"]
if self.property.walls["is_timber_frame"]:
return description_map["timber_frame"]
if "Average thermal transmittance" in self.property.walls["clean_description"]:
if new_u_value is None:
raise ValueError("New u value is None")
return f'Average thermal transmittance {new_u_value} W/m-¦K'
raise NotImplementedError("Not implemented yet")
def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase):
lowest_selected_u_value = None
@ -286,6 +403,10 @@ class WallRecommendations(Definitions):
if already_installed:
cost_result = override_costs(cost_result)
new_description = self.get_internal_external_wall_description(
self.INTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value
)
elif material["type"] == "external_wall_insulation":
cost_result = self.costs.external_wall_insulation(
wall_area=self.property.insulation_wall_area,
@ -295,9 +416,31 @@ class WallRecommendations(Definitions):
already_installed = "external_wall_insulation" in self.property.already_installed
if already_installed:
cost_result = override_costs(cost_result)
new_description = self.get_internal_external_wall_description(
self.EXTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value
)
else:
raise ValueError("Invalid material type")
wall_ending_config = WallAttributes(new_description).process()
simulation_config = {}
if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
simulation_config = {
"walls_energy_eff_ending": "Good"
}
walls_simulation_config = check_simulation_difference(
new_config=wall_ending_config, old_config=self.property.walls, prefix="walls_"
)
simulation_config = {
**walls_simulation_config,
**simulation_config,
"walls_thermal_transmittance_ending": new_u_value
}
recommendations.append(
{
"phase": phase,
@ -315,6 +458,7 @@ class WallRecommendations(Definitions):
"new_u_value": new_u_value,
"already_installed": already_installed,
"sap_points": None,
"simulation_config": simulation_config,
**cost_result
}
)

2
recommendations/WindowsRecommendations.py
View file

@ -4,7 +4,7 @@ import numpy as np
from backend.Property import Property
from recommendations.Costs import Costs
from recommendation_utils import override_costs
from recommendations.recommendation_utils import override_costs
class WindowsRecommendations:

9
recommendations/recommendation_utils.py
View file

@ -756,15 +756,18 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned):
return cavity_age
def check_simulation_difference(old_config, new_config):
def check_simulation_difference(old_config, new_config, prefix=""):
"""
Given two dictionaries, that describe the heating control configurations, this method will compare the two
and pick out the differences. These differences will be things that have been added and things that have been
removed. This will be used to determine how we should be updating the configuration in the simulation
:return:
"""
differences = {key + "_ending": new_config[key] for key in new_config if old_config[key] != new_config[key]}
differences = {}
for key in new_config:
if old_config[key] != new_config[key]:
new_key = prefix + key + "_ending" if key in ["is_assumed", "thermal_transmittance"] else key + "_ending"
differences[new_key] = new_config[key]
return differences

944
recommendations/tests/test_air_source_heat_pump.py Normal file
View file

@ -0,0 +1,944 @@
import pandas as pd
import msgpack
from datetime import datetime
from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
from backend.Property import Property
from recommendations.HeatingRecommender import HeatingRecommender
from recommendations.Recommendations import Recommendations
from etl.epc.Record import EPCRecord
from etl.solar.SolarPhotoSupply import SolarPhotoSupply
from backend.ml_models.api import ModelApi
def find_examples():
""" Some scrappy helper code to find EPC examples"""
# Let's look for some testing data, where the only thing different pre and post is the installation of an
# air source heat pump
data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev",
file_key="sap_change_model/2024-03-24-15-51-13/dataset_no_cleaning.parquet"
)
# Firstly, take records where before there was no air source heat pump and afterwards there was
data = data[
data["has_air_source_heat_pump_ending"] & ~data["has_air_source_heat_pump"]
]
# Start with a property that has a boiler
data = data[data["has_boiler"]]
static_columns = [
# Walls
'walls_thermal_transmittance_ending',
'is_filled_cavity_ending',
'is_park_home_ending',
'walls_insulation_thickness_ending',
'external_insulation_ending',
'internal_insulation_ending',
# Floors
# 'floor_thermal_transmittance_ending', # Don't subset on this, because it changes based on floor area
'floor_insulation_thickness_ending',
# Roof
'roof_thermal_transmittance_ending',
'is_at_rafters_ending',
'roof_insulation_thickness_ending',
# Hot water - air source heat pump will shange the hot water system (probably from whatever it was -> main)
# 'heater_type_ending',
# 'system_type_ending',
# 'thermostat_characteristics_ending',
# 'heating_scope_ending',
# 'energy_recovery_ending',
# 'hotwater_tariff_type_ending',
# 'extra_features_ending',
# 'chp_systems_ending',
# 'distribution_system_ending',
# 'no_system_present_ending',
# 'appliance_ending',
# Heating - Will change when installing an ASHP
# 'has_radiators_ending',
# 'has_fan_coil_units_ending',
# 'has_pipes_in_screed_above_insulation_ending',
# 'has_pipes_in_insulated_timber_floor_ending',
# 'has_pipes_in_concrete_slab_ending',
# 'has_boiler_ending',
# 'has_air_source_heat_pump_ending', # We want the air source heat pump to change
# 'has_room_heaters_ending',
# 'has_electric_storage_heaters_ending',
# 'has_warm_air_ending',
# 'has_electric_underfloor_heating_ending',
# 'has_electric_ceiling_heating_ending',
# 'has_community_scheme_ending',
# 'has_ground_source_heat_pump_ending',
# 'has_no_system_present_ending',
# 'has_portable_electric_heaters_ending',
# 'has_water_source_heat_pump_ending',
# 'has_electric_heat_pump_ending',
# 'has_micro-cogeneration_ending',
# 'has_solar_assisted_heat_pump_ending',
# 'has_exhaust_source_heat_pump_ending',
# 'has_community_heat_pump_ending',
# 'has_electric_ending',
# 'has_mains_gas_ending',
# 'has_wood_logs_ending', 'has_coal_ending', 'has_oil_ending',
# 'has_wood_pellets_ending', 'has_anthracite_ending', 'has_dual_fuel_mineral_and_wood_ending',
# 'has_smokeless_fuel_ending', 'has_lpg_ending', 'has_b30k_ending', 'has_electricaire_ending',
# 'has_assumed_for_most_rooms_ending', 'has_underfloor_heating_ending',
# 'thermostatic_control_ending',
# 'charging_system_ending',
# 'switch_system_ending',
# 'no_control_ending',
# 'dhw_control_ending',
# 'community_heating_ending',
# 'multiple_room_thermostats_ending',
# 'auxiliary_systems_ending',
# 'trvs_ending',
# 'rate_control_ending',
# Window
'glazing_type_ending',
# Fuel - could change with ASHP
# 'fuel_type_ending',
# 'main-fuel_tariff_type_ending',
# 'is_community_ending',
# 'no_individual_heating_or_community_network_ending',
# 'complex_fuel_type_ending',
'mechanical_ventilation_ending', 'secondheat_description_ending', 'glazed_type_ending',
'multi_glaze_proportion_ending', 'low_energy_lighting_ending', 'number_open_fireplaces_ending',
'solar_water_heating_flag_ending',
'photo_supply_ending',
'energy_tariff_ending',
'extension_count_ending',
'total_floor_area_ending',
# 'hot_water_energy_eff_ending',
'floor_energy_eff_ending',
'windows_energy_eff_ending',
'walls_energy_eff_ending',
'sheating_energy_eff_ending',
'roof_energy_eff_ending',
# 'mainheat_energy_eff_ending',
# 'mainheatc_energy_eff_ending',
'lighting_energy_eff_ending',
'number_habitable_rooms_ending',
'number_heated_rooms_ending',
]
for col in static_columns:
base_starting = col.split("_ending")[0]
if base_starting + "_starting" in data.columns:
starting_col = base_starting + "_starting"
else:
starting_col = base_starting
# Filter
print("Column: %s" % col)
print("Starting size: %s" % data.shape[0])
data = data[data[starting_col] == data[col]]
print("Ending size: %s" % data.shape[0])
z = data[['uprn', col, starting_col]]
# Great example UPRNs
# 100030969273
# 10034685399 - Completely transforms the heating and hot water systems in the home (goes from oil -> electricity)
# 100091200828 - goes from a liquid petroleum gas boiler to ashp
# Look for starting with a gas boiler
data[
data["has_boiler"] & data["has_radiators"] & data["has_mains_gas"] & ~data["has_boiler_ending"]
]
# UPRN: 100011776843
class TestAirSourceHeatPump:
def test_eligible(self):
# This tests a house, which will be suitable for an air source heat pump
epc_record = EPCRecord()
epc_record.prepared_epc = {
"county": "Broxbourne",
"mainheat-energy-eff": "Good",
"hot-water-energy-eff": "Good",
"mainheatc-energy-eff": "Good",
"number-heated-rooms": 5,
"property-type": "House",
"built-form": "Semi-Detached"
}
property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
property_instance.main_heating = {
'original_description': 'Boiler and radiators, mains gas',
"clean_description": "Boiler and radiators, mains gas",
'has_radiators': True,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
'has_air_source_heat_pump': False,
'has_room_heaters': False, 'has_electric_storage_heaters': False,
'has_warm_air': False,
'has_electric_underfloor_heating': False,
'has_electric_ceiling_heating': False, 'has_community_scheme': False,
'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False,
'has_water_source_heat_pump': False, 'has_electric': False,
'has_mains_gas': True, 'has_wood_logs': False,
'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
'has_anthracite': False,
'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False,
'has_lpg': False, 'has_assumed': False,
'has_electricaire': False, 'has_assumed_for_most_rooms': False,
'has_underfloor_heating': False,
"has_electric_heat_pumps": False,
"has_micro-cogeneration": False
}
property_instance.main_fuel = {
'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas',
'tariff_type': None,
'is_community': False, 'no_individual_heating_or_community_network': False,
'complex_fuel_type': None
}
property_instance.hotwater = {
'original_description': 'From main system',
'clean_description': 'From main system',
'heater_type': None,
'system_type': 'from main system',
'thermostat_characteristics': None, 'heating_scope': None,
'energy_recovery': None, 'tariff_type': None,
'extra_features': None, 'chp_systems': None, 'distribution_system': None,
'no_system_present': None,
'assumed': False, "appliance": None
}
property_instance.main_heating_controls = {
'original_description': 'Programmer, room thermostat and TRVs',
'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer',
'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None
}
recommender = HeatingRecommender(property_instance=property_instance)
assert not recommender.heating_recommendations
recommender.recommend(phase=0)
assert recommender.recommendation is None
def test_air_source_heat_pump_gas_boiler_starting(self):
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor',
'floor-height': '2.62', 'heating-cost-potential': '599', 'unheated-corridor-length': '',
'hot-water-cost-potential': '67', 'construction-age-band': 'England and Wales: 1950-1966',
'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Good',
'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '72',
'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '913',
'address3': '', 'mainheatcont-description': 'Programmer, no room thermostat', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'Wigan', 'fixed-lighting-outlets-count': '9',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '210',
'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', 'constituency': 'E14001039',
'co2-emissions-potential': '2.6', 'number-heated-rooms': '4',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '180',
'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-02-15',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '78', 'address1': '430 Gidlow Lane',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan',
'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112',
'environment-impact-current': '38', 'co2-emissions-current': '6.2',
'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN',
'mainheatc-energy-eff': 'Very Poor', 'main-fuel': 'mains gas (not community)',
'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A',
'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
'lighting-cost-potential': '67', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
'main-heating-controls': '', 'lodgement-datetime': '2022-02-23 16:39:41', 'flat-top-storey': '',
'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas',
'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843',
'current-energy-efficiency': '45', 'energy-consumption-current': '441',
'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '67',
'lodgement-date': '2022-02-23', 'extension-count': '1', 'mainheatc-env-eff': 'Very Poor',
'lmk-key': '46cb404438a6d88ddff8965cab8b3027ec15c32d93e0b6a5f0381a5109b9bb0d', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '77',
'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '100',
'walls-description': 'Cavity wall, filled cavity',
'hotwater-description': 'From main system, no cylinder thermostat'
}
ending_epc = {
'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor',
'floor-height': '2.62', 'heating-cost-potential': '803', 'unheated-corridor-length': '',
'hot-water-cost-potential': '292', 'construction-age-band': 'England and Wales: 1950-1966',
'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good',
'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '78',
'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '861',
'address3': '', 'mainheatcont-description': 'Time and temperature zone control',
'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Wigan',
'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
'hot-water-cost-current': '434', 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N',
'constituency': 'E14001039', 'co2-emissions-potential': '2.0', 'number-heated-rooms': '4',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '147',
'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-05-11',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '43', 'address1': '430 Gidlow Lane',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan',
'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112',
'environment-impact-current': '63', 'co2-emissions-current': '3.4',
'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN',
'mainheatc-energy-eff': 'Very Good', 'main-fuel': 'electricity (not community)',
'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A',
'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
'lighting-cost-potential': '67', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100',
'main-heating-controls': '', 'lodgement-datetime': '2022-06-06 13:01:20', 'flat-top-storey': '',
'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas',
'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843',
'current-energy-efficiency': '53', 'energy-consumption-current': '252',
'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '67',
'lodgement-date': '2022-06-06', 'extension-count': '1', 'mainheatc-env-eff': 'Very Good',
'lmk-key': '672d5947f3d4a55d97255af71651d6127a939418fa66a687070af77e0ba90df2', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '70',
'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '100',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
# differences = []
# for k, v in ending_epc.items():
# if v != starting_epc[k]:
# differences.append(
# {
# "variable": k,
# "starting_value": starting_epc[k],
# "ending_value": v
# }
# )
# differences = pd.DataFrame(differences)
#
# diffs = differences[
# differences["variable"].isin(
# [
# "mainheat-energy-eff",
# "mainheatcont-description",
# "mainheatc-energy-eff",
# "main-fuel",
# "mainheat-env-eff",
# "mainheat-description",
# "hot-water-energy-eff",
# "hotwater-description"
# ]
# )
# ]
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = HeatingRecommender(property_instance=home)
recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
# Patch - for this property, the hot water energy efficiency is very poor. it's not clear why this is,
# but we insert this for this test
recommender.heating_recommendations[0]["simulation_config"]["hot_water_energy_eff_ending"] = "Very Poor"
property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
assert len(recommender.heating_recommendations) == 1
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 52.2
def test_air_source_heat_pump_gas_boiler_starting_2(self):
"""
This property seems to have miniscule movement in SAP - just 2 poins
:return:
"""
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', 'uprn-source': 'Energy Assessor',
'floor-height': '2.3', 'heating-cost-potential': '394', 'unheated-corridor-length': '',
'hot-water-cost-potential': '48', 'construction-age-band': 'England and Wales: 1967-1975',
'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
'lighting-energy-eff': 'Good', 'environment-impact-potential': '87',
'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '487', 'address3': '',
'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', 'fixed-lighting-outlets-count': '5',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '86',
'county': '', 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614',
'co2-emissions-potential': '0.8', 'number-heated-rooms': '2',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '105',
'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-25',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '56', 'address1': '31 Whinney Hill Park',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Calder Valley',
'roof-energy-eff': 'Good', 'total-floor-area': '44.0', 'building-reference-number': '10001772583',
'environment-impact-current': '62', 'co2-emissions-current': '2.5',
'roof-description': 'Pitched, 250 mm loft insulation', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '2', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'BRIGHOUSE',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good',
'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2021-11-25 11:39:35', 'flat-top-storey': '', 'current-energy-rating': 'D',
'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '62',
'energy-consumption-current': '322', 'mainheat-description': 'Boiler and radiators, mains gas',
'lighting-cost-current': '56', 'lodgement-date': '2021-11-25', 'extension-count': '0',
'mainheatc-env-eff': 'Good', 'lmk-key': '077f70657e9c3f1f0ce5392798398398616b159493b2a8ca2338961596631c27',
'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '',
'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '60',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
ending_epc = {
'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park',
'uprn-source': 'Energy Assessor', 'floor-height': '2.3', 'heating-cost-potential': '277',
'unheated-corridor-length': '', 'hot-water-cost-potential': '266',
'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B',
'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good',
'environment-impact-potential': '90', 'glazed-type': 'double glazing, unknown install date',
'heating-cost-current': '331', 'address3': '',
'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A',
'property-type': 'Bungalow', 'local-authority-label': 'Calderdale',
'fixed-lighting-outlets-count': '5', 'energy-tariff': 'Single',
'mechanical-ventilation': 'natural', 'hot-water-cost-current': '404', 'county': '',
'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614',
'co2-emissions-potential': '0.7', 'number-heated-rooms': '2',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '92',
'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
'inspection-date': '2021-11-25', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '48',
'address1': '31 Whinney Hill Park', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Calder Valley', 'roof-energy-eff': 'Good', 'total-floor-area': '44.0',
'building-reference-number': '10001772583', 'environment-impact-current': '68',
'co2-emissions-current': '2.1', 'roof-description': 'Pitched, 250 mm loft insulation',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '2', 'address2': '',
'hot-water-env-eff': 'Poor', 'posttown': 'BRIGHOUSE', 'mainheatc-energy-eff': 'Average',
'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good',
'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40',
'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2022-03-23 16:06:21', 'flat-top-storey': '', 'current-energy-rating': 'D',
'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '64',
'energy-consumption-current': '283',
'mainheat-description': 'Air source heat pump, radiators, electric',
'lighting-cost-current': '57', 'lodgement-date': '2022-03-23', 'extension-count': '0',
'mainheatc-env-eff': 'Average',
'lmk-key': '6296248141447b53426a40f1c39da17dad5f4786485db55ee38737891111a4d4',
'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '',
'potential-energy-efficiency': '89', 'hot-water-energy-eff': 'Very Poor',
'low-energy-lighting': '60', 'walls-description': 'Cavity wall, filled cavity',
'hotwater-description': 'From main system'
}
# differences = []
# for k, v in ending_epc.items():
# if v != starting_epc[k]:
# differences.append(
# {
# "variable": k,
# "starting_value": starting_epc[k],
# "ending_value": v
# }
# )
# differences = pd.DataFrame(differences)
#
# diffs = differences[
# differences["variable"].isin(
# [
# "mainheat-energy-eff",
# "mainheatcont-description",
# "mainheatc-energy-eff",
# "main-fuel",
# "mainheat-env-eff",
# "mainheat-description",
# "hot-water-energy-eff",
# "hotwater-description"
# ]
# )
# ]
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = HeatingRecommender(property_instance=home)
recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
assert len(recommender.heating_recommendations) == 1
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 69.3
# In actuality with this property, the heating controls get downgraded, so we test a manual patch of this
patched_simulation_config = {
'mainheat_energy_eff_ending': "Very Good",
'hot_water_energy_eff_ending': 'Very Poor',
'has_boiler_ending': False,
'has_air_source_heat_pump_ending': True,
'has_electric_ending': True,
'has_mains_gas_ending': False,
'fuel_type_ending': 'electricity',
'trvs_ending': None,
"mainheatc_energy_eff_ending": 'Average'
}
# PATCHING
property_recommendations_patch = Recommendations.insert_temp_recommendation_id(
[recommender.heating_recommendations]
)
property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations_patch, []
)
scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict_patch = model_api.predict_all(
df=scoring_data_patch,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
# The error is only 0.3, so the model is working
assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 64.3
assert ending_epc["current-energy-efficiency"] == '64'
def test_air_source_heat_pump_lpg_boiler(self):
starting_epc = {
'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry',
'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '1628',
'unheated-corridor-length': '', 'hot-water-cost-potential': '175',
'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'D',
'mainheat-energy-eff': 'Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average',
'environment-impact-potential': '70', 'glazed-type': 'double glazing, unknown install date',
'heating-cost-current': '2158', 'address3': 'Perry',
'mainheatcont-description': 'No time or thermostatic control of room temperature',
'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire',
'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
'hot-water-cost-current': '257', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX',
'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '3.3',
'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)',
'energy-consumption-potential': '128', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached',
'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
'inspection-date': '2023-08-31', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '51',
'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0',
'building-reference-number': '10005199915', 'environment-impact-current': '50',
'co2-emissions-current': '5.9', 'roof-description': 'Pitched, 270 mm loft insulation',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive',
'hot-water-env-eff': 'Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Very Poor',
'main-fuel': 'LPG (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average',
'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good',
'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '166',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2023-10-30 13:46:54', 'flat-top-storey': '', 'current-energy-rating': 'F',
'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '32',
'energy-consumption-current': '243', 'mainheat-description': 'Boiler and radiators, LPG',
'lighting-cost-current': '277', 'lodgement-date': '2023-10-30', 'extension-count': '0',
'mainheatc-env-eff': 'Very Poor',
'lmk-key': 'f1d3bd4b8b50bc9b006231ccb158537c408523b748b3f4ef7e98cd03b144afa5', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '56',
'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '33',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
ending_epc = {
'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry',
'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '917',
'unheated-corridor-length': '', 'hot-water-cost-potential': '328',
'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'A',
'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average',
'environment-impact-potential': '96', 'glazed-type': 'double glazing, unknown install date',
'heating-cost-current': '1098', 'address3': 'Perry',
'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A',
'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire',
'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
'hot-water-cost-current': '328', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX',
'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '0.3',
'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)',
'energy-consumption-potential': '16', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached',
'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
'inspection-date': '2023-10-05', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '6',
'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0',
'building-reference-number': '10005199915', 'environment-impact-current': '92',
'co2-emissions-current': '0.7', 'roof-description': 'Pitched, 270 mm loft insulation',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive',
'hot-water-env-eff': 'Very Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Average',
'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average',
'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good',
'walls-energy-eff': 'Average', 'photo-supply': '', 'lighting-cost-potential': '166',
'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2023-11-01 16:29:16', 'flat-top-storey': '', 'current-energy-rating': 'A',
'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '92',
'energy-consumption-current': '37', 'mainheat-description': 'Air source heat pump, radiators, electric',
'lighting-cost-current': '277', 'lodgement-date': '2023-11-01', 'extension-count': '0',
'mainheatc-env-eff': 'Average',
'lmk-key': 'cb7f2838b727907767c8c2a385cd22f722b1e4745463391d910d228e52124515', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '95',
'hot-water-energy-eff': 'Good', 'low-energy-lighting': '33',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = HeatingRecommender(property_instance=home)
recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
assert len(recommender.heating_recommendations) == 1
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
# We predict a huge uplift but not quite as much as the EPC, due to some distinct differences between our
# recommendation and the EPC
assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 81.3
assert ending_epc['current-energy-efficiency'] == '92'
# PATCH
# We patch the simulation config, to reflect the ending EPC, to see if we get the ending EPC's config
patched_simulation_config = {
'mainheat_energy_eff_ending': "Very Good",
'hot_water_energy_eff_ending': 'Good',
'has_boiler_ending': False,
'has_air_source_heat_pump_ending': True,
'has_electric_ending': True,
'has_lpg_ending': False,
'fuel_type_ending': 'electricity',
'switch_system_ending': 'programmer',
'no_control_ending': None,
'auxiliary_systems_ending': 'bypass',
'trvs_ending': 'trvs',
"mainheatc_energy_eff_ending": 'Average'
}
# PATCHING
property_recommendations_patch = Recommendations.insert_temp_recommendation_id(
[recommender.heating_recommendations]
)
property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations_patch, []
)
scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict_patch = model_api.predict_all(
df=scoring_data_patch,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 88.9
# We still underpredict but the improvement is notable
def test_offgrid(self):
"""
We test on a property we've worked with before, where we compare two options
a) Upgrading to a boiler
b) Upgrading to a heat pump
:return:
"""
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '6 Beech Road', 'uprn-source': 'Energy Assessor',
'floor-height': '2.4', 'heating-cost-potential': '612', 'unheated-corridor-length': '',
'hot-water-cost-potential': '123', 'construction-age-band': 'England and Wales: 1930-1949',
'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Good',
'lighting-energy-eff': 'Good', 'environment-impact-potential': '87',
'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '2278',
'address3': '', 'mainheatcont-description': 'Appliance thermostats', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'Dudley', 'fixed-lighting-outlets-count': '9',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '604',
'county': '', 'postcode': 'DY1 4BP', 'solar-water-heating-flag': 'N', 'constituency': 'E14000671',
'co2-emissions-potential': '1.0', 'number-heated-rooms': '4',
'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '93',
'local-authority': 'E08000027', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2024-03-13',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '83', 'address1': '6 Beech Road',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Dudley North',
'roof-energy-eff': 'Very Poor', 'total-floor-area': '60.0', 'building-reference-number': '10005780080',
'environment-impact-current': '41', 'co2-emissions-current': '5.0',
'roof-description': 'Pitched, 12 mm loft insulation', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'DUDLEY',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good',
'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 67% of fixed outlets', 'roof-env-eff': 'Very Poor',
'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '113',
'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2024-03-13 11:29:11', 'flat-top-storey': '', 'current-energy-rating': 'F',
'secondheat-description': 'None', 'walls-env-eff': 'Average', 'transaction-type': 'rental',
'uprn': '90055152', 'current-energy-efficiency': '32', 'energy-consumption-current': '491',
'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '113',
'lodgement-date': '2024-03-13', 'extension-count': '1', 'mainheatc-env-eff': 'Good',
'lmk-key': '78ddf851b660e599a0894924d0e6b503980f5e0ad1aa711f8411718dc2989c44', 'wind-turbine-count': '0',
'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '87',
'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '67',
'walls-description': 'Cavity wall, filled cavity',
'hotwater-description': 'Electric immersion, standard tariff'
}
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = HeatingRecommender(property_instance=home)
recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
recommender.recommend_boiler_upgrades(phase=0, system_change=True, exising_room_heaters=False)
assert len(recommender.heating_recommendations) == 3
property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
# The ASHP isn't better under SAP, compared to a gas boiler with good heat controls
assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [66.9, 65.5, 65.9]

325
recommendations/tests/test_solar_pv_recommendations.py
View file

@ -2,6 +2,13 @@ import pytest
from recommendations.SolarPvRecommendations import SolarPvRecommendations
from backend.Property import Property
from etl.epc.Record import EPCRecord
import pandas as pd
from datetime import datetime
from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
from etl.solar.SolarPhotoSupply import SolarPhotoSupply
from recommendations.Recommendations import Recommendations
from backend.ml_models.api import ModelApi
import msgpack
class TestSolarPvRecommendations:
@ -82,3 +89,321 @@ class TestSolarPvRecommendations:
'photo_supply': 4000
}
]
def test_model(self):
"""
This function tests the recommendation engine, in conjunction with the model
:return:
"""
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor',
'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '',
'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900',
'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '85',
'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '',
'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79',
'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N',
'constituency': 'E14000707', 'co2-emissions-potential': '1.5', 'number-heated-rooms': '5',
'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '92',
'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-17',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '61', 'address1': '27 Cromwell Street',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough',
'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430',
'environment-impact-current': '47', 'co2-emissions-current': '5.4',
'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor',
'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '72',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2021-12-01 10:12:23', 'flat-top-storey': '', 'current-energy-rating': 'E',
'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor',
'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '54',
'energy-consumption-current': '346', 'mainheat-description': 'Boiler and radiators, mains gas',
'lighting-cost-current': '144', 'lodgement-date': '2021-12-01', 'extension-count': '2',
'mainheatc-env-eff': 'Good', 'lmk-key': '3ec5533af02ec78361c1f9bea8dd2e878c2c6fa6cf59e5cc505c3eeb038e0f91',
'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0',
'walls-description': 'Solid brick, as built, no insulation (assumed)',
'hotwater-description': 'From main system'
}
ending_epc = {
'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor',
'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '',
'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900',
'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '86',
'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '',
'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79',
'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N',
'constituency': 'E14000707', 'co2-emissions-potential': '1.4', 'number-heated-rooms': '5',
'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '84',
'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-12-21',
'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '49', 'address1': '27 Cromwell Street',
'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough',
'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430',
'environment-impact-current': '55', 'co2-emissions-current': '4.4',
'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor',
'walls-energy-eff': 'Very Poor', 'photo-supply': '50.0', 'lighting-cost-potential': '72',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2021-12-21 17:33:09', 'flat-top-storey': '', 'current-energy-rating': 'D',
'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor',
'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '65',
'energy-consumption-current': '277', 'mainheat-description': 'Boiler and radiators, mains gas',
'lighting-cost-current': '144', 'lodgement-date': '2021-12-21', 'extension-count': '2',
'mainheatc-env-eff': 'Good', 'lmk-key': 'b0b19583c59afbc69db12f4d6c98cd8837e80da3214d577c426eb3e672d424fc',
'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
'potential-energy-efficiency': '88', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0',
'walls-description': 'Solid brick, as built, no insulation (assumed)',
'hotwater-description': 'From main system'
}
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = SolarPvRecommendations(property_instance=home)
recommender.recommend(phase=0)
coverage_50_percent = [x for x in recommender.recommendation if x["photo_supply"] == 50]
assert len(coverage_50_percent) == 2
property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_50_percent])
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [65.9, 65.9]
assert ending_epc["current-energy-efficiency"] == '65'
def test_model2(self):
data[["uprn", "sap_ending"]]
#
searcher = SearchEpc(
address1="",
postcode="",
auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
os_api_key="",
full_address="",
uprn=100030952942,
)
searcher.find_property(False)
ending_epc = {
'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent',
'uprn-source': 'Energy Assessor', 'floor-height': '2.49', 'heating-cost-potential': '464',
'unheated-corridor-length': '', 'hot-water-cost-potential': '46',
'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B',
'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good',
'environment-impact-potential': '91', 'glazed-type': 'not defined', 'heating-cost-current': '535',
'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs',
'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow',
'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69',
'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N',
'constituency': 'E14000707', 'co2-emissions-potential': '0.7', 'number-heated-rooms': '3',
'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '56',
'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical',
'inspection-date': '2022-08-24', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '18',
'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0',
'building-reference-number': '10002845316', 'environment-impact-current': '85',
'co2-emissions-current': '1.2', 'roof-description': 'Pitched, 300 mm loft insulation',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '',
'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good',
'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Good',
'walls-energy-eff': 'Average', 'photo-supply': '40.0', 'lighting-cost-potential': '65',
'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
'lodgement-datetime': '2022-08-24 15:39:42', 'flat-top-storey': '', 'current-energy-rating': 'B',
'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
'transaction-type': 'ECO assessment', 'uprn': '100030952942', 'current-energy-efficiency': '87',
'energy-consumption-current': '100', 'mainheat-description': 'Boiler and radiators, mains gas',
'lighting-cost-current': '65', 'lodgement-date': '2022-08-24', 'extension-count': '0',
'mainheatc-env-eff': 'Good',
'lmk-key': 'e20be883431b1fed15db7fa1f52634fb7655d2b80c2fdad37df779f93ec4dafd',
'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
'potential-energy-efficiency': '91', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
starting_epc = {
'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', 'uprn-source': 'Energy Assessor',
'floor-height': '2.49', 'heating-cost-potential': '464', 'unheated-corridor-length': '',
'hot-water-cost-potential': '46', 'construction-age-band': 'England and Wales: 1967-1975',
'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '85', 'glazed-type': 'not defined',
'heating-cost-current': '535', 'address3': '',
'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
'property-type': 'Bungalow', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9',
'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69',
'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N',
'constituency': 'E14000707', 'co2-emissions-potential': '1.2', 'number-heated-rooms': '3',
'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '102',
'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0',
'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical',
'inspection-date': '2022-05-31', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40',
'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '',
'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0',
'building-reference-number': '10002845316', 'environment-impact-current': '68',
'co2-emissions-current': '2.6', 'roof-description': 'Pitched, 300 mm loft insulation',
'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', 'hot-water-env-eff': 'Good',
'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)',
'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A',
'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
'lighting-cost-potential': '65', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
'main-heating-controls': '', 'lodgement-datetime': '2022-06-15 08:38:02', 'flat-top-storey': '',
'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric',
'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100030952942',
'current-energy-efficiency': '68', 'energy-consumption-current': '227',
'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '65',
'lodgement-date': '2022-06-15', 'extension-count': '0', 'mainheatc-env-eff': 'Good',
'lmk-key': 'ce181970b7077cb9b4626242bfb010b30a0e48541b5f22427e81f1adbeeec4f2', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '85',
'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100',
'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
}
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
)
cleaned = read_from_s3(
s3_file_name="cleaned_epc_data/cleaned.bson",
bucket_name="retrofit-data-dev"
)
cleaned = msgpack.unpackb(cleaned, raw=False)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
epc = EPCRecord(
epc_records={
'original_epc': starting_epc,
'full_sap_epc': {},
'old_data': []
},
run_mode="newdata",
cleaning_data=cleaning_data
)
home = Property(
id=0,
address="",
postcode="",
epc_record=epc,
already_installed={},
non_invasive_recommendations={},
)
home.in_conservation_area = False
home.is_listed = False
home.is_heritage = False
home.restricted_measures = True
home.get_components(
cleaned=cleaned,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
recommender = SolarPvRecommendations(property_instance=home)
recommender.recommend(phase=0)
coverage_40_percent = [x for x in recommender.recommendation if x["photo_supply"] == 40]
assert len(coverage_40_percent) == 2
property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_40_percent])
home.create_base_difference_epc_record(cleaned_lookup=cleaned)
home.adjust_difference_record_with_recommendations(
property_recommendations, []
)
scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
"carbon_ending"]
)
model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
model_api.MODEL_PREFIXES = ["sap_change_predictions"]
predictions_dict = model_api.predict_all(
df=scoring_data,
bucket="retrofit-data-dev",
prediction_buckets={
"sap_change_predictions": "retrofit-sap-predictions-dev",
}
)
assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [87.1, 87.1]
assert ending_epc["current-energy-efficiency"] == '87'
assert starting_epc["current-energy-efficiency"] == '68'

6
utils/s3.py
View file

@ -198,13 +198,14 @@ def read_pickle_from_s3(bucket_name, s3_file_name):
return data
def read_excel_from_s3(bucket_name, file_key, header_row):
def read_excel_from_s3(bucket_name, file_key, header_row, drop_all_na=True):
"""
Read an Excel file from an S3 bucket and return it as a pandas DataFrame.
:param bucket_name: Name of the S3 bucket.
:param file_key: Key of the file (including directory path within the bucket).
:param header_row: The row number to use as the header (0-indexed).
:param drop_all_na: Whether to drop columns where all values are NaN.
:return: A pandas DataFrame containing the data from the Excel file.
"""
@ -219,7 +220,8 @@ def read_excel_from_s3(bucket_name, file_key, header_row):
df = pd.read_excel(excel_buffer, header=header_row)
# Drop columns where all values are NaN
df.dropna(axis=1, how='all', inplace=True)
if drop_all_na:
df.dropna(axis=1, how='all', inplace=True)
# Reset index if the first column is just an index or entirely NaN
df.reset_index(drop=True, inplace=True)