diff --git a/.idea/Model.iml b/.idea/Model.iml
index b0f9c00d..4413bb06 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
-
+
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 1122b380..6f308057 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,7 +3,7 @@
-
+
diff --git a/backend/OrdnanceSurvey.py b/backend/OrdnanceSurvey.py
index 837e76bd..a4d716d0 100644
--- a/backend/OrdnanceSurvey.py
+++ b/backend/OrdnanceSurvey.py
@@ -38,7 +38,11 @@ class OrdnanceSuveyClient:
raise ValueError("No results found - run get_places_api first")
self.address_os = self.most_relevant_result["ADDRESS"]
- self.postcode_os = self.most_relevant_result["POSTCODE"]
+
+ if "POSTCODE" in self.most_relevant_result:
+ self.postcode_os = self.most_relevant_result["POSTCODE"]
+ else:
+ self.postcode_os = self.most_relevant_result["POSTCODE_LOCATOR"]
# We strip out the postcode from the address as this is already stored separately
self.address_os = self.address_os.replace(self.postcode_os, "").strip()
# Remove trailing comma
@@ -49,7 +53,7 @@ class OrdnanceSuveyClient:
self.postcode_os = self.postcode_os.upper()
@lru_cache(maxsize=128)
- def get_places_api(self):
+ def get_places_api(self, filter_by_postcode=False):
"""
This method is tasked with getting the places api from the Ordnance Survey.
"""
@@ -58,16 +62,35 @@ class OrdnanceSuveyClient:
raise ValueError("Ordnance Survey API key not specified")
encoded_address_query = urllib.parse.quote(self.full_address)
- url = (f"https://api.os.uk/search/places/v1/find?query={encoded_address_query}&key="
- f"{self.api_key}")
+
+ url = (
+ f"https://api.os.uk/search/places/v1/find?query={encoded_address_query}&dataset=DPA,LPI&matchprecision=10"
+ f"&key={self.api_key}"
+ )
+
response = requests.get(url)
if response.status_code == 200:
data = response.json()
- results = data['results']
+ res = data["results"]
+
+ if filter_by_postcode:
+ results = []
+ for r in res:
+ if "DPA" in r:
+ if r["DPA"]["POSTCODE"] == self.postcode:
+ results.append(r)
+ elif "LPI" in r:
+ if r["LPI"]["POSTCODE_LOCATOR"] == self.postcode:
+ results.append(r)
+ else:
+ raise ValueError("Could not find postcode in either DPA or LPI")
+ else:
+ results = res
+
self.results = results
# Extract some details about the best match
- self.most_relevant_result = self.results[0]["DPA"]
+ self.most_relevant_result = self.results[0]["DPA"] if "DPA" in self.results[0] else self.results[0]["LPI"]
self.parse_classification_code(self.most_relevant_result["CLASSIFICATION_CODE"])
self.set_places_address()
@@ -94,11 +117,14 @@ class OrdnanceSuveyClient:
value_map = {
# In the OS api, "RD" is a "Dwelling" however this is not valid property type in the EPC database
'RD': {},
- 'RD02': {'property_type': 'House', 'built_form': 'Detatched'},
- 'RD03': {'property_type': 'House', 'built_form': 'Semi-Detatched'},
+ 'RD02': {'property_type': 'House', 'built_form': 'Detached'},
+ 'RD03': {'property_type': 'House', 'built_form': 'Semi-Detached'},
'RD04': {'property_type': 'House', 'built_form': 'Mid-Terrace'},
'RD06': {'property_type': 'Flat'},
}
+ # Other classifications can be found in here:
+ # https://osdatahub.os.uk/docs/places/technicalSpecification in the CLASSIFICATION_CODE description.
+ # A lookup table csv can be downloaded which contains all of the codes
mapped = value_map.get(classification_code, {})
self.property_type = mapped.get("property_type", "")
diff --git a/backend/Property.py b/backend/Property.py
index b7753413..411a4db0 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -18,6 +18,7 @@ from recommendations.recommendation_utils import (
esimtate_pitched_roof_area,
estimate_windows,
)
+from backend.ml_models.AnnualBillSavings import AnnualBillSavings
ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev")
DATA_BUCKET = os.environ.get(
@@ -93,7 +94,10 @@ class Property:
non_invasive_recommendations else []
)
# This is a list of measures that have been recommended for the property
- self.measures = ast.literal_eval(measures) if measures else None
+ if isinstance(measures, list):
+ self.measures = measures
+ else:
+ self.measures = ast.literal_eval(measures) if measures else None
self.uprn = epc_record.get("uprn")
self.full_sap_epc = epc_record.get("full_sap_epc")
@@ -159,6 +163,9 @@ class Property:
self.current_energy_bill = None
self.expected_energy_bill = None
+ self.heating_energy_source = None
+ self.hot_water_energy_source = None
+
self.recommendations_scoring_data = []
self.parse_kwargs(kwargs)
@@ -200,11 +207,11 @@ class Property:
# difference_record = self.epc_record - self.epc_record
# TODO: change these lower and replace in the settings file
- print(
- "CHANGE THE LATEST FIELD TO REMOVE NUMBER HABITABLE ROOMS IF WE WANT TO USE STARTING/ENDING"
- )
+ # print(
+ # "CHANGE THE LATEST FIELD TO REMOVE NUMBER HABITABLE ROOMS IF WE WANT TO USE STARTING/ENDING"
+ # )
fixed_data_col_names = MANDATORY_FIXED_FEATURES + LATEST_FIELD
- print("NEED TO CHANGE THE DASH TO LOWER CASE")
+ # print("NEED TO CHANGE THE DASH TO LOWER CASE")
fixed_data_col_names = [
x.lower().replace("_", "-") for x in fixed_data_col_names
]
@@ -582,6 +589,26 @@ class Property:
floor_area_decile_thresholds=floor_area_decile_thresholds,
)
self.set_energy_source()
+ self.find_energy_sources()
+ self.set_current_energy_bill()
+
+ def set_current_energy_bill(self):
+ """
+ Given what we know about the property now, estimates the current energy consumption using the UCL paper
+ https://www.sciencedirect.com/science/article/pii/S0378778823002542
+ :return:
+ """
+ starting_heat_demand = (
+ float(self.data["energy-consumption-current"]) * self.floor_area
+ )
+
+ self.current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+ epc_energy_consumption=starting_heat_demand,
+ current_epc_rating=self.data["current-energy-rating"],
+ total_floor_area=self.floor_area
+ )
+
+ self.current_energy_bill = AnnualBillSavings.calculate_annual_bill(self.current_adjusted_energy)
def set_spatial(self, spatial: pd.DataFrame):
"""
@@ -844,8 +871,8 @@ class Property:
# where a property is marked as being on the first floor
if self.floor_level > 0:
- # We check if there is another property below
- if not self.floor["another_property_below"]:
+ # We check if there is another property below (for a non-sap assessment)
+ if not self.floor["another_property_below"] and self.floor["thermal_transmittance_unit"] is None:
self.floor_level = 0
return
@@ -902,14 +929,13 @@ class Property:
return component_data
def set_adjusted_energy(
- self, current_adjusted_energy, expected_adjusted_energy, current_energy_bill, expected_energy_bill
+ self, expected_adjusted_energy, expected_energy_bill
):
"""
Stores these values for usage later
"""
- self.current_adjusted_energy = current_adjusted_energy
+
self.expected_adjusted_energy = expected_adjusted_energy
- self.current_energy_bill = current_energy_bill
self.expected_energy_bill = expected_energy_bill
def set_windows_count(self):
@@ -990,3 +1016,66 @@ class Property:
# Set the energy source based on the conditions above
self.energy_source = energy_source
+
+ def find_energy_sources(self):
+ # Based on the heating and the hot water
+ heating_fuel_mapping = {
+ 'has_mains_gas': 'Natural Gas',
+ 'has_electric': 'Electricity',
+ 'has_oil': 'Oil',
+ 'has_wood_logs': 'Wood Logs',
+ 'has_coal': 'Coal',
+ 'has_anthracite': 'Anthracite',
+ 'has_smokeless_fuel': 'Smokeless Fuel',
+ 'has_lpg': 'LPG',
+ 'has_b30k': 'B30K Biofuel',
+ 'has_air_source_heat_pump': 'Electricity',
+ 'has_ground_source_heat_pump': 'Electricity',
+ 'has_water_source_heat_pump': 'Electricity',
+ 'has_electric_heat_pump': 'Electricity',
+ 'has_solar_assisted_heat_pump': 'Electricity',
+ 'has_exhaust_source_heat_pump': 'Electricity',
+ 'has_community_heat_pump': 'Electricity',
+ 'has_wood_pellets': 'Wood Pellets',
+ 'has_community_scheme': 'Varied (Community Scheme)'
+ }
+
+ # Hot water
+ heater_type_to_fuel = {
+ 'gas instantaneous': 'Natural Gas',
+ 'electric heat pump': 'Electricity',
+ 'electric immersion': 'Electricity',
+ 'gas boiler': 'Natural Gas',
+ 'oil boiler': 'Oil',
+ 'electric instantaneous': 'Electricity',
+ 'gas multipoint': 'Natural Gas',
+ 'heat pump': 'Electricity',
+ 'solid fuel boiler': 'Solid Fuel',
+ 'solid fuel range cooker': 'Solid Fuel',
+ 'room heaters': 'Varied' # Could be any fuel, further specifics needed based on context
+ }
+
+ # Define a mapping from system types to general categories or modifications of fuel types
+ system_type_modification = {
+ 'from main system': 'Main System',
+ 'from secondary system': 'Secondary System',
+ 'from second main heating system': 'Secondary System',
+ 'community scheme': 'Community Scheme'
+ }
+
+ self.heating_energy_source = [
+ fuel for key, fuel in heating_fuel_mapping.items() if self.main_heating.get(key, False)
+ ]
+ if len(self.heating_energy_source) == 0 or len(self.heating_energy_source) > 1:
+ raise Exception("Investigate em")
+
+ self.heating_energy_source = self.heating_energy_source[0]
+
+ if self.hotwater["heater_type"] is not None:
+ self.hot_water_energy_source = heater_type_to_fuel[self.hotwater["heater_type"]]
+ else:
+ fuel = system_type_modification[self.hotwater["system_type"]]
+ if fuel == 'Main System':
+ self.hot_water_energy_source = self.heating_energy_source
+ else:
+ raise Exception("Investiage me")
diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py
index db9ec4ff..37c2b7f9 100644
--- a/backend/SearchEpc.py
+++ b/backend/SearchEpc.py
@@ -11,6 +11,7 @@ from BaseUtility import Definitions
from utils.logger import setup_logger
from typing import List
from fuzzywuzzy import process
+from backend.app.utils import sap_to_epc
logger = setup_logger()
@@ -190,15 +191,15 @@ class SearchEpc:
self.property_type = property_type
self.fast = fast
- @classmethod
- def get_house_number(cls, address: str) -> str | None:
+ @staticmethod
+ def get_house_number(address: str, postcode=None) -> str | None:
"""
This method uses the usaddress library to parse an address and extract the primary house or flat number.
"""
- try:
- # Custom regex to catch a broad range of cases
- pattern = r'(?i)(?:flat|apartment)\s*(\d+)|^\s*(\d+)'
+ try:
+ # Updated regex to catch house numbers including alphanumeric ones
+ pattern = r'(?i)(?:flat|apartment)\s*(\d+\w*)|^\s*(\d+\w*)'
match = re.search(pattern, address)
if match:
return next(g for g in match.groups() if g is not None)
@@ -207,6 +208,11 @@ class SearchEpc:
# First, try to get the 'OccupancyIdentifier' if 'OccupancyType' is detected
for part, type_ in parsed:
if type_ == 'OccupancyIdentifier':
+ if postcode is not None:
+ if part == postcode.split(" ")[0]:
+ continue
+ if part == postcode.split(" ")[1]:
+ continue
return part # This assumes the first 'OccupancyIdentifier' after 'OccupancyType' is the primary
# number
@@ -216,7 +222,7 @@ class SearchEpc:
return address_number.replace(",", "") # Remove any trailing commas
except Exception as e:
- print(f"Error parsing address: {e}")
+ raise Exception(f"Error parsing address: {e}")
return None
@@ -428,7 +434,8 @@ class SearchEpc:
self, initial_postcode: str,
lmks_to_drop: list[str] | None = None,
built_form: str = "",
- property_type: str = ""
+ property_type: str = "",
+ exclude_old: bool = False
):
"""
Fetches and processes EPC data for a given initial postcode, applying successive trimming
@@ -447,6 +454,7 @@ class SearchEpc:
:param lmks_to_drop: List of 'lmk-key' values to be excluded from the EPC data.
:param built_form: The 'built-form' value to be used for filtering the EPC data.
:param property_type: The 'property-type' value to be used for filtering the EPC data.
+ :param exclude_old: Flag to exclude EPC data older than 10 years.
:return:
"""
@@ -474,9 +482,23 @@ class SearchEpc:
if lmks_to_drop is not None:
epc_data = epc_data[~epc_data["lmk-key"].isin(lmks_to_drop)]
+ try:
+ epc_data['lodgement-datetime'] = pd.to_datetime(
+ epc_data['lodgement-datetime'], format='%Y-%m-%d %H:%M:%S', errors='coerce'
+ )
+ except Exception as e:
+ logger.error("Problem formatting lodgement-datime, appling fallback: " + str(e))
+ epc_data['lodgement-datetime'] = pd.to_datetime(epc_data['lodgement-datetime'], errors='coerce')
+
+ if exclude_old:
+ # Exclude EPC data older than 10 years
+ epc_data = epc_data[
+ epc_data["lodgement-datetime"] > (pd.Timestamp.now() - pd.DateOffset(years=10))
+ ]
+
if not epc_data.empty:
# Further processing of the EPC data
- epc_data['lodgement-datetime'] = pd.to_datetime(epc_data['lodgement-datetime'], errors='coerce')
+
epc_data = epc_data.sort_values("lodgement-datetime", ascending=False).groupby("uprn").head(1)
epc_data["house_number"] = epc_data["address"].apply(lambda add1: self.get_house_number(add1))
epc_data["numeric_house_number"] = epc_data["house_number"].apply(
@@ -554,7 +576,7 @@ class SearchEpc:
# If loop finishes without a valid response, raise an exception
raise Exception("Unable to find postcode data after trimming - investigate me")
- def estimate_epc(self, property_type, built_form, lmks_to_drop=None):
+ def estimate_epc(self, property_type, built_form, lmks_to_drop=None, exclude_old=False):
"""
For a property that does not have an EPC, we retrieve the EPC data for the closest properties
and estimate the EPC for the property in question.
@@ -567,6 +589,7 @@ class SearchEpc:
the ordnance survey api
:param lmks_to_drop: This is a list of LMK keys that should be dropped from the estimation process. This
is used as an override for testing, to drop EPCs for the property we are testing
+ :param exclude_old: Used to drop any expired EPCs (more than 10 years old)
:return:
"""
@@ -576,7 +599,8 @@ class SearchEpc:
initial_postcode=self.postcode,
lmks_to_drop=lmks_to_drop,
built_form=built_form,
- property_type=property_type
+ property_type=property_type,
+ exclude_old=exclude_old
)
# If we have missing lodgment date, we fill it with inspection-date
@@ -624,6 +648,8 @@ class SearchEpc:
else:
estimated_epc["lodgement-date"] = estimated_epc["lodgement-datetime"].strftime("%Y-%m-%d")
+ estimated_epc["current-energy-rating"] = sap_to_epc(estimated_epc["current-energy-efficiency"])
+
estimated_epc["postcode"] = self.postcode
estimated_epc["uprn"] = self.uprn
estimated_epc["address"] = self.full_address
diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py
index 205a3560..6d2ddf6c 100644
--- a/backend/apis/GoogleSolarApi.py
+++ b/backend/apis/GoogleSolarApi.py
@@ -1,336 +1,329 @@
-from backend.Property import Property
-from backend.SearchEpc import SearchEpc
-from etl.epc.Record import EPCRecord
-from dotenv import load_dotenv
-from utils.s3 import read_dataframe_from_s3_parquet
-import os
+import pandas as pd
+import numpy as np
+from recommendations.Costs import MCS_SOLAR_PV_COST_DATA
+from backend.ml_models.AnnualBillSavings import AnnualBillSavings
import requests
+from functools import lru_cache
+import time
-load_dotenv(dotenv_path="backend/.env")
-EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
-# This is for 6 Laura Close, Tintagel, PL34 0EB (same property that Cotswolrd energy used)
-uprn = 100040099104
-# This is for 353A, Hermitage Lane, ME16 9NT (one of the e.on properties)
-uprn = 200000964454
+class GoogleSolarApi:
+ NORTH_FACING_AZIMUTH_RANGE = (-30, 30)
-cleaning_data = read_dataframe_from_s3_parquet(
- bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-)
+ # Conservative estimate of the proportion of electricity that will be consumed, whereas the rest will
+ # be exported
+ SOLAR_CONSUMPTION_PROPORTION = 0.5
-searcher = SearchEpc(address1="", postcode="", uprn=uprn, auth_token=EPC_AUTH_TOKEN, os_api_key="")
+ # These are variables, described in the documentation for cost analysis for non-us locations, seen here
+ # https://developers.google.com/maps/documentation/solar/calculate-costs-non-us
+ # We use the default figures that the API uses for US locations
-searcher.find_property(skip_os=True)
+ # The factor by which the cost of electricity increases annually. The Solar API uses 1.022 (2.2% annual increase)
+ # for US locations.
+ cost_increase_factor = 1.022
-epc_records = {
- 'original_epc': searcher.newest_epc.copy(),
- 'full_sap_epc': searcher.full_sap_epc.copy(),
- 'old_data': searcher.older_epcs.copy(),
-}
+ # The efficiency at which an inverter converts the DC electricity that is produced by the solar panels to the AC
+ # electricity that is used in a household. The Solar API uses 85% for US locations. We use 0.95.5 which is the
+ # middle value of the 93-98% range, cited by Sunsave:
+ # https://www.sunsave.energy/solar-panels-advice/system-size/inverters
+ dc_to_ac_rate = 0.955
-epc = EPCRecord(
- epc_records=epc_records,
- run_mode="newdata",
- cleaning_data=cleaning_data
-)
+ # The Solar API uses 1.04 (4% annual increase) for US locations
+ discount_rate = 1.04
-uprn_filenames = read_dataframe_from_s3_parquet(
- bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
-)
+ # How much the efficiency of the solar panels declines each year. The Solar API uses 0.995 (0.5% annual decrease)
+ # for US locations
+ efficiency_depreciation_factor = 0.995
-p = Property(
- id=0,
- address=searcher.address_clean,
- postcode=searcher.postcode_clean,
- epc_record=epc,
- already_installed={},
- non_invasive_recommendations={},
-)
+ # The expected lifespan of the solar installation. The Solar API uses 20 years. Adjust this value as needed for
+ # your area
+ installation_life_span = 20
-p.get_spatial_data(uprn_filenames)
+ def __init__(self, api_key, max_retries=5):
+ """
+ Initialize the GoogleSolarApi class with the provided API key and maximum retries.
-longitude = p.spatial["longitude"]
-latitude = p.spatial["latitude"]
+ :param api_key: The API key to authenticate requests to the Google Solar API.
+ :param max_retries: The maximum number of retries for the API request (default is 5).
+ """
+ self.api_key = api_key
+ self.max_retries = max_retries
+ self.base_url = "https://solar.googleapis.com/v1"
-api_key = "AIzaSyCIz8Psu5h-1txuDX0rQpUTgkvdj8yohqU"
-url = 'https://solar.googleapis.com/v1/solarPotential'
-params = {
- 'location.latitude': f'{latitude:.5f}',
- 'location.longitude': f'{longitude:.5f}',
- 'requiredQuality': "MEDIUM",
- 'key': api_key
-}
+ self.insights_data = None
+ self.roof_segments = []
-insights_url = 'https://solar.googleapis.com/v1/buildingInsights:findClosest'
+ # property attributes:
+ self.floor_area = None
+ self.roof_area = None
+ self.roof_segment_indexes = None
+ self.panel_area = None
+ self.panel_wattage = None
+ self.panel_performance = None
-# Make the GET request to the Solar API
-insights_response = requests.get(insights_url, params=params)
-insights_data = insights_response.json()
+ def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None):
+ """
+ Make an API request to retrieve building insights based on the given longitude and latitude, with retry
+ mechanism.
-solar_potential = insights_data["solarPotential"]
+ :param longitude: The longitude of the location.
+ :param latitude: The latitude of the location.
+ :param required_quality: The required quality of the data (default is "MEDIUM").
+ :param max_retries: The maximum number of retries for the API request (default is None, which uses the
+ instance's max_retries).
+ :return: The JSON response containing the building insights data.
+ """
+ if max_retries is None:
+ max_retries = self.max_retries
-from pprint import pprint
+ insights_url = f"{self.base_url}/buildingInsights:findClosest"
+ params = {
+ 'location.latitude': f'{latitude:.5f}',
+ 'location.longitude': f'{longitude:.5f}',
+ 'requiredQuality': required_quality,
+ 'key': self.api_key
+ }
-pprint(solar_potential)
+ attempt = 0
+ while attempt < max_retries:
+ try:
+ response = requests.get(insights_url, params=params)
+ response.raise_for_status() # Raise an error for bad status codes
+ return response.json()
+ except requests.exceptions.RequestException as e:
+ attempt += 1
+ print(f"Attempt {attempt} failed: {e}")
+ time.sleep(2 ** attempt) # Exponential backoff
+ if attempt >= max_retries:
+ raise
-# This is the size of the panels used in the calculation - 400 watt
-solar_potential["panelCapacityWatts"]
-# Height of the panels used
-solar_potential["panelHeightMeters"]
-# Width of the panels used
-solar_potential["panelWidthMeters"]
+ @lru_cache(maxsize=128)
+ def get(self, longitude, latitude, required_quality="MEDIUM"):
+ """
+ Wrapper function that calls get_building_insights and extracts roof segments, with caching.
-solar_potential["wholeRoofStats"]
+ :param longitude: The longitude of the location.
+ :param latitude: The latitude of the location.
+ :param required_quality: The required quality of the data (default is "MEDIUM").
+ :return: The JSON response containing the building insights data.
+ """
-# Copy of response for testing - 6 Laura Close, Tintagel, PL34 0EB
-# {'name': 'buildings/ChIJ2yC6t4KEa0gRh2TIssogI7k', 'center': {'latitude': 50.667375, 'longitude': -4.7416833},
-# 'imageryDate': {'year': 2021, 'month': 7, 'day': 19}, 'regionCode': 'GB', 'solarPotential': {'maxArrayPanelsCount':
-# 39, 'maxArrayAreaMeters2': 76.578636, 'maxSunshineHoursPerYear': 1172.0627, 'carbonOffsetFactorKgPerMwh':
-# 478.99942, 'wholeRoofStats': {'areaMeters2': 129.65686, 'sunshineQuantiles': [537, 738.3836, 805.62445, 842.6802,
-# 909.8431, 972.15234, 1036.1013, 1092.051, 1135.8192, 1163.1444, 1193.6012], 'groundAreaMeters2': 112.33},
-# 'roofSegmentStats': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'stats': {'areaMeters2': 44.08321,
-# 'sunshineQuantiles': [614, 940.86975, 982.39124, 1057.0664, 1109.6869, 1137.5837, 1152.9211, 1163.1106, 1168.2212,
-# 1170.8883, 1193.6012], 'groundAreaMeters2': 37.61}, 'center': {'latitude': 50.6673664, 'longitude':
-# -4.741714099999999}, 'boundingBox': {'sw': {'latitude': 50.6673354, 'longitude': -4.741777}, 'ne': {'latitude':
-# 50.6674029, 'longitude': -4.7416472}}, 'planeHeightAtCenterMeters': 93.0221}, {'pitchDegrees': 34.39779,
-# 'azimuthDegrees': 31.74401, 'stats': {'areaMeters2': 44.622986, 'sunshineQuantiles': [537, 671.49774, 733.84985,
-# 780.82733, 801.4026, 814.0189, 824.0077, 847.77484, 895.08295, 950.1469, 1123.3503], 'groundAreaMeters2': 36.82},
-# 'center': {'latitude': 50.6673966, 'longitude': -4.7416813}, 'boundingBox': {'sw': {'latitude': 50.667361,
-# 'longitude': -4.7417497}, 'ne': {'latitude': 50.6674303, 'longitude': -4.741615599999999}},
-# 'planeHeightAtCenterMeters': 92.87593}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'stats': {
-# 'areaMeters2': 17.074476, 'sunshineQuantiles': [644.71136, 731.0546, 782.89813, 842.7107, 908.55585, 966.6212,
-# 1010.6367, 1038.2543, 1053.2788, 1090.6831, 1128.0178], 'groundAreaMeters2': 17.050001}, 'center': {'latitude':
-# 50.66740850000001, 'longitude': -4.7416025}, 'boundingBox': {'sw': {'latitude': 50.6673895, 'longitude':
-# -4.7416436}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415572}}, 'planeHeightAtCenterMeters':
-# 90.630356}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'stats': {'areaMeters2': 13.501617,
-# 'sunshineQuantiles': [749, 976.85345, 1059.0062, 1081.6173, 1097.4441, 1110.3171, 1128.2186, 1133.9421, 1142.068,
-# 1148.2168, 1157.632], 'groundAreaMeters2': 12.02}, 'center': {'latitude': 50.667315699999996, 'longitude':
-# -4.741675400000001}, 'boundingBox': {'sw': {'latitude': 50.667291399999996, 'longitude': -4.7417066},
-# 'ne': {'latitude': 50.6673372, 'longitude': -4.741648400000001}}, 'planeHeightAtCenterMeters': 92.36334},
-# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'stats': {'areaMeters2': 10.374564, 'sunshineQuantiles': [
-# 617.9507, 752.2504, 847.66315, 872.0505, 881.26227, 900.9639, 933.3188, 967.4747, 1000.8129, 1038.3002, 1105.545],
-# 'groundAreaMeters2': 8.83}, 'center': {'latitude': 50.6673295, 'longitude': -4.7417128}, 'boundingBox': {'sw': {
-# 'latitude': 50.6673134, 'longitude': -4.7417422}, 'ne': {'latitude': 50.6673413, 'longitude': -4.7416775}},
-# 'planeHeightAtCenterMeters': 92.31146}], 'solarPanelConfigs': [{'panelsCount': 4, 'yearlyEnergyDcKwh': 1867.1516,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 4,
-# 'yearlyEnergyDcKwh': 1867.1515, 'segmentIndex': 0}]}, {'panelsCount': 5, 'yearlyEnergyDcKwh': 2335.0068,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 5,
-# 'yearlyEnergyDcKwh': 2335.0068, 'segmentIndex': 0}]}, {'panelsCount': 6, 'yearlyEnergyDcKwh': 2799.8508,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 6,
-# 'yearlyEnergyDcKwh': 2799.8508, 'segmentIndex': 0}]}, {'panelsCount': 7, 'yearlyEnergyDcKwh': 3264.6506,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 7,
-# 'yearlyEnergyDcKwh': 3264.6506, 'segmentIndex': 0}]}, {'panelsCount': 8, 'yearlyEnergyDcKwh': 3726.2405,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 8,
-# 'yearlyEnergyDcKwh': 3726.2405, 'segmentIndex': 0}]}, {'panelsCount': 9, 'yearlyEnergyDcKwh': 4187.721,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 9,
-# 'yearlyEnergyDcKwh': 4187.721, 'segmentIndex': 0}]}, {'panelsCount': 10, 'yearlyEnergyDcKwh': 4646.094,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
-# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}]}, {'panelsCount': 11, 'yearlyEnergyDcKwh': 5103.777,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
-# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
-# 'panelsCount': 1, 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}]}, {'panelsCount': 12, 'yearlyEnergyDcKwh':
-# 5559.845, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10,
-# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 13, 'yearlyEnergyDcKwh':
-# 6013.053, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 11,
-# 'yearlyEnergyDcKwh': 5099.302, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 14, 'yearlyEnergyDcKwh':
-# 6461.664, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
-# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 15, 'yearlyEnergyDcKwh':
-# 6902.33, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
-# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162,
-# 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 16, 'yearlyEnergyDcKwh':
-# 7321.6436, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12,
-# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
-# 'panelsCount': 1, 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
-# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 17,
-# 'yearlyEnergyDcKwh': 7740.388, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331,
-# 'panelsCount': 12, 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
-# 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees':
-# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]},
-# {'panelsCount': 18, 'yearlyEnergyDcKwh': 8154.265, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 13, 'yearlyEnergyDcKwh': 5961.7896, 'segmentIndex': 0},
-# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579,
-# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh':
-# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 19, 'yearlyEnergyDcKwh': 8566.032, 'roofSegmentSummaries': [{
-# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 14, 'yearlyEnergyDcKwh': 6373.556,
-# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh':
-# 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 20, 'yearlyEnergyDcKwh': 8976.624,
-# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
-# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 21,
-# 'yearlyEnergyDcKwh': 9380.78, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331,
-# 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
-# 'azimuthDegrees': 301.1099, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1242.214, 'segmentIndex': 2}, {'pitchDegrees':
-# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]},
-# {'panelsCount': 22, 'yearlyEnergyDcKwh': 9784.078, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh': 1645.5122,
-# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh':
-# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 23, 'yearlyEnergyDcKwh': 10162.354, 'roofSegmentSummaries': [{
-# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484,
-# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh':
-# 1645.5122, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount': 24, 'yearlyEnergyDcKwh':
-# 10535.894, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099,
-# 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees':
-# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294,
-# 'azimuthDegrees': 308.42334, 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount':
-# 25, 'yearlyEnergyDcKwh': 10901.273, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees':
-# 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775,
-# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees':
-# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3},
-# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497,
-# 'segmentIndex': 4}]}, {'panelsCount': 26, 'yearlyEnergyDcKwh': 11242.756, 'roofSegmentSummaries': [{'pitchDegrees':
-# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 1, 'yearlyEnergyDcKwh': 341.4827,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 27, 'yearlyEnergyDcKwh':
-# 11579.401, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 678.1277, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
-# {'panelsCount': 28, 'yearlyEnergyDcKwh': 11919.106, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1017.83356,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 29, 'yearlyEnergyDcKwh':
-# 12255.358, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 4, 'yearlyEnergyDcKwh': 1354.0854, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
-# {'panelsCount': 30, 'yearlyEnergyDcKwh': 12586.448, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 5, 'yearlyEnergyDcKwh': 1685.1748,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 31, 'yearlyEnergyDcKwh':
-# 12911.502, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 6, 'yearlyEnergyDcKwh': 2010.2289, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
-# {'panelsCount': 32, 'yearlyEnergyDcKwh': 13233.139, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 7, 'yearlyEnergyDcKwh': 2331.8652,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 33, 'yearlyEnergyDcKwh':
-# 13554.602, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 8, 'yearlyEnergyDcKwh': 2653.3286, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
-# {'panelsCount': 34, 'yearlyEnergyDcKwh': 13893.903, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 9, 'yearlyEnergyDcKwh': 2992.6301,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 35, 'yearlyEnergyDcKwh':
-# 14221.166, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 10, 'yearlyEnergyDcKwh': 3319.893, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]},
-# {'panelsCount': 36, 'yearlyEnergyDcKwh': 14536.154, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022,
-# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 11, 'yearlyEnergyDcKwh': 3634.8809,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 37, 'yearlyEnergyDcKwh':
-# 14850.317, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 12, 'yearlyEnergyDcKwh': 3949.0444, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775,
-# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees':
-# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3},
-# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497,
-# 'segmentIndex': 4}]}, {'panelsCount': 38, 'yearlyEnergyDcKwh': 15160.658, 'roofSegmentSummaries': [{'pitchDegrees':
-# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0},
-# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 13, 'yearlyEnergyDcKwh': 4259.385,
-# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh':
-# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3,
-# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334,
-# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 39, 'yearlyEnergyDcKwh':
-# 15438.986, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15,
-# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401,
-# 'panelsCount': 14, 'yearlyEnergyDcKwh': 4537.713, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees':
-# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596,
-# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees':
-# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}],
-# 'panelCapacityWatts': 400, 'panelHeightMeters': 1.879, 'panelWidthMeters': 1.045, 'panelLifetimeYears': 20,
-# 'buildingStats': {'areaMeters2': 138.38115, 'sunshineQuantiles': [537, 728.5604, 799.23975, 833.99713, 900.88086,
-# 959.65875, 1024.2743, 1086.1285, 1132.8774, 1162.1904, 1193.6012], 'groundAreaMeters2': 117.16}, 'solarPanels': [{
-# 'center': {'latitude': 50.667371499999994, 'longitude': -4.7417235}, 'orientation': 'LANDSCAPE',
-# 'yearlyEnergyDcKwh': 468.5037, 'segmentIndex': 0}, {'center': {'latitude': 50.6673614, 'longitude': -4.7417023},
-# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.61072, 'segmentIndex': 0}, {'center': {'latitude':
-# 50.667365100000005, 'longitude': -4.7417311}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.55005,
-# 'segmentIndex': 0}, {'center': {'latitude': 50.6673512, 'longitude': -4.741681000000001}, 'orientation':
-# 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.48712, 'segmentIndex': 0}, {'center': {'latitude': 50.667357599999995,
-# 'longitude': -4.7416734}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.8553, 'segmentIndex': 0},
-# {'center': {'latitude': 50.6673779, 'longitude': -4.741715999999999}, 'orientation': 'LANDSCAPE',
-# 'yearlyEnergyDcKwh': 464.84396, 'segmentIndex': 0}, {'center': {'latitude': 50.6673678, 'longitude': -4.7416947},
-# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 464.79984, 'segmentIndex': 0}, {'center': {'latitude': 50.6673549,
-# 'longitude': -4.7417098}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 461.58975, 'segmentIndex': 0},
-# {'center': {'latitude': 50.6673816, 'longitude': -4.7417448}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
-# 461.48065, 'segmentIndex': 0}, {'center': {'latitude': 50.6673881, 'longitude': -4.7417372}, 'orientation':
-# 'LANDSCAPE', 'yearlyEnergyDcKwh': 458.3733, 'segmentIndex': 0}, {'center': {'latitude': 50.6673149, 'longitude':
-# -4.7416768}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}, {'center': {
-# 'latitude': 50.6673204, 'longitude': -4.7416867}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 456.06827,
-# 'segmentIndex': 3}, {'center': {'latitude': 50.667375199999995, 'longitude': -4.7417524}, 'orientation':
-# 'LANDSCAPE', 'yearlyEnergyDcKwh': 453.20776, 'segmentIndex': 0}, {'center': {'latitude': 50.667364, 'longitude':
-# -4.7416659}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 448.61087, 'segmentIndex': 0}, {'center': {
-# 'latitude': 50.6673094, 'longitude': -4.741666899999999}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
-# 440.66626, 'segmentIndex': 3}, {'center': {'latitude': 50.667403799999995, 'longitude': -4.741588900000001},
-# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'center': {'latitude':
-# 50.66740850000001, 'longitude': -4.7416016999999995}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 418.74448,
-# 'segmentIndex': 2}, {'center': {'latitude': 50.6673688, 'longitude': -4.7417599}, 'orientation': 'LANDSCAPE',
-# 'yearlyEnergyDcKwh': 413.877, 'segmentIndex': 0}, {'center': {'latitude': 50.667348499999996, 'longitude':
-# -4.7417174}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 411.76657, 'segmentIndex': 0}, {'center': {
-# 'latitude': 50.6673587, 'longitude': -4.7417387}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 410.5925,
-# 'segmentIndex': 0}, {'center': {'latitude': 50.6673992, 'longitude': -4.7415761}, 'orientation': 'LANDSCAPE',
-# 'yearlyEnergyDcKwh': 404.15607, 'segmentIndex': 2}, {'center': {'latitude': 50.6674132, 'longitude': -4.7416145},
-# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 403.29822, 'segmentIndex': 2}, {'center': {'latitude': 50.6673324,
-# 'longitude': -4.7417015}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}, {'center':
-# {'latitude': 50.667417799999996, 'longitude': -4.7416273}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh':
-# 373.53967, 'segmentIndex': 2}, {'center': {'latitude': 50.667324900000004, 'longitude': -4.7417104}, 'orientation':
-# 'PORTRAIT', 'yearlyEnergyDcKwh': 365.37958, 'segmentIndex': 4}, {'center': {'latitude': 50.6674043, 'longitude':
-# -4.741680800000001}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 341.4827, 'segmentIndex': 1}, {'center': {
-# 'latitude': 50.667392299999996, 'longitude': -4.7416919}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh':
-# 336.64502, 'segmentIndex': 1}, {'center': {'latitude': 50.667397, 'longitude': -4.741704599999999}, 'orientation':
-# 'PORTRAIT', 'yearlyEnergyDcKwh': 339.7059, 'segmentIndex': 1}, {'center': {'latitude': 50.6674018, 'longitude':
-# -4.7417174}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 336.25195, 'segmentIndex': 1}, {'center': {'latitude':
-# 50.6673875, 'longitude': -4.7416791}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 331.08936, 'segmentIndex':
-# 1}, {'center': {'latitude': 50.6674065, 'longitude': -4.7417301}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh':
-# 325.05405, 'segmentIndex': 1}, {'center': {'latitude': 50.6673828, 'longitude': -4.7416664}, 'orientation':
-# 'PORTRAIT', 'yearlyEnergyDcKwh': 321.63647, 'segmentIndex': 1}, {'center': {'latitude': 50.667378, 'longitude':
-# -4.741653599999999}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 321.46332, 'segmentIndex': 1}, {'center': {
-# 'latitude': 50.667373299999994, 'longitude': -4.7416409}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 339.3016,
-# 'segmentIndex': 1}, {'center': {'latitude': 50.6673853, 'longitude': -4.7416298}, 'orientation': 'PORTRAIT',
-# 'yearlyEnergyDcKwh': 327.26282, 'segmentIndex': 1}, {'center': {'latitude': 50.667399499999995, 'longitude':
-# -4.741668}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.9878, 'segmentIndex': 1}, {'center': {'latitude':
-# 50.6673948, 'longitude': -4.7416553}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.16364, 'segmentIndex':
-# 1}, {'center': {'latitude': 50.667390000000005, 'longitude': -4.7416425}, 'orientation': 'PORTRAIT',
-# 'yearlyEnergyDcKwh': 310.3404, 'segmentIndex': 1}, {'center': {'latitude': 50.6674186, 'longitude': -4.7417191},
-# 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 278.3281, 'segmentIndex': 1}]}, 'boundingBox': {'sw': {'latitude':
-# 50.6672904, 'longitude': -4.741778}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415536}},
-# 'imageryQuality': 'MEDIUM', 'imageryProcessedDate': {'year': 2024, 'month': 4, 'day': 18}}
+ self.insights_data = self.get_building_insights(longitude, latitude, required_quality)
+
+ # Extract key data from the insights response
+ self.roof_segments = self.insights_data["solarPotential"].get('roofSegmentStats', [])
+ self.floor_area = self.insights_data["solarPotential"]["wholeRoofStats"]['groundAreaMeters2']
+ self.roof_area = self.insights_data["solarPotential"]["wholeRoofStats"]['areaMeters2']
+ self.panel_area = (
+ self.insights_data["solarPotential"]["panelHeightMeters"] *
+ self.insights_data["solarPotential"]["panelWidthMeters"]
+ )
+ self.panel_wattage = self.insights_data["solarPotential"]["panelCapacityWatts"]
+ if self.panel_wattage != 400:
+ # In the API documentation, it claims that the default output is 250W, however we've only seen 400W, so if
+ # we get anything other than 400W, we'll need to adjust the calculations in the output. For this, we should
+ # refer to https://developers.google.com/maps/documentation/solar/calculate-costs-non-us
+ # Where the documentation explains how to adjust the yearlyEnergyDcKwh figures.
+ # It should be straightforward, but I'd rather see an actual instance of this happening
+ raise NotImplementedError("Panel wattage is not 400W - implement me")
+
+ # Automatically exclude north-facing segments
+ self.exclude_north_facing_segments()
+
+ self.roof_segment_indexes = [segment['segmentIndex'] for segment in self.roof_segments]
+
+ # We now start finding the solar panel configurations
+ self.optimise_solar_configuration()
+
+ @staticmethod
+ def lifetime_production_ac_kwh(
+ row,
+ efficiency_depreciation_factor,
+ installation_life_span
+ ):
+ """
+ Mimics the function described in the Google Solar API documentation, presenting the lifetime production
+ AC KWH as a geometri sum
+ """
+
+ return (
+ row["initial_ac_kwh_per_year"] *
+ (1 - pow(
+ efficiency_depreciation_factor,
+ installation_life_span)) /
+ (1 - efficiency_depreciation_factor))
+
+ @staticmethod
+ def annualUtilityBillEstimate(
+ yearlyKWhEnergyConsumption,
+ initialAcKwhPerYear,
+ efficiencyDepreciationFactor,
+ year,
+ costIncreaseFactor,
+ discountRate):
+ """
+ Implements the bill costing model for esimating annual bill
+ :param yearlyKWhEnergyConsumption:
+ :param initialAcKwhPerYear:
+ :param efficiencyDepreciationFactor:
+ :param year:
+ :param costIncreaseFactor:
+ :param discountRate:
+ :return:
+ """
+
+ return (
+ billCostModel(
+ yearlyKWhEnergyConsumption -
+ annualProduction(
+ initialAcKwhPerYear,
+ efficiencyDepreciationFactor,
+ year)) *
+ pow(costIncreaseFactor, year) /
+ pow(discountRate, year))
+
+ def lifetimeUtilityBill(
+ yearlyKWhEnergyConsumption,
+ initialAcKwhPerYear,
+ efficiencyDepreciationFactor,
+ installationLifeSpan,
+ costIncreaseFactor,
+ discountRate):
+ bill = [0] * installationLifeSpan
+ for year in range(installationLifeSpan):
+ bill[year] = annualUtilityBillEstimate(
+ yearlyKWhEnergyConsumption,
+ initialAcKwhPerYear,
+ efficiencyDepreciationFactor,
+ year,
+ costIncreaseFactor,
+ discountRate)
+ return bill
+
+ def estimate_solar_costs(self, panel_performance):
+ """
+ This method implements the recommended costing approach, to estimate the ROI of a solar panel
+ configuration, as described in the Google Solar API documentation
+ :param panel_performance: dataframe containing the solar panel array configuration and energy generation data
+ :return:
+ """
+
+ # we now estiamte the financial benefits of solar panels for the household, using the framework described
+ # by the Google Solar API
+ # 1) Convert Solar Energy AD production from the DC production
+ panel_performance["initial_ac_kwh_per_year"] = panel_performance["yearly_dc_energy"] * self.dc_to_ac_rate
+
+ # Remove anything where the total ac energy is less than half of the array wattage
+ panel_performance = panel_performance[
+ (panel_performance["initial_ac_kwh_per_year"] / panel_performance["array_warrage"]) >= 0.5
+ ]
+
+ # 2) Calculate the liftime solar energy production
+ panel_performance['lifetime_ac_kwh'] = panel_performance.apply(
+ self.lifetime_production_ac_kwh,
+ axis=1,
+ efficiency_depreciation_factor=self.efficiency_depreciation_factor,
+ installation_life_span=self.installation_life_span
+ )
+
+ # TODO: Complete the rest of the solar model
+
+ def optimise_solar_configuration(self):
+ """
+ Optimise the solar panel configuration for the building.
+ :return:
+ """
+
+ # Remove any north facing roof segments
+ panel_performance = []
+ for config in self.insights_data["solarPotential"]["solarPanelConfigs"]:
+ roof_segment_summaries = config["roofSegmentSummaries"]
+ # Filter on just the segments in self.roof_segment_indexes
+ roof_segment_summaries = [
+ segment for segment in roof_segment_summaries if segment["segmentIndex"] in self.roof_segment_indexes
+ ]
+
+ roi_summary = []
+ for segment in roof_segment_summaries:
+ wattage = segment["panelsCount"] * self.insights_data["solarPotential"]["panelCapacityWatts"]
+ generated_dc_energy = segment["yearlyEnergyDcKwh"]
+ ratio = generated_dc_energy / wattage
+ cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (generated_dc_energy / 1000)
+ roi_summary.append(
+ {
+ "segmentIndex": segment["segmentIndex"],
+ "wattage": wattage,
+ "generated_dc_energy": generated_dc_energy,
+ "ratio": ratio,
+ "n_panels": segment["panelsCount"],
+ "cost": cost,
+ "panneled_roof_area": self.panel_area * int(segment["panelsCount"])
+ }
+ )
+
+ roi_summary = pd.DataFrame(roi_summary)
+
+ weighted_ratio = np.average(
+ roi_summary["ratio"].values, weights=roi_summary["generated_dc_energy"].values
+ )
+ total_cost = roi_summary["cost"].sum()
+ yearly_dc_energy = roi_summary["generated_dc_energy"].sum()
+
+ panel_performance.append(
+ {
+ "n_panels": roi_summary["n_panels"].sum(),
+ "yearly_dc_energy": yearly_dc_energy,
+ "total_cost": total_cost,
+ "weighted_ratio": weighted_ratio,
+ "panneled_roof_area": roi_summary["panneled_roof_area"].sum(),
+ "array_warrage": roi_summary["n_panels"].sum() * self.panel_wattage
+ }
+ )
+
+ panel_performance = pd.DataFrame(panel_performance)
+ # We can have duplicate configurations
+ panel_performance = panel_performance.drop_duplicates()
+ # Ensure more than 4 panels
+ panel_performance = panel_performance[panel_performance["n_panels"] >= 4]
+
+ self.estimate_solar_costs()
+
+ # This first bracket is the value of the energy bill savings
+ panel_performance["bill_savings"] = (
+ self.SOLAR_CONSUMPTION_PROPORTION *
+ panel_performance["total_energy"] *
+ AnnualBillSavings.ELECTRICITY_PRICE_CAP
+ )
+ # This is the amount of energy exported
+ panel_performance["export_value"] = (
+ (1 - self.SOLAR_CONSUMPTION_PROPORTION) *
+ panel_performance["total_energy"] *
+ AnnualBillSavings.ELECTRICITY_EXPORT_PAYMENT
+ )
+ panel_performance["energy_value"] = panel_performance["bill_savings"] + panel_performance["export_value"]
+ panel_performance["payback_years"] = panel_performance["total_cost"] / panel_performance["energy_value"]
+
+ panel_performance = panel_performance.sort_values("weighted_ratio", ascending=False)
+ # TODO: Finish this!!
+
+ panel_performance["roof_area_percentage"] = panel_performance["panneled_roof_area"] / self.roof_area
+
+ self.panel_performance = panel_performance
+
+ def exclude_north_facing_segments(self):
+ """
+ Filter out any north-facing roof segments from the roof_segments attribute.
+
+ North-facing segments are defined as those with an azimuth between -30 and 30 degrees.
+ """
+
+ filtered_segments = []
+ for segment_index, segment in enumerate(self.roof_segments):
+ segment["segmentIndex"] = segment_index
+ # Check if the segment is north-facing
+ if self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]:
+ continue
+
+ filtered_segments.append(segment)
+
+ self.roof_segments = filtered_segments
diff --git a/backend/app/config.py b/backend/app/config.py
index 764bddf5..6f2e405b 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -14,6 +14,7 @@ class Settings(BaseSettings):
PLAN_TRIGGER_BUCKET: str
EPC_AUTH_TOKEN: str
ORDNANCE_SURVEY_API_KEY: str
+ GOOGLE_SOLAR_API_KEY: str
DB_HOST: str
DB_PASSWORD: str
DB_USERNAME: str
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 1e2c1e6f..bfe5a9e4 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -23,12 +23,13 @@ from backend.app.db.functions.recommendations_functions import (
)
from backend.app.db.models.portfolio import rating_lookup
from backend.app.dependencies import validate_token
-from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.plan.schemas import PlanTriggerRequest, MdsRequest
from backend.app.plan.utils import get_cleaned
from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc
from backend.ml_models.api import ModelApi
from backend.Property import Property
+from backend.apis.GoogleSolarApi import GoogleSolarApi
from etl.solar.SolarPhotoSupply import SolarPhotoSupply
from recommendations.optimiser.CostOptimiser import CostOptimiser
@@ -347,10 +348,14 @@ async def trigger_plan(body: PlanTriggerRequest):
bucket_name=get_settings().DATA_BUCKET, file_key="spatial/filename_meta.parquet"
)
photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket=get_settings().DATA_BUCKET)
+ solar_api_client = GoogleSolarApi(api_key=get_settings().GOOGLE_SOLAR_API_KEY)
logger.info("Getting spatial data")
for p in input_properties:
+ p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
p.get_spatial_data(uprn_filenames)
+ # Call Google Solar API
+ solar_performance = solar_api_client.get(longitude=p.spatial["longitude"], latitude=p.spatial["latitude"])
logger.info("Getting components and epc recommendations")
recommendations = {}
@@ -358,9 +363,6 @@ async def trigger_plan(body: PlanTriggerRequest):
representative_recommendations = {}
for p in tqdm(input_properties):
- # Property recommendations
- p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
-
recommender = Recommendations(property_instance=p, materials=materials, exclusions=body.exclusions)
property_recommendations, property_representative_recommendations = recommender.recommend()
@@ -422,9 +424,7 @@ async def trigger_plan(body: PlanTriggerRequest):
(
recommendations_with_impact,
- current_adjusted_energy,
expected_adjusted_energy,
- current_energy_bill,
expected_energy_bill
) = (
Recommendations.calculate_recommendation_impact(
@@ -436,9 +436,7 @@ async def trigger_plan(body: PlanTriggerRequest):
# Store the resulting adjusted energy in the property instance
property_instance.set_adjusted_energy(
- current_adjusted_energy=current_adjusted_energy,
expected_adjusted_energy=expected_adjusted_energy,
- current_energy_bill=current_energy_bill,
expected_energy_bill=expected_energy_bill
)
@@ -622,7 +620,7 @@ async def trigger_plan(body: PlanTriggerRequest):
@router.post("/mds")
-async def build_mds(body: PlanTriggerRequest):
+async def build_mds(body: MdsRequest):
# TODO: This is a placeholder location for the MDS endpoint, which this is being assembled
logger.info("Connecting to db")
@@ -633,6 +631,8 @@ async def build_mds(body: PlanTriggerRequest):
session.begin()
logger.info("Getting the inputs")
plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path)
+ measure_set = body.measures
+ optimise_measures = measure_set is not None
cleaning_data = read_dataframe_from_s3_parquet(
bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet",
@@ -659,10 +659,14 @@ async def build_mds(body: PlanTriggerRequest):
epc_searcher.find_property(skip_os=True)
if config["address"] == "35b High Street":
- print("Performing temporary patch")
+ print("Performing temporary patch on 35b High Street")
epc_searcher.newest_epc["uprn"] = 10002911892
epc_searcher.full_sap_epc["uprn"] = 10002911892
+ if config["address"] == "Cobnut Barn":
+ print("Performing temporary patch on Cobnut Barn")
+ epc_searcher.newest_epc["uprn"] = 10013924689
+
# Create a record in db
# TODO: If we productionise the creation of this mds report, we will need to store this in the db
# property_id, is_new = create_property(
@@ -706,7 +710,10 @@ async def build_mds(body: PlanTriggerRequest):
# (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
# ), {})
- measures = config["measures"] if "measures" in config else None
+ if measure_set is None:
+ measures = config["measures"] if "measures" in config else None
+ else:
+ measures = measure_set
input_properties.append(
Property(
@@ -737,24 +744,49 @@ async def build_mds(body: PlanTriggerRequest):
logger.info("Getting components and epc recommendations")
recommendations_scoring_data = []
representative_recommendations = {}
+ recommendations = {}
for p in tqdm(input_properties):
-
p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
- mds = Mds(property_instance=p, materials=materials)
- property_representative_recommendations, errors = mds.build()
+ mds = Mds(property_instance=p, materials=materials, optimise_measures=optimise_measures)
+ mds_recommendations, property_representative_recommendations, errors = mds.build()
- if errors:
- logger.info("Errors occurred during MDS build")
+ if isinstance(errors, list):
+ if errors:
+ raise Exception("Errors occurred during MDS build")
+ else:
+ if any([len(x) for x in errors.values()]):
+ raise Exception("Errors occurred during MDS build")
+ recommendations[p.id] = mds_recommendations
representative_recommendations[p.id] = property_representative_recommendations
# Build the scoring data
p.create_base_difference_epc_record(cleaned_lookup=cleaned)
- recommendations_scoring_data.append(
- p.simulate_all_representative_recommendations(property_representative_recommendations)
- )
+ if optimise_measures:
+ for _id, mds_recs in mds_recommendations.items():
+ representative_ids = [r["recommendation_id"] for r in property_representative_recommendations[_id]]
+ simulation_mds_recs = []
+ for recs in mds_recs:
+ simulation_mds_recs.append(
+ [r for r in recs if r["recommendation_id"] in representative_ids]
+ )
+
+ p.adjust_difference_record_with_recommendations(
+ simulation_mds_recs, property_representative_recommendations[_id]
+ )
+
+ data = p.recommendations_scoring_data.copy()
+ for d in data:
+ d["id"] = d["id"] + "*" + _id
+
+ recommendations_scoring_data.extend(data)
+
+ else:
+ recommendations_scoring_data.append(
+ p.simulate_all_representative_recommendations(property_representative_recommendations)
+ )
logger.info("Preparing data for scoring in sap change api")
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
@@ -787,13 +819,198 @@ async def build_mds(body: PlanTriggerRequest):
for key, scored in predictions_dict.items():
all_predictions[key] = pd.concat([all_predictions[key], scored])
- # We now produce a table of results for the mds report
+ # TODO: 1) walls_insulation_thickness_ending is not being set in the recommendations_scoring_data,
+ # insulation_thickness_ending is being set instead
+ # 2)
# TODO: TEMP
for p in plan_input:
if p["uprn"]:
p["uprn"] = str(int(float(p["uprn"])))
+ import re
+ from backend.ml_models.AnnualBillSavings import AnnualBillSavings
+
+ if optimise_measures:
+ results = []
+ for p in input_properties:
+
+ sap_before = int(p.data["current-energy-efficiency"])
+ epc_before = p.data["current-energy-rating"]
+ heat_demand_before = p.data["energy-consumption-current"]
+ carbon_before = p.data["co2-emissions-current"]
+ current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+ epc_energy_consumption=heat_demand_before * p.floor_area,
+ current_epc_rating=epc_before,
+ )
+ current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy)
+
+ package_comparison = []
+ for _id in recommendations[p.id].keys():
+
+ sap_prediction = all_predictions["sap_change_predictions"][
+ (all_predictions["sap_change_predictions"]["property_id"] == str(p.id)) &
+ (all_predictions["sap_change_predictions"]["recommendation_id"].str.contains(re.escape(_id)))
+ ].copy().reset_index(drop=True)
+ sap_prediction["row_id"] = sap_prediction.index
+
+ heat_demand_prediction = all_predictions["heat_demand_predictions"][
+ (all_predictions["heat_demand_predictions"]["property_id"] == str(p.id)) &
+ (all_predictions["heat_demand_predictions"]["recommendation_id"].str.contains(re.escape(_id)))
+ ].copy().reset_index(drop=True)
+ heat_demand_prediction["row_id"] = heat_demand_prediction.index
+
+ carbon_prediction = all_predictions["carbon_change_predictions"][
+ (all_predictions["carbon_change_predictions"]["property_id"] == str(p.id)) &
+ (all_predictions["carbon_change_predictions"]["recommendation_id"].str.contains(re.escape(_id)))
+ ].copy().reset_index(drop=True)
+ carbon_prediction["row_id"] = carbon_prediction.index
+
+ epc_target = body.goal_value
+ if epc_before == epc_target:
+ continue
+
+ sap_target = epc_to_sap_lower_bound(epc_target)
+ # Define the measures
+ sap_threshold_barrier = sap_prediction[sap_prediction["predictions"] >= sap_target]
+ meets_threshold = True
+ if sap_threshold_barrier.empty:
+ sap_threshold_barrier = sap_prediction.tail(1)
+ meets_threshold = False
+ sap_threshold_barrier = sap_threshold_barrier.head(1)
+
+ sap_prediction = sap_prediction[
+ sap_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0]
+ ]
+ heat_demand_prediction = heat_demand_prediction[
+ heat_demand_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0]
+ ]
+ carbon_prediction = carbon_prediction[
+ carbon_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0]
+ ]
+
+ reverse_map = {v: k for k, v in Mds.format_map.items()}
+
+ selected_measures = [
+ reverse_map[x.split("-")[0]] for x in sap_prediction["recommendation_id"].values
+ ]
+ selected_measure_ids = [x.split("*")[0] for x in sap_prediction["recommendation_id"].values]
+
+ costs = [
+ r["total"] for r in representative_recommendations[p.id][_id] if
+ r["recommendation_id"] in selected_measure_ids
+ ]
+ costs = sum(costs)
+
+ sap_after = sap_prediction["predictions"].values[-1]
+ epc_after = sap_to_epc(sap_after)
+ heat_demand_after = heat_demand_prediction["predictions"].values[-1]
+ carbon_after = carbon_prediction["predictions"].values[-1]
+
+ expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+ epc_energy_consumption=heat_demand_after * p.floor_area,
+ current_epc_rating=epc_before,
+ )
+
+ expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy)
+
+ bill_savings = current_energy_bill - expected_energy_bill
+ energy_savings = current_adjusted_energy - expected_adjusted_energy
+
+ package_comparison.append(
+ {
+ "id": _id,
+ "cost": costs,
+ "measures": selected_measures,
+ "sap_before": sap_before,
+ "sap_after": sap_after,
+ "epc_before": epc_before,
+ "epc_after": epc_after,
+ "heat_demand_before": heat_demand_before,
+ "heat_demand_after": heat_demand_after,
+ "carbon_before": carbon_before,
+ "carbon_after": carbon_after,
+ "bill_savings": bill_savings,
+ "energy_savings": energy_savings,
+ "current_energy_bill": current_energy_bill,
+ "meets_threshold": meets_threshold
+ }
+ )
+
+ package_comparison = pd.DataFrame(package_comparison)
+ # Find the smallest cost package
+ if not package_comparison.empty:
+
+ # We check if any of the packages meet the threshold
+ # If none of them do, take the one that gets closest to the target
+ if package_comparison["meets_threshold"].any():
+ package_comparison = package_comparison[package_comparison["meets_threshold"]]
+ package_comparison = package_comparison.sort_values("cost")
+ else:
+ package_comparison = package_comparison.sort_values("sap_after", ascending=False)
+
+ package_comparison = package_comparison.head(1).to_dict("records")[0]
+ else:
+ package_comparison = {
+ "measures": [],
+ "sap_before": sap_before,
+ "sap_after": sap_before,
+ "epc_before": epc_before,
+ "epc_after": epc_before,
+ "heat_demand_before": heat_demand_before,
+ "heat_demand_after": heat_demand_before,
+ "carbon_before": carbon_before,
+ "carbon_after": carbon_before,
+ "bill_savings": 0,
+ "energy_savings": 0,
+ "current_energy_bill": current_energy_bill,
+ "meets_threshold": False
+ }
+
+ config = [c for c in plan_input if c["uprn"] == str(p.uprn)]
+ if not config:
+ config = {"address": None, "postcode": None}
+ else:
+ config = config[0]
+
+ results.append({
+ "config_address": config["address"],
+ "config_postcode": config["postcode"],
+ "uprn": p.uprn,
+ "address": p.address,
+ "postcode": p.postcode,
+ "measures": package_comparison["measures"],
+ "year_of_epc": p.data['lodgement-date'],
+ "sap_before": package_comparison["sap_before"],
+ "sap_after": package_comparison["sap_after"],
+ "epc_before": package_comparison["epc_before"],
+ "epc_after": package_comparison["epc_after"],
+ "heat_demand_before": package_comparison["heat_demand_before"],
+ "heat_demand_after": package_comparison["heat_demand_after"],
+ "carbon_before": package_comparison["carbon_before"],
+ "carbon_after": package_comparison["carbon_after"],
+ "bill_savings": round(package_comparison["bill_savings"], 2),
+ "energy_savings": round(package_comparison["energy_savings"], 2),
+ "current_energy_bill": round(package_comparison["current_energy_bill"], 2),
+ "EWI": "EWI" if "external_wall_insulation" in package_comparison["measures"] else None,
+ "CWI": "CWI" if "cavity_wall_insulation" in package_comparison["measures"] else None,
+ "LI": "LI" if "loft_insulation" in package_comparison["measures"] else None,
+ "ASHP Htg": "ASHP Htg" if "air_source_heat_pump" in package_comparison["measures"] else None,
+ "Elec Storage": (
+ "Elec Storage Htrs (Out of scope -Prov sum only)" if "high_heat_retention_storage_heaters" in
+ package_comparison["measures"] else None
+ ),
+ "Solar PV": "Solar PV" if "solar_pv" in package_comparison["measures"] else None,
+ })
+
+ results = pd.DataFrame(results)
+
+ # For the different measures, we check the impact with a few debugging functions
+
+ walls_check, hhr_check = check_mds(results, input_properties, recommendations, optimise_measures)
+
+ results.to_excel("optimised mds_results 5th June.xlsx")
+
results = []
for p in input_properties:
measures = p.measures
@@ -842,11 +1059,14 @@ async def build_mds(body: PlanTriggerRequest):
)
# TODO: We should determine if the home is gas & electricity or just electricity
+
+ # Determine if the heating and hotwater was previously electric only or both
+
current_energy_bill = AnnualBillSavings.calculate_annual_bill(
- current_adjusted_energy,
+ kwh=current_adjusted_energy,
)
expected_energy_bill = AnnualBillSavings.calculate_annual_bill(
- expected_adjusted_energy,
+ kwh=expected_adjusted_energy,
)
bill_savings = current_energy_bill - expected_energy_bill
@@ -861,6 +1081,7 @@ async def build_mds(body: PlanTriggerRequest):
to_append = {
"config_address": config["address"],
"config_postcode": config["postcode"],
+ "uprn": p.uprn,
"address": p.address,
"postcode": p.postcode,
"measures": measures,
@@ -874,14 +1095,19 @@ async def build_mds(body: PlanTriggerRequest):
"heat_demand_after": heat_demand_after,
"carbon_before": carbon_before,
"carbon_after": carbon_after,
- "bill_savings": bill_savings,
- "energy_savings": energy_savings,
+ "bill_savings": round(bill_savings, 2),
+ "energy_savings": round(energy_savings, 2),
+ "current_energy_bill": round(current_energy_bill, 2),
+ "fuel_type": p.main_fuel["fuel_type"],
}
results.append(to_append)
results = pd.DataFrame(results)
results["sap_uplift"] = results["sap_after"] - results["sap_before"]
+ # results.to_excel("mds_results 5th June.xlsx")
+
+ walls_check, hhr_check = check_mds(results, input_properties, recommendations, optimise_measures)
except IntegrityError:
logger.error("Database integrity error occurred", exc_info=True)
@@ -901,3 +1127,80 @@ async def build_mds(body: PlanTriggerRequest):
return Response(status_code=500, content="An unexpected error occurred.")
finally:
session.close()
+
+
+def check_mds(results, input_properties, recommendations, optimise_measures):
+ import ast
+ walls_check = []
+ hhr_check = []
+ for p in input_properties:
+ res = results[results["uprn"] == p.uprn]
+ wall = p.walls
+ heating = p.main_heating
+ heating_controls = p.main_heating_controls
+
+ if optimise_measures:
+ measures = res["measures"].values[0]
+ else:
+ measures = [list(z.keys())[0] for z in res["measures"].values[0]]
+
+ wall_recommendation = [
+ x for x in measures if
+ x in ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"]
+ ]
+
+ hhr_recommendation = [
+ x for x in measures if
+ x in ["high_heat_retention_storage_heaters"]
+ ]
+
+ if optimise_measures:
+ possible_measures = [ast.literal_eval(x) for x in list(recommendations[p.id].keys())]
+ # Unlist them
+ possible_measures = [x for sublist in possible_measures for x in sublist]
+ possible_measures = list(set(possible_measures))
+ else:
+ possible_measures = p.measures
+
+ if wall_recommendation:
+ if len(wall_recommendation) > 1:
+ raise Exception("something went wrong")
+ wall_recommendation = wall_recommendation[0]
+ else:
+ wall_recommendation = None
+
+ hhr_recommendation = hhr_recommendation[0] if hhr_recommendation else None
+
+ walls_check.append(
+ {
+ "uprn": p.uprn,
+ "address": p.address,
+ "postcode": p.postcode,
+ "property_type": p.data['property-type'],
+ "conservation_status": p.spatial["conservation_status"],
+ "is_listed_building": p.spatial["is_listed_building"],
+ "is_heritage_building": p.spatial["is_heritage_building"],
+ "wall": wall["clean_description"],
+ "recommendation": wall_recommendation,
+ "possible_measures": possible_measures,
+ "selected_measures": res["measures"].values[0],
+ }
+ )
+
+ hhr_check.append(
+ {
+ "uprn": p.uprn,
+ "address": p.address,
+ "postcode": p.postcode,
+ "heating": heating["clean_description"],
+ "heating_controls": heating_controls["clean_description"],
+ "recommendation": hhr_recommendation,
+ "possible_measures": possible_measures,
+ "selected_measures": res["measures"].values[0],
+ }
+ )
+
+ walls_check = pd.DataFrame(walls_check)
+ hhr_check = pd.DataFrame(hhr_check)
+
+ return walls_check, hhr_check
diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py
index 59c0ebef..fbc4d4f2 100644
--- a/backend/app/plan/schemas.py
+++ b/backend/app/plan/schemas.py
@@ -52,3 +52,9 @@ class PlanTriggerRequest(BaseModel):
if v not in cls._allowed_housing_types:
raise ValueError(f"{v} is not a valid housing type")
return v
+
+
+class MdsRequest(PlanTriggerRequest):
+ # When creating the mds report, we allow an optional list of measures to select from. If this is passed, it will
+ # cause the service to select the optimal package from the list of measures
+ measures: Optional[conlist(str, min_items=1)] = None
diff --git a/backend/ml_models/AnnualBillSavings.py b/backend/ml_models/AnnualBillSavings.py
index b92077e4..7395ab6b 100644
--- a/backend/ml_models/AnnualBillSavings.py
+++ b/backend/ml_models/AnnualBillSavings.py
@@ -1,3 +1,6 @@
+import numpy as np
+
+
class AnnualBillSavings:
"""
This is a simple class which will estimate the annual bill savings, based on the kwh savings.
@@ -14,6 +17,8 @@ class AnnualBillSavings:
# https://www.ofgem.gov.uk/publications/new-energy-price-cap-level-april-june-2024-starts-today
ELECTRICITY_PRICE_CAP = 0.245
GAS_PRICE_CAP = 0.0604
+ # This is the most recent export payment figure, at 12p per kwh
+ ELECTRICITY_EXPORT_PAYMENT = 0.12
# This is a weighted mean of the price caps, using the consumption figures above as weights
PRICE_FACTOR = 0.09549999999999999
@@ -58,8 +63,58 @@ class AnnualBillSavings:
return cls.ELECTRICITY_PRICE_CAP * kwh + (cls.DAILY_STANDARD_CHARGE_ELECTRICITY * 365)
+ @staticmethod
+ def calculate_occupants(total_floor_area):
+ """
+ From Table 1b of the SAP 2012 documentation https://bregroup.com/documents/d/bre-group/sap-2012_9-92
+ Provides a methodology to estimate occupancy, based on floor area. This is used to calculate the amount of
+ electricity used be appliances and during cooking.
+ :param total_floor_area:
+ :return:
+ """
+
+ if total_floor_area <= 13.9:
+ return 1
+
+ return 1 + (1.76 * (1 - np.exp(-0.000349 * (total_floor_area - 13.9) * (total_floor_area - 13.9))) + 0.0013 * (
+ total_floor_area - 13.9))
+
+ @staticmethod
+ def estimate_electrical_appliances(occupants, total_floor_area):
+ """
+ From secion L2 of SAP2012 Electrical appliances
+ https://bregroup.com/documents/d/bre-group/sap-2012_9-92
+ Used to estimate the amount of energy used by electrical appliances
+ :param occupants:
+ :param total_floor_area:
+ :return:
+ """
+ e_a = 207.8 * np.power(total_floor_area * occupants, 0.4717)
+
+ days_in_month = {
+ 1: 31,
+ 2: 28,
+ 3: 31,
+ 4: 30,
+ 5: 31,
+ 6: 30,
+ 7: 31,
+ 8: 31,
+ 9: 30,
+ 10: 31,
+ 11: 30,
+ 12: 31
+ }
+
+ eam = 0
+ for m in range(1, 13):
+ nm = days_in_month[m]
+ eam += e_a * (1 + 0.157 * np.cos(2 * np.pi * (m - 1.78) / 12)) * nm / 365
+
+ return eam
+
@classmethod
- def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating):
+ def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating, total_floor_area):
"""
The over-prediction of energy use by EPCs in Great Britain: A comparison
of EPC-modelled and metered primary energy use intensity
@@ -70,6 +125,13 @@ class AnnualBillSavings:
:return:
"""
+ # The EPC energy consumption does not factor in cooking and applicance use, so this is estimated using the
+ # methodology outlined in SAP, and is discussed in the UCL paper in section 3.1.1
+ estimated_occupants = cls.calculate_occupants(total_floor_area=total_floor_area)
+ appliances_energy_use = cls.estimate_electrical_appliances(estimated_occupants, total_floor_area)
+
+ epc_energy_consumption += appliances_energy_use
+
gradients = {
"A": -0.1,
"B": -0.1,
diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py
index dd77fb4b..1af38194 100644
--- a/backend/ml_models/Valuation.py
+++ b/backend/ml_models/Valuation.py
@@ -90,6 +90,9 @@ class PropertyValuation:
41222760: 46_000, # Based on Zoopla
41222761: 270_000, # Based on Zoopla
41212534: 38_000, # Based on Zoopla
+ # Northern Group Pilot - search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
+ 10070868263: 194_000, # Based on Zoopla
+ 10070868244: 195_000, # Based on Zoopla
}
# We base our valuation uplifts on a number of sources
diff --git a/etl/customers/eon/deck_examples.py b/etl/customers/eon/deck_examples.py
new file mode 100644
index 00000000..e0e3abe9
--- /dev/null
+++ b/etl/customers/eon/deck_examples.py
@@ -0,0 +1,27 @@
+"""
+This script contains bits of codes for examples to be included in the Deck
+"""
+
+from backend.SearchEpc import SearchEpc
+from dotenv import load_dotenv
+import os
+
+load_dotenv(dotenv_path="backend/.env")
+
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+searcher = SearchEpc(
+ address1="108 Blacklands",
+ postcode="ME19 6DP",
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=False,
+)
+
+res = searcher.estimate_epc(
+ property_type="Bungalow",
+ built_form="Detached",
+ lmks_to_drop=["849273656952012102323315196229804"],
+ exclude_old=True
+)
diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py
index 4f79e05e..05e459cb 100644
--- a/etl/customers/eon/pilot_asset_list.py
+++ b/etl/customers/eon/pilot_asset_list.py
@@ -64,7 +64,7 @@ def extract_mds_measures(config):
measures.append({"district_heating_networks": "District heating networks"})
if not pd.isnull(config["Elec Storage Htrs (Out of scope -Prov sum only)"]):
- measures.append({"electric_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)"})
+ measures.append({"high_heat_retention_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)"})
if not pd.isnull(config["Low Energy Bulbs"]):
measures.append({"low_energy_lighting": "Low Energy Bulbs"})
@@ -229,7 +229,8 @@ def app():
"35a High Street",
"35b High Street",
"Flat Over 20 Holborough Road",
- "Flat above 7 Malling Road"
+ "Flat above 7 Malling Road",
+ "Cobnut Barn",
]:
print(config["Address"])
uprn = None
@@ -269,3 +270,33 @@ def app():
"budget": None,
}
print(body)
+
+ # Optimised version where we specify the measures
+ measures = [
+ "external_wall_insulation",
+ "cavity_wall_insulation",
+ "loft_insulation",
+ "air_source_heat_pump",
+ "high_heat_retention_storage_heaters",
+ "solar_pv"
+ ]
+
+ body = {
+ "portfolio_id": str(PORTFOLIO_ID),
+ "housing_type": "Social",
+ "goal": "Increase EPC",
+ "goal_value": "C",
+ "trigger_file_path": filename,
+ "already_installed_file_path": "",
+ "patches_file_path": "",
+ "non_invasive_recommendations_file_path": "",
+ "measures": measures,
+ "budget": None,
+ }
+
+
+output = []
+for r in self.results:
+ output.append(r["DPA"])
+
+output = pd.DataFrame(output)
diff --git a/etl/customers/goldman/property_ownership.py b/etl/customers/goldman/property_ownership.py
index d30205ae..500963a1 100644
--- a/etl/customers/goldman/property_ownership.py
+++ b/etl/customers/goldman/property_ownership.py
@@ -3,6 +3,7 @@ import pandas as pd
from tqdm import tqdm
import Levenshtein
from backend.SearchEpc import SearchEpc
+from utils.s3 import read_dataframe_from_s3_parquet
# Average value of a property in the midlands in 2024 was £238,000. Since these are EPC F & G properties, we assume
# £207,000 since they trade at a discount. This is based on the rightmove study where moving from an EPC F/G -> C has a
@@ -248,6 +249,13 @@ def app():
"""
This script is for scoping property ownership for EPC F & G rated properties in Birmingam, for Goldman Sachs
"""
+
+ # TODO: This property:
+ # https://epc.opendatacommunities.org/domestic/search?address=&postcode=&local-authority=&constituency
+ # =&uprn=100031179243&from-month=1&from-year=2008&to-month=12&to-year=2024
+ # is actually listed in two local authorities causing us to think it's an EPC F & G property, but it's
+ # it's actually EPC E. Need to handle this, probably by reading in all of the EPC data, concatenating together
+ # and performing a singular filter for most recent EPC by UPRN
# paths = [
# "local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv",
# "local_data/all-domestic-certificates/domestic-E08000031-Wolverhampton/certificates.csv",
@@ -477,6 +485,35 @@ def app():
portfolio_epc_data_50m.to_excel("portfolio_epc_data_50m 28th May.xlsx", index=False)
portfolio_epc_data_20m.to_excel("portfolio_epc_data_20m 28th May.xlsx", index=False)
+ # We check if any of these properties are in a conservation area
+ valuations = pd.read_excel("property value.xlsx")
+
+ uprn_filenames = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
+ )
+
+ geospatial_data = []
+ for _, row in tqdm(valuations.iterrows(), total=len(valuations)):
+ filtered_df = uprn_filenames[
+ (uprn_filenames["lower"] <= row["UPRN"])
+ & (uprn_filenames["upper"] >= row["UPRN"])
+ ]
+ if filtered_df.empty:
+ raise Exception("No match found")
+
+ filename = filtered_df.iloc[0]["filenames"]
+
+ spatial_data = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key=f"spatial/{filename}"
+ )
+ spatial = spatial_data[
+ spatial_data["UPRN"] == row["UPRN"]
+ ][["UPRN", "conservation_status", "is_listed_building", "is_heritage_building"]]
+ geospatial_data.append(spatial.to_dict("records")[0])
+
+ geospatial_data = pd.DataFrame(geospatial_data)
+ geospatial_data.to_excel("geospatial_data.xlsx", index=False)
+
def company_aggregation():
company_ownership = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_04.csv")
@@ -490,3 +527,79 @@ def company_aggregation():
aggregation = aggregation.sort_values("Number of Properties", ascending=False)
aggregation.to_excel("Company ownership aggregation.xlsx")
+
+
+def prepare_anonymised_data():
+ investment_50m_properties = pd.read_excel("investment_50m_properties 28th May.xlsx", header=0)
+ investment_epc_data = pd.read_excel("portfolio_epc_data_50m 28th May.xlsx", header=0)
+ valuations = pd.read_excel("property value.xlsx", header=0)
+
+ # Merge these datasets
+ df = investment_50m_properties.merge(
+ investment_epc_data[
+ ["UPRN", "PROPERTY_TYPE", "BUILT_FORM", "TOTAL_FLOOR_AREA", "LODGEMENT_DATE", "POSTCODE"]
+ ].rename(
+ columns={
+ "PROPERTY_TYPE": "Property Type",
+ "BUILT_FORM": "Property Archetype",
+ "TOTAL_FLOOR_AREA": "Total Floor Area",
+ "LODGEMENT_DATE": "Date EPC Lodged",
+ "POSTCODE": "Postcode on EPC"
+ }
+ ),
+ how="inner",
+ on="UPRN"
+ ).merge(
+ valuations.drop(columns=["ADDRESS", "POSTCODE"]).rename(
+ columns={
+ "Zoopla Valuation": "Expected Valuation",
+ "Zoopla Lower Bound": "Valuation - Lower Bound",
+ "Zoopla Upper Bound": "Valuation - Upper Bound",
+ }
+ ),
+ how="inner",
+ on="UPRN"
+ ).rename(
+ columns={
+ "CURRENT_ENERGY_RATING": "Current EPC",
+ "CURRENT_ENERGY_EFFICIENCY": "Current SAP Score",
+ "epc_address": "Address on EPC"
+ }
+ ).drop(
+ columns=["Title Number", "match_type", "UPRN"]
+ )
+
+ redacted_owner_names = df[["Company Registration No. (1)"]].drop_duplicates()
+ redacted_owner_names["Owner"] = ["Owner" + str(i) for i in range(1, len(redacted_owner_names) + 1)]
+
+ df = df.merge(
+ redacted_owner_names, how="left", on="Company Registration No. (1)"
+ )
+
+ df = df.drop(columns=["Company Registration No. (1)", "Proprietor Name (1)", "Property Address"])
+ df = df.sort_values(["Owner", "Date EPC Lodged"], ascending=False)
+
+ redacted_index = []
+ for _, owner_properties in df.groupby("Owner"):
+ top_50_percent = round(owner_properties.shape[0] / 2 + 0.00001)
+ indexes = owner_properties.tail(
+ owner_properties.shape[0] - top_50_percent
+ ).index
+
+ redacted_index.extend(indexes.tolist())
+
+ import numpy as np
+ # Redact addresses and postcodes
+ df["Address on EPC"] = np.where(
+ df.index.isin(redacted_index),
+ "Redacted",
+ df["Address on EPC"]
+ )
+
+ df["Postcode on EPC"] = np.where(
+ df.index.isin(redacted_index),
+ "Redacted",
+ df["Postcode on EPC"]
+ )
+
+ df.to_excel("Property List - 50% redacted.xlsx", index=False)
diff --git a/etl/customers/lhp/30_may_2024_data_pull.py b/etl/customers/lhp/30_may_2024_data_pull.py
new file mode 100644
index 00000000..4bf15caa
--- /dev/null
+++ b/etl/customers/lhp/30_may_2024_data_pull.py
@@ -0,0 +1,148 @@
+import os
+
+import pandas as pd
+from tqdm import tqdm
+
+from dotenv import load_dotenv
+from utils.s3 import read_excel_from_s3
+from backend.SearchEpc import SearchEpc
+from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
+
+from recommendations.recommendation_utils import (
+ estimate_perimeter,
+ estimate_external_wall_area,
+ estimate_number_of_floors
+)
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+ """
+ This app is EPC pulling data for some properties owned by LHP
+ :return:
+ """
+ # asset_list = read_excel_from_s3(
+ # bucket_name="retrofit-datalake-dev",
+ # file_key="customers/guiness/TGP CW Properties PV.xlsx",
+ # header_row=0
+ # )
+ asset_list = pd.read_excel("/Users/khalimconn-kowlessar/Downloads/Echo4 3.4.24.xlsx", header=0)
+
+ epc_data = []
+ for _, home in tqdm(asset_list.iterrows(), total=len(asset_list)):
+
+ full_address = home["ADDRESS"]
+ address_split = full_address.split(",")
+ address1 = address_split[0].strip()
+ postcode = address_split[-1].strip()
+
+ searcher = SearchEpc(
+ address1=address1,
+ postcode=postcode,
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=True,
+ full_address=full_address
+ )
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+ if searcher.newest_epc is None:
+ continue
+
+ epc = {
+ "asset_list_address": full_address,
+ **searcher.newest_epc.copy()
+ }
+
+ epc_data.append(epc)
+
+ epc_df = pd.DataFrame(epc_data)
+
+ # Retrieve just the data we need
+ epc_df = epc_df[
+ [
+ "asset_list_address",
+ "uprn",
+ "property-type",
+ "built-form",
+ "inspection-date",
+ "current-energy-rating",
+ "current-energy-efficiency",
+ "roof-description",
+ "walls-description",
+ "transaction-type",
+ # New fields needed
+ "secondheat-description",
+ "total-floor-area",
+ "construction-age-band",
+ "floor-height",
+ "number-habitable-rooms",
+ "mainheat-description"
+ ]
+ ]
+
+ asset_list = asset_list.merge(
+ epc_df,
+ how="left",
+ left_on=["ADDRESS"],
+ right_on=["asset_list_address"]
+ )
+
+ asset_list = asset_list.drop(columns=["asset_list_address"])
+
+ # Rename the columns
+ asset_list = asset_list.rename(columns={
+ "inspection-date": "Date of last EPC",
+ "current-energy-efficiency": "SAP score on register",
+ "current-energy-rating": "EPC rating on register",
+ "property-type": "Property Type",
+ "built-form": "Archetype",
+ "total-floor-area": "Property Floor Area",
+ "construction-age-band": "Property Age Band",
+ "floor-height": "Property Floor Height",
+ "number-habitable-rooms": "Number of Habitable Rooms",
+ "walls-description": "Wall Construction",
+ "roof-description": "Roof Construction",
+ "mainheat-description": "Heating Type",
+ "secondheat-description": "Secondary Heating",
+ "transaction-type": "Reason for last EPC"
+ })
+
+ asset_list["Estimated Number of Floors"] = asset_list.apply(
+ lambda x: estimate_number_of_floors(property_type=x["Property Type"]), axis=1
+ )
+
+ asset_list["Property Floor Area"] = asset_list["Property Floor Area"].astype(float)
+ asset_list["Number of Habitable Rooms"] = asset_list["Number of Habitable Rooms"].astype(float)
+
+ asset_list["Estimated Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_perimeter(
+ floor_area=x["Property Floor Area"] / x["Estimated Number of Floors"],
+ num_rooms=x["Number of Habitable Rooms"] / x["Estimated Number of Floors"],
+ ), axis=1
+ )
+
+ asset_list["Estimated Heat Loss Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_external_wall_area(
+ num_floors=x["Estimated Number of Floors"],
+ floor_height=float(x["Property Floor Height"]) if x["Property Floor Height"] else 2.5,
+ perimeter=x["Estimated Perimeter (m)"],
+ built_form=x["Archetype"]
+ ),
+ axis=1
+ )
+
+ asset_list["Roof Insulation Thickness"] = asset_list.apply(
+ lambda x: RoofAttributes(description=x["Roof Construction"]).process()["insulation_thickness"],
+ axis=1
+ )
+
+ # Store as an excel
+ filename = "LHP EPC Data pull.xlsx"
+ asset_list.to_excel(filename, index=False)
diff --git a/etl/customers/northern_gorup/test_asset_list.py b/etl/customers/northern_gorup/test_asset_list.py
new file mode 100644
index 00000000..46a4bb75
--- /dev/null
+++ b/etl/customers/northern_gorup/test_asset_list.py
@@ -0,0 +1,43 @@
+import pandas as pd
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 81
+
+
+def app():
+ asset_list = [
+ {
+ 'uprn': 10070868263,
+ "address": "Apartment 307, Flint Glass Wharf",
+ "postcode": "M4 6AD",
+ },
+ {
+ 'uprn': 10070868244,
+ "address": "Apartment 106, Flint Glass Wharf",
+ "postcode": "M4 6AD",
+ }
+ ]
+
+ asset_list = pd.DataFrame(asset_list)
+
+ # Store the asset list in s3
+ filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
+ save_csv_to_s3(
+ dataframe=asset_list,
+ bucket_name="retrofit-plan-inputs-dev",
+ file_name=filename
+ )
+
+ body = {
+ "portfolio_id": str(PORTFOLIO_ID),
+ "housing_type": "Private",
+ "goal": "Increase EPC",
+ "goal_value": "B",
+ "trigger_file_path": filename,
+ "already_installed_file_path": "",
+ "patches_file_path": "",
+ "non_invasive_recommendations_file_path": "",
+ "budget": None,
+ }
+ print(body)
diff --git a/etl/customers/places_for_people/EPC data pull - 12th June.py b/etl/customers/places_for_people/EPC data pull - 12th June.py
new file mode 100644
index 00000000..45a70ad4
--- /dev/null
+++ b/etl/customers/places_for_people/EPC data pull - 12th June.py
@@ -0,0 +1,156 @@
+import os
+
+import pandas as pd
+from tqdm import tqdm
+import numpy as np
+
+from dotenv import load_dotenv
+from backend.SearchEpc import SearchEpc
+from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
+
+from recommendations.recommendation_utils import (
+ estimate_perimeter,
+ estimate_external_wall_area,
+ estimate_number_of_floors
+)
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+ """
+ This app is EPC pulling data for some properties owned by LHP
+ :return:
+ """
+
+ asset_list = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/Places for People NORTH WEST - EPC DATA PULL REQUEST.xlsx", header=0
+ )
+
+ epc_data = []
+ for _, home in tqdm(asset_list.iterrows(), total=len(asset_list)):
+
+ full_address = home["Address"]
+
+ address1 = home["AddressLine1"]
+ postcode = home["Postcode"]
+
+ searcher = SearchEpc(
+ address1=address1,
+ postcode=postcode,
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=True,
+ full_address=full_address
+ )
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+ if searcher.newest_epc is None:
+ continue
+
+ epc = {
+ "asset_list_address": full_address,
+ **searcher.newest_epc.copy()
+ }
+
+ epc_data.append(epc)
+
+ epc_df = pd.DataFrame(epc_data)
+
+ # Retrieve just the data we need
+ epc_df = epc_df[
+ [
+ "asset_list_address",
+ "uprn",
+ "property-type",
+ "built-form",
+ "inspection-date",
+ "current-energy-rating",
+ "current-energy-efficiency",
+ "roof-description",
+ "walls-description",
+ "transaction-type",
+ # New fields needed
+ "secondheat-description",
+ "total-floor-area",
+ "construction-age-band",
+ "floor-height",
+ "number-habitable-rooms",
+ "mainheat-description"
+ ]
+ ]
+
+ # epc_df.to_csv("pfp sales data.csv", index=False)
+
+ asset_list = asset_list.merge(
+ epc_df,
+ how="left",
+ left_on=["Address"],
+ right_on=["asset_list_address"]
+ )
+
+ asset_list = asset_list.drop(columns=["asset_list_address"])
+
+ # Rename the columns
+ asset_list = asset_list.rename(columns={
+ "inspection-date": "Date of last EPC",
+ "current-energy-efficiency": "SAP score on register",
+ "current-energy-rating": "EPC rating on register",
+ "property-type": "EPC Property Type",
+ "built-form": "EPC Archetype",
+ "total-floor-area": "EPC Property Floor Area",
+ "construction-age-band": "EPC Property Age Band",
+ "floor-height": "EPC Property Floor Height",
+ "number-habitable-rooms": "EPC Number of Habitable Rooms",
+ "walls-description": "EPC Wall Construction",
+ "roof-description": "EPC Roof Construction",
+ "mainheat-description": "EPC Heating Type",
+ "secondheat-description": "EPC Secondary Heating",
+ "transaction-type": "Reason for last EPC"
+ })
+
+ asset_list["Estimated Number of Floors"] = asset_list.apply(
+ lambda x: estimate_number_of_floors(
+ property_type=x["EPC Property Type"]
+ ) if not pd.isnull(x["EPC Property Type"]) else None, axis=1
+ )
+
+ asset_list["EPC Property Floor Area"] = asset_list["EPC Property Floor Area"].astype(float)
+ asset_list["EPC Number of Habitable Rooms"] = np.where(
+ asset_list["EPC Number of Habitable Rooms"] == "",
+ None,
+ asset_list["EPC Number of Habitable Rooms"]
+ )
+ asset_list["EPC Number of Habitable Rooms"] = asset_list["EPC Number of Habitable Rooms"].astype(float)
+
+ asset_list["Estimated Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_perimeter(
+ floor_area=x["EPC Property Floor Area"] / x["Estimated Number of Floors"],
+ num_rooms=x["EPC Number of Habitable Rooms"] / x["Estimated Number of Floors"],
+ ), axis=1
+ )
+
+ asset_list["Estimated Heat Loss Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_external_wall_area(
+ num_floors=x["Estimated Number of Floors"],
+ floor_height=float(x["EPC Property Floor Height"]) if x["EPC Property Floor Height"] else 2.5,
+ perimeter=x["Estimated Perimeter (m)"],
+ built_form=x["EPC Archetype"]
+ ),
+ axis=1
+ )
+
+ asset_list["Roof Insulation Thickness"] = asset_list.apply(
+ lambda x: RoofAttributes(description=x["EPC Roof Construction"]).process()[
+ "insulation_thickness"] if not pd.isnull(x["EPC Roof Construction"]) else None,
+ axis=1
+ )
+
+ # Store as an excel
+ filename = "Places for People NORTH WEST - EPC DATA PULL.xlsx"
+ asset_list.to_excel(filename, index=False)
diff --git a/etl/customers/places_for_people/parity_comparison.py b/etl/customers/places_for_people/parity_comparison.py
new file mode 100644
index 00000000..64ab8591
--- /dev/null
+++ b/etl/customers/places_for_people/parity_comparison.py
@@ -0,0 +1,164 @@
+"""
+This script is used to pull together some case studies for the Parity Projects comparison
+"""
+
+import pandas as pd
+from backend.SearchEpc import SearchEpc
+from dotenv import load_dotenv
+import os
+
+load_dotenv("backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+parity_measures = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Places For People/Parity Sample All Addresses and Measures.xlsx",
+ sheet_name="Total Measures"
+)
+
+solar_measures = parity_measures[parity_measures["Category"] == "SolarPV"]
+
+example_1 = parity_measures[
+ parity_measures["Address Id (used by website)"] == 6125299
+ ].copy()
+
+config = {
+ "address": "14 Victoria Road",
+ "postcode": "BD20 8SY",
+ "uprn": 100050346517
+}
+
+# Point 1:
+# Parity tends to re-score the EPCs, even if they're extrememly recent.
+# For example for '14, Victoria Road, Cross Hills, KEIGHLEY, North Yorkshire, ENGLAND, BD20 8SY'
+# The most recent EPC was done 15 May 2023, and landed at a 66D, however for some reason, parity re-score this
+# home to be a 63.91. It's unclear why this is done
+
+example_1_measures = example_1[["MeasureGroupName", "Individual SAP increase"]].copy()
+# - LEDS: 0.25 SAP points
+# - 300mm of loft insulation from 200mm: 0.43 SAP points - where is this deduced from? Since the latest survey
+# indicates 250mm insulation in place
+# - Check construction of unknown party wall and fill cavity if appropriate: 0.12 SAP points (highly speculative,
+# not based on any data)
+# - Block open chimneys: 1.61 SAP points - latest survey showed 0 open fireplaces
+# - ASHP (45 degree emitters) with enhanced existing radiator central heating and hot water, from E rated gas boiler
+# 6.38 SAP points
+# - 4kWp PV array south and 30 degree pitch with no shading: 30.24 SAP points
+
+# Notes on solar - 30.34 seems like a lot
+# 400 watt is the solar panel output
+# Let's do a test for this property
+# This would be 10 solar panels
+# Using typical solar panel dimensions, this would be 19.63555m2 of roof space
+# The area of the roof is between 60 - 64.5 m2 (we use a API to get the roof data), implying only
+# around 30% of the roof is covered by solar panels
+# Using our machine learning model to simulate the impact of this on SAP, this would more likely result in
+# a
+
+from utils.s3 import read_dataframe_from_s3_parquet
+
+training_data = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev",
+ file_key="sap_change_model/2024-06-09-10-36-53/dataset_rooms.parquet"
+)
+# Look for properties where the only difference is solar
+ending_cols = [
+ c for c in training_data.columns if "_ending" in c and "photo_supply" not in c
+]
+ending_cols = [
+ c for c in ending_cols if
+ c not in ["sap_ending", "heat_demand_ending", "carbon_ending", "transaction_type_ending", "days_to_ending"]
+]
+
+column_pairs = {}
+for col in ending_cols:
+ starting = col.split("_ending")[0]
+ if starting + "_starting" in training_data.columns:
+ starting_col = starting + "_starting"
+ else:
+ starting_col = starting
+
+ column_pairs[col] = starting_col
+
+filtered = training_data.copy()
+# Take rows that had solar installs
+filtered = filtered[filtered["photo_supply_ending"] != filtered["photo_supply_starting"]]
+for ending_col, starting_col in column_pairs.items():
+ filtered = filtered[filtered[ending_col] == filtered[starting_col]]
+ print(f"ending_col: {ending_col}, filtered shape: {filtered.shape}")
+
+avg_change = filtered.groupby("photo_supply_ending")["rdsap_change"].mean().reset_index()
+
+# I've take every single case of there being two EPCs for a property, where the only difference between the first
+# and second is the solar installation. This is 2692 properties, across the UK. In only 4 instances has this resulted in
+# 30 or more SAP points
+
+
+# Some functions based on the SAP methodology:
+import numpy as np
+
+total_floor_area = 50
+occupants = calculate_occupants(total_floor_area)
+appliances_energy_use = estimate_electrical_appliances(occupants, total_floor_area)
+cooking_energy_use = estimate_cooking(occupants)
+
+
+def calculate_occupants(total_floor_area):
+ """
+ From Table 1b
+ :param total_floor_area:
+ :return:
+ """
+ return 1 + (1.76 * (1 - np.exp(-0.000349 * (total_floor_area - 13.9) * (total_floor_area - 13.9))) + 0.0013 * (
+ total_floor_area - 13.9))
+
+
+def estimate_electrical_appliances(occupants, total_floor_area):
+ """
+ From seciont L2 Electrical appliances
+ :param occupants:
+ :param total_floor_area:
+ :return:
+ """
+ e_a = 207.8 * np.power(total_floor_area * occupants, 0.4717)
+
+ days_in_month = {
+ 1: 31,
+ 2: 28,
+ 3: 31,
+ 4: 30,
+ 5: 31,
+ 6: 30,
+ 7: 31,
+ 8: 31,
+ 9: 30,
+ 10: 31,
+ 11: 30,
+ 12: 31
+ }
+
+ eam = 0
+ for m in range(1, 13):
+ nm = days_in_month[m]
+ eam += e_a * (1 + 0.157 * np.cos(2 * np.pi * (m - 1.78) / 12)) * nm / 365
+
+ return eam
+
+
+def estimate_cooking(occupants):
+ """
+ From section L3 Cooking
+ :param occupants:
+ :return:
+ """
+
+ return 35 + 7 * occupants
+
+
+primary_energy_per_m2 = 288 # kWh/m2 per year
+primary_energy_regulated = primary_energy_per_m2 * total_floor_area
+
+primary_energy_factor_electricity = 1.1 # Example factor
+primary_energy_appliances = appliances_energy_use * primary_energy_factor_electricity
+primary_energy_cooking = cooking_energy_use * primary_energy_factor_electricity * 365 # Annualize cooking energy
+
+total_primary_energy_use = primary_energy_regulated + primary_energy_appliances
diff --git a/etl/customers/places_for_people/route_march.py b/etl/customers/places_for_people/route_march.py
index 5da1c2f7..7b55702c 100644
--- a/etl/customers/places_for_people/route_march.py
+++ b/etl/customers/places_for_people/route_march.py
@@ -295,6 +295,49 @@ def main():
addresses_df2.to_excel("Places For People EPC data with surveyor.xlsx", index=False)
+ # Read in
+ df = pd.read_excel("Places For People EPC data with surveyor.xlsx")
+ df = df[
+ df["assessor_name"].isin(
+ [
+ "Arsalan Khalid", "Kieran Bradnock", "Wayne Davies", "Lindsay Sands", "Bruce Nethercot",
+ "Christopher Hearn", "Robert Sigerson", "Daniel Riddle", "Leroy Sands",
+ ]
+ )
+ ]
+
+ # Get the EPC
+ heights = []
+ for _, row in tqdm(df.iterrows(), total=len(df)):
+ searcher = SearchEpc(
+ address1=str(row["Matched EPC Address"]),
+ postcode=str(row["POSTCODE"]),
+ uprn=str(int(row["uprn"])),
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=True,
+ )
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+
+ height = {
+ "uprn": row["uprn"],
+ "floor_height": searcher.newest_epc["floor-height"]
+ }
+ heights.append(height)
+
+ df = df.merge(
+ pd.DataFrame(heights),
+ how="left",
+ on="uprn"
+ )
+
+ df.to_excel("WF surveyors with floor heights.xlsx", index=False)
+
if __name__ == "__main__":
main()
diff --git a/etl/customers/stonewater/no_matches.py b/etl/customers/stonewater/no_matches.py
new file mode 100644
index 00000000..e7c122b1
--- /dev/null
+++ b/etl/customers/stonewater/no_matches.py
@@ -0,0 +1,165 @@
+no_matches = [
+ {
+ 'internal_id': 4626, 'full_address': '1 Dean Lane, Sixpenny Handley, Salisbury, SP5 5AS', 'postcode': 'SP5 5AS',
+ 'Note': 'No match found - all addresses in this postcode are for Mulberry Court, Sixpenny Handley, Salisbury, '
+ 'SP5 5AS, addresses not recognised by Zoopla - possibly the postcode is incorrect and this could be'
+ 'Handley Enterprises Ltd, Unit 1 Dean Lane, Sixpenny Handley, Salisbury, SP5 5PA.'
+ 'Or this could be 1 Mulberry Court Sixpenny Handley, Salisbury SP5 5AS'
+ },
+ {
+ 'internal_id': 4627, 'full_address': '3 Dean Lane, Sixpenny Handley, Salisbury, SP5 5AS', 'postcode': 'SP5 5AS',
+ 'Note': 'No match found - all addresses in this postcode are for Mulberry Court, Sixpenny Handley, Salisbury, '
+ 'SP5 5AS, addresses not recognised by Zoopla - possibly the postcode is incorrect and this could be'
+ '2 Town Farm House, Dean Lane, Sixpenny Handley, Salisbury, SP5 5PA'
+ 'Or this could be 3 Mulberry Court Sixpenny Handley, Salisbury SP5 5AS'
+ },
+ {
+ 'internal_id': 4628, 'full_address': '5 Dean Lane, Sixpenny Handley, Salisbury, SP5 5AS', 'postcode': 'SP5 5AS',
+ 'Note': 'No match found - all addresses in this postcode are for Mulberry Court, Sixpenny Handley, Salisbury, '
+ 'SP5 5AS, addresses not recognised by Zoopla - possibly the postcode is incorrect and this could be'
+ '4 Town Farm House, Dean Lane, Sixpenny Handley, Salisbury, SP5 5PA'
+ 'Or this could be 5 Mulberry Court Sixpenny Handley, Salisbury SP5 5AS'
+ },
+ {
+ 'internal_id': 544, 'full_address': 'Room 1, Sawr, PO Box 1354, Bedford, MK41 5AB', 'postcode': 'MK41 5AB',
+ "Note": "Postcode deleted in April 2024: https://checkmypostcode.uk/mk415ab"
+ },
+ {
+ 'internal_id': 5116, 'full_address': '3 Huntspond Road, Titchfield, Fareham, PO14 4SS', 'postcode': 'PO14 4SS',
+ 'Note': 'Is this 3 St Francis Court, 195 Hunts Pond Road, Fareham, PO14 4SS, uprn: 100061988896'
+ },
+ {
+ 'internal_id': 5114, 'full_address': '4 Huntspond Road, Titchfield, Fareham, PO14 4SS', 'postcode': 'PO14 4SS',
+ 'Note': 'Is this 4 St Francis Court, 195 Hunts Pond Road, Fareham, PO14 4SS, uprn: 100061988897'
+ },
+ {
+ 'internal_id': 5115, 'full_address': '2 Huntspond Road, Titchfield, Fareham, PO14 4SS', 'postcode': 'PO14 4SS',
+ 'Note': 'Is this 2 St Francis Court, 195 Hunts Pond Road, Fareham, PO14 4SS, uprn: 100061988895'
+ },
+ {
+ 'internal_id': 5113, 'full_address': '6 Huntspond Road, Titchfield, Fareham, PO14 4SS', 'postcode': 'PO14 4SS',
+ 'Note': 'Is this 6 St Francis Court, 195 Hunts Pond Road, Fareham, PO14 4SS, uprn: 100061988899'
+ },
+ {
+ 'internal_id': 5112, 'full_address': '1 Huntspond Road, Titchfield, Fareham, PO14 4SS', 'postcode': 'PO14 4SS',
+ 'Note': 'Is this 1 St Francis Court, 195 Hunts Pond Road, Fareham, PO14 4SS, uprn: 100061988894'
+ },
+ {
+ 'internal_id': 3846, 'full_address': '2 Beaufort Road, Southbourne, Bournemouth, BH6 5BD',
+ 'postcode': 'BH6 5BD',
+ 'Note': "2 Beaufort Road, Southbourne, Bournemouth is listed under the postcode BH6 5AL - is there a typo in "
+ "the postcode?"
+ },
+ {
+ 'internal_id': 4497, 'full_address': '11 Brokenford Lane, Totton, Southampton, SO40 9LZ',
+ 'postcode': 'SO40 9LZ',
+ 'Note': "This postcode doesn't appear to exist, closest is 10 brokenford lane, Totton, Southampton, SO40 9DW."
+ "What should this be?"
+ },
+ {
+ 'internal_id': 4181, 'full_address': '25a Eastcott Road, Old Town, Swindon, SN1 3PA', 'postcode': 'SN1 3PA',
+ 'Note': 'All addresses at this postcode are for Bow Court. '
+ 'Closest match is 25 Eastcott Road, Swindon, SN1 3LT, but there is no 25A'
+ },
+ {
+ 'internal_id': 5447, 'full_address': '3 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "These is no 'Send Road' at this postcode. There are a few possible matches, e.g. Flat 3, "
+ "1 Send Road, RG4 8EH"
+ },
+ {
+ 'internal_id': 5449, 'full_address': '5 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "Same as for 3 Send Road"
+ },
+ {
+ 'internal_id': 5450, 'full_address': '6 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "Same as for 3 Send Road"
+ },
+ {
+ 'internal_id': 5446, 'full_address': '1 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "Same as for 3 Send Road"
+ },
+ {
+ 'internal_id': 5448, 'full_address': '4 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "Same as for 3 Send Road"
+ },
+ {
+ 'internal_id': 5451, 'full_address': '7 Send Road, Send Road, Reading, RG4 8EP', 'postcode': 'RG4 8EP',
+ "Note": "Same as for 3 Send Road"
+ },
+ {
+ 'internal_id': 4547, 'full_address': '2 Cecil Terrace, Bemerton, Salisbury, SP2 9NE', 'postcode': 'SP2 9NE',
+ "Note": "Addresses for this postcode are for The Croft, SP2 9NE. Should this be 2 Cecil Terrace SP2 9ND, with"
+ "uprn: 100121039798 ?"
+ },
+ {
+ 'internal_id': 4549, 'full_address': '4 Cecil Terrace, Bemerton, Salisbury, SP2 9NE', 'postcode': 'SP2 9NE',
+ "Note": "Addresses for this postcode are for The Croft, SP2 9NE. Should this be 4 Cecil Terrace SP2 9ND?"
+ },
+ {
+ 'internal_id': 3601, 'full_address': '20 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should this be 20 Constitution Hill Gardens, Poole, BH14 0PY? (i.e. postcode is wrong) "
+ "uprn: 10001086693"
+ },
+ {
+ 'internal_id': 3592, 'full_address': '7 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"
+ },
+ {
+ 'internal_id': 3594, 'full_address': '9 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"
+ },
+ {
+ 'internal_id': 3591, 'full_address': '6 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"
+ },
+ {
+ 'internal_id': 3593, 'full_address': '8 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {
+ 'internal_id': 3590, 'full_address': '5 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {
+ 'internal_id': 3589, 'full_address': '3 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {
+ 'internal_id': 3600, 'full_address': '18 Constitution Hill, Parkstone, Poole, BH14 0PX',
+ 'postcode': 'BH14 0PX', "Note": "Should the postcode be BH14 0PY ?"},
+ {
+ 'internal_id': 3599, 'full_address': '17 Constitution Hill, Parkstone, Poole, BH14 0PX',
+ 'postcode': 'BH14 0PX', "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3598, 'full_address': '15 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3608, 'full_address': '26 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3610, 'full_address': '30 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3603, 'full_address': '22 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3612, 'full_address': '32 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3595, 'full_address': '10 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+ {'internal_id': 3613, 'full_address': '34 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0PY ?"},
+
+ {'internal_id': 3597, 'full_address': '12 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3602, 'full_address': '21 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3606, 'full_address': '19 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3604, 'full_address': '23 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3605, 'full_address': '25 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3609, 'full_address': '29 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3596, 'full_address': '11 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3607, 'full_address': '27 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 3611, 'full_address': '31 Constitution Hill, Parkstone, Poole, BH14 0PX', 'postcode': 'BH14 0PX',
+ "Note": "Should the postcode be BH14 0QB ?"},
+ {'internal_id': 5622, 'full_address': '26 Roman Way, Andover, SP10 5HZ', 'postcode': 'SP10 5HZ',
+ 'Note': 'Shoul this postcode be SP10 5JU ?'}
+]
diff --git a/etl/customers/stonewater/shdf_3_clustering.py b/etl/customers/stonewater/shdf_3_clustering.py
new file mode 100644
index 00000000..6c7a0fc6
--- /dev/null
+++ b/etl/customers/stonewater/shdf_3_clustering.py
@@ -0,0 +1,1917 @@
+import json
+from tqdm import tqdm
+import os
+from dotenv import load_dotenv
+from backend.SearchEpc import SearchEpc
+import urllib.parse
+import requests
+from datetime import datetime
+from scipy import stats
+
+from fuzzywuzzy import fuzz
+import numpy as np
+import pandas as pd
+import time
+from utils.s3 import save_data_to_s3, read_excel_from_s3, read_from_s3, read_dataframe_from_s3_parquet, \
+ save_dataframe_to_s3_parquet, save_pickle_to_s3
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+# We create a MAP of uprns, for EPCs that didn't give use the UPRN
+missing_uprn_map = [
+ # This is a map from internal_id to UPRN, for properties where we do have an EPC, but we don't have
+ # a uprn
+ # 1 Church Street, Alfreton, DE55 7AH
+ {"internal_id": 78, "mapped_uprn": None}, # Doesn't seem to exist any more
+ # 1 Granville Road, Luton, LU1 1PA
+ {"internal_id": 315, "mapped_uprn": 100080148856},
+ # 11 College Street, Birstall, Batley, WF17 9HF
+ # The EPC record is for 11 and 11a
+ {"internal_id": 1090, "mapped_uprn": 83190440},
+ # 11a College Street, Birstall, Batley, WF17 9HF
+ {"internal_id": 1092, "mapped_uprn": 83143766},
+ # Flat 5 Friars Street, Hereford, HR4 0AS
+ # TODO: Check this
+ {"internal_id": 1384, "mapped_uprn": 200002600892},
+ # This UPRN is for 5 Friars Court, which is a flat
+ # Flat 7 Friars Street, Hereford, HR4 0AS
+ # TODO: Check this
+ {"internal_id": 1385, "mapped_uprn": 200002600894},
+ # This UPRN is for 7 Friars Court, which is a flat
+ # 1 Waverley Street, Dudley, DY2 0YE
+ {"internal_id": 3349, "mapped_uprn": 90022438},
+ # 5 Brighton Road, Burgh Heath, Tadworth, KT20 6BQ
+ # TODO: Check this
+ # This UPRN is for 5 Copthorne, Brighton Road, Burgh Heath, KT20 6BQ, which is a flat
+ {"internal_id": 5027, "mapped_uprn": 100062145273},
+ # Room 1, 21 Coxford Road, Southampton, SO16 5FG
+ # This is for 21 Coxford Road
+ {"internal_id": 5554, "mapped_uprn": 100060692392},
+
+]
+missing_uprn_map = pd.DataFrame(missing_uprn_map)
+
+internal_id_epcs_to_drop = [315, 1384, 1385, 3349]
+
+
+def remove_commas_and_full_stops(input_string: str) -> str:
+ """
+ Removes commas and full stops from the input string.
+
+ Args:
+ input_string (str): The string from which to remove commas and full stops.
+
+ Returns:
+ str: The string with commas and full stops removed.
+ """
+ return input_string.replace(',', '').replace('.', '')
+
+
+def get_places_with_retry(searcher, max_retries=5, wait_time=2):
+ """
+ Tries to call the get_places_api method up to max_retries times,
+ with a wait_time interval between attempts in case of failure.
+
+ Args:
+ searcher (object): The searcher object with the ordnance_survey_client.
+ max_retries (int): Maximum number of retry attempts.
+ wait_time (int): Wait time in seconds between retries.
+
+ Returns:
+ result: The result from the get_places_api method or None if all attempts fail.
+ """
+ for attempt in range(max_retries):
+ try:
+ response = searcher.ordnance_survey_client.get_places_api()
+ status = response.get("status")
+ if status == 200:
+ return response # Return the result if successful
+ else:
+ print(f"Attempt {attempt + 1} failed with status code: {status}")
+ except Exception as e:
+ print(f"Attempt {attempt + 1} failed with error: {e}")
+
+ if attempt < max_retries - 1:
+ print(f"Retrying in {wait_time} seconds...")
+ time.sleep(wait_time)
+
+ print(f"All {max_retries} attempts failed.")
+ return None
+
+
+def app():
+ """
+ This script handles the preparation of the data from Stonewater, to archetype a collection
+ of 5.3k properties and reduce that down to a representative set of 450 properties.
+
+ Here, we prepare the input data for clustering
+ :return:
+ """
+
+ # TODO: Temp read from local machine - move to s3
+ # asset_list = pd.read_excel(
+ # "/Users/khalimconn-kowlessar/Downloads/Stonewater SHDF_3_0_Board Triage 22.05.24.xlsx", header=4
+ # )
+
+ asset_list = read_excel_from_s3(
+ file_key="customers/Stonewater/Stonewater SHDF_3_0_Board Triage 22.05.24.xlsx",
+ bucket_name="retrofit-data-dev",
+ header_row=4
+ )
+
+ # Drop the bottom 4 rows, which are completely missing
+ asset_list = asset_list.head(-4)
+
+ # Keep just the columns we're interested in
+ asset_list = asset_list[
+ [
+ "Osm. ID",
+ "Org. ref.",
+ "Postcode",
+ "House no",
+ "Name",
+ "Address line 2",
+ "City/Town",
+ "County",
+ "Address ID", # This is not uprn
+ ]
+ ].rename(
+ columns={
+ "Osm. ID": "internal_id",
+ "Org. ref.": "customer_asset_id",
+ "Postcode": "postcode",
+ "House no": "house_number",
+ "Name": "address1",
+ "Address line 2": "address2",
+ "City/Town": "city_town",
+ "County": "county",
+ "Address ID": "external_address_id",
+ }
+ )
+
+ # Create full address
+ asset_list["full_address"] = np.where(
+ ~pd.isnull(asset_list["address2"]),
+ (
+ asset_list["address1"] + ", " +
+ asset_list["address2"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ ),
+ asset_list["address1"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ )
+
+ if pd.isnull(asset_list["full_address"]).sum():
+ raise ValueError("Missing full addresses")
+
+ # Pull in the data
+ # This data has already been pulled as much as it can be, so we retrieve the existing extraction from S3
+
+ # Perform an initial pull without ordnance survey data
+ # epc_data = []
+ # older_epc_data = {}
+ #
+ # for row_number, asset in tqdm(asset_list.iterrows(), total=len(asset_list)):
+ # searcher = SearchEpc(
+ # address1=str(asset["address1"]),
+ # postcode=str(asset["postcode"]),
+ # auth_token=EPC_AUTH_TOKEN,
+ # os_api_key="",
+ # full_address=str(asset["full_address"]),
+ # uprn=asset.get("uprn", None),
+ # )
+ # searcher.find_property(skip_os=True)
+ #
+ # if searcher.newest_epc is None:
+ # continue
+ #
+ # epc_data.append(
+ # {
+ # "internal_id": asset["internal_id"],
+ # **searcher.newest_epc
+ # }
+ # )
+ #
+ # if searcher.older_epcs is not None:
+ # older_epc_data[asset["internal_id"]] = searcher.older_epcs
+ #
+ # # Store to S3
+ # save_data_to_s3(
+ # data=json.dumps(epc_data),
+ # s3_file_name="customers/Stonewater/clustering/epc_data.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ #
+ # save_data_to_s3(
+ # data=json.dumps(older_epc_data),
+ # s3_file_name="customers/Stonewater/clustering/old_epc_data.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ # We read this directly from s3
+ epc_data = json.loads(
+ read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/epc_data.json"
+ )
+ )
+
+ older_epc_data = json.loads(
+ read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/old_epc_data.json"
+ )
+ )
+
+ # Perform a comparison between the EPC address and the asset list address, just to double check
+
+ epc_data_df = pd.DataFrame(epc_data)
+ address_comparison = (
+ asset_list[["internal_id", "full_address", "postcode", "house_number", "address1"]].merge(
+ epc_data_df[["internal_id", "address", "postcode", "address1"]].rename(
+ columns={
+ "address": "epc_address",
+ "postcode": "epc_postcode",
+ "address1": "epc_address1"
+ }
+ ),
+ how="inner",
+ on="internal_id"
+ )
+ )
+
+ # Produce a metric, showing the matching confidence between the two
+ address_comparison["epc_extracted_house_number"] = address_comparison["epc_address1"].apply(
+ lambda x: SearchEpc.get_house_number(x)
+ )
+
+ address_comparison["house_numbers_match"] = (
+ address_comparison["house_number"].str.lower() == address_comparison["epc_extracted_house_number"].str.lower()
+ )
+
+ # We also produce a address similarity metric
+ # We convert the strings to lower and remove common punctuation
+
+ address_comparison["address_similarity_score"] = address_comparison.apply(
+ lambda x: fuzz.ratio(
+ remove_commas_and_full_stops(x["address1"].lower()),
+ remove_commas_and_full_stops(x["epc_address1"].lower())
+ ),
+ axis=1
+ )
+
+ address_comparison = address_comparison.sort_values("address_similarity_score", ascending=True)
+ address_comparison = address_comparison[
+ ["internal_id", "full_address", "epc_address", "address_similarity_score", "house_numbers_match"]
+ ]
+
+ # Anything with less than a 90 similarity score, let's do again
+ needs_ordnance_survey = address_comparison[
+ (address_comparison["address_similarity_score"] <= 90) |
+ (~address_comparison["house_numbers_match"])
+ ].copy()
+
+ is_ok = address_comparison[~address_comparison["internal_id"].isin(needs_ordnance_survey["internal_id"])]
+ is_ok = is_ok.sort_values("address_similarity_score", ascending=True)
+
+ os_data_pull_asset_list = asset_list[
+ ~asset_list["internal_id"].isin(is_ok["internal_id"].values)
+ ].copy()
+
+ # We have already done a partial pull of the Ordnance survey data so we can skip some of the records
+ # os_most_relevant_1 = json.loads(
+ # read_from_s3(
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/os_most_relevant_1.json"
+ # )
+ # )
+ #
+ # os_most_relevant_2 = json.loads(
+ # read_from_s3(
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/os_most_relevant_2.json"
+ # )
+ # )
+ #
+ # fetched_internal_ids = (
+ # [x["internal_id"] for x in os_most_relevant_1] + [x["internal_id"] for x in os_most_relevant_2]
+ # )
+ #
+ # # We remove any ids we've already fetched
+ # os_data_pull_asset_list = os_data_pull_asset_list[
+ # ~os_data_pull_asset_list["internal_id"].isin(fetched_internal_ids)
+ # ]
+ #
+ # # Our OK EPC data (is_ok) + ordnance survey fetched data + the data we need to fetch should equal the total
+ # # number of assets
+ # assert len(is_ok) + len(fetched_internal_ids) + len(os_data_pull_asset_list) == len(asset_list)
+
+ os_data_pull_asset_list = os_data_pull_asset_list.reset_index(drop=True)
+
+ # For each of these records, we pull the OS data
+ # ORDNANCE_SURVEY_API_KEY = "" # This API key is a temp key which I have copied locally
+ # os_most_relevant = []
+ # os_all = {}
+ # errors = []
+ # for _, asset in tqdm(os_data_pull_asset_list.iterrows(), total=len(os_data_pull_asset_list)):
+ # # Calls are throttled to 50 per minute in development mode, so lets just slow this down
+ # time.sleep(2)
+ #
+ # searcher = SearchEpc(
+ # address1=str(asset["address1"]),
+ # postcode=str(asset["postcode"]),
+ # auth_token=EPC_AUTH_TOKEN,
+ # os_api_key=ORDNANCE_SURVEY_API_KEY,
+ # full_address=str(asset["full_address"]),
+ # uprn=asset.get("uprn", None),
+ # )
+ # searcher.ordnance_survey_client.full_address = asset["full_address"]
+ # # Attempt to get places data with retry logic
+ # result = get_places_with_retry(searcher)
+ #
+ # if result:
+ # # Get the most relevant response
+ # os_most_relevant.append(
+ # {
+ # "internal_id": asset["internal_id"],
+ # **searcher.ordnance_survey_client.most_relevant_result
+ # }
+ # )
+ #
+ # # Also keep the best 100 results
+ # os_all[asset["internal_id"]] = searcher.ordnance_survey_client.results
+ # else:
+ # # Record the internal_id of the asset that failed
+ # print("Error for address: " + asset["full_address"])
+ # errors.append(asset["internal_id"])
+
+ # Store to S3
+ # save_data_to_s3(
+ # data=json.dumps(os_most_relevant),
+ # s3_file_name="customers/Stonewater/clustering/os_most_relevant_3.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ #
+ # save_data_to_s3(
+ # data=json.dumps(os_all),
+ # s3_file_name="customers/Stonewater/clustering/os_all_3.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ #
+ # save_data_to_s3(
+ # data=json.dumps(errors),
+ # s3_file_name="customers/Stonewater/clustering/errors_3.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+
+ # We now collate all of the data for the following steps:
+ # 1) Checking the retrieve ordnance survey data against ordnance survey data
+ # 2) A second round of querying the EPC api to find the EPC data, in case we retrieve something using uprn
+ # 3) Predicting the EPC data for the properties we have no data for
+ # 4) Retrieveing additional data against the internal_id
+ # 5) Creation of final dataset for clustering
+
+ os_most_relevant = []
+ os_all = {}
+ for i in ["1", "2", "3"]:
+ most_relevant_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_most_relevant_{i}.json"
+ )
+ os_most_relevant.extend(json.loads(most_relevant_segment))
+ os_all_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_all_{i}.json"
+ )
+ os_all = {**os_all, **json.loads(os_all_segment)}
+
+ os_most_relevant = pd.DataFrame(os_most_relevant)
+
+ os_address_comparison = os_data_pull_asset_list[
+ ["internal_id", "full_address", "postcode", "house_number", "address1"]
+ ].merge(
+ os_most_relevant[["internal_id", "ADDRESS", "POSTCODE", "UPRN"]],
+ how="inner",
+ on="internal_id"
+ )
+
+ # Compare house number
+ # Check for records where the postcode doesn't match
+ os_address_comparison["postcodes_match"] = (
+ os_address_comparison["postcode"].str.lower() == os_address_comparison["POSTCODE"].str.lower()
+ )
+
+ # extract it from ADDRESS
+ os_address_comparison["extracted_house_number"] = os_address_comparison["ADDRESS"].apply(
+ lambda x: SearchEpc.get_house_number(x)
+ )
+
+ # Compare house number
+ os_address_comparison["house_numbers_match"] = (
+ os_address_comparison["house_number"].str.lower() == os_address_comparison["extracted_house_number"].str.lower()
+ )
+
+ # String similarity
+ os_address_comparison["address_similarity_score"] = os_address_comparison.apply(
+ lambda x: fuzz.ratio(
+ remove_commas_and_full_stops(x["full_address"].lower()),
+ remove_commas_and_full_stops(x["ADDRESS"].lower())
+ ),
+ axis=1
+ )
+
+ os_address_comparison = os_address_comparison.sort_values("address_similarity_score", ascending=True)
+
+ problematic = os_address_comparison.copy()
+
+ problematic = problematic[
+ (problematic["address_similarity_score"] <= 80) |
+ (~problematic["house_numbers_match"]) |
+ (~problematic["postcodes_match"])
+ ]
+
+ # TODO: We'll label these problematic records as problematic, in the final output
+
+ # different_postcodes = problematic[~problematic["postcodes_match"]].copy().reset_index(drop=True)
+
+ ORDNANCE_SURVEY_API_KEY = "" # This API key is a temp key which I have copied locally
+ problematic_os = []
+ problematic_os_all = {}
+ problematic_errors = []
+ for _, row in tqdm(problematic.iterrows(), total=len(problematic)):
+ # Let's just do a backup pull - we're now using LPI too
+ time.sleep(2)
+ backup_searher = SearchEpc(
+ address1=row["address1"],
+ postcode=row["postcode"],
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key=ORDNANCE_SURVEY_API_KEY,
+ uprn=None,
+ )
+ # Attempt to get places data with retry logic
+ result = get_places_with_retry(backup_searher)
+
+ if result:
+ # Get the most relevant response
+ problematic_os.append(
+ {
+ "internal_id": row["internal_id"],
+ **backup_searher.ordnance_survey_client.most_relevant_result
+ }
+ )
+
+ # Also keep the best 100 results
+ problematic_os_all[row["internal_id"]] = backup_searher.ordnance_survey_client.results
+ else:
+ # Record the internal_id of the asset that failed
+ print("Error for address: " + row["full_address"])
+ problematic_errors.append(row["internal_id"])
+
+ # Store to S3
+ # save_data_to_s3(
+ # data=json.dumps(problematic_os),
+ # s3_file_name="customers/Stonewater/clustering/problematic_os.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ #
+ # save_data_to_s3(
+ # data=json.dumps(problematic_os_all),
+ # s3_file_name="customers/Stonewater/clustering/problematic_os_all.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+ #
+ # save_data_to_s3(
+ # data=json.dumps(problematic_errors),
+ # s3_file_name="customers/Stonewater/clustering/problematic_errors.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+
+ # Next steps: We should collate all of the data and produce 1 big dataset
+
+ problematic_os_df = pd.DataFrame(problematic_os)
+ problematic_address_comparison = problematic[["internal_id", "full_address", "postcode", "house_number"]].merge(
+ problematic_os_df[["internal_id", "ADDRESS", "POSTCODE", "UPRN"]],
+ how="inner",
+ on="internal_id"
+ )
+
+ problematic_address_comparison["OS_POSTCODE"] = problematic_address_comparison["ADDRESS"].str.split(", ").str[-1]
+ problematic_address_comparison["postcodes_match"] = (
+ problematic_address_comparison["postcode"].str.lower() == problematic_address_comparison[
+ "OS_POSTCODE"].str.lower()
+ )
+
+ problematic_address_comparison["match_similarity_score"] = problematic_address_comparison.apply(
+ lambda x: fuzz.ratio(
+ remove_commas_and_full_stops(x["full_address"].lower()),
+ remove_commas_and_full_stops(x["ADDRESS"].lower())
+ ),
+ axis=1
+ )
+ problematic_address_comparison = problematic_address_comparison.sort_values(
+ "match_similarity_score", ascending=True
+ )
+
+ # let's do a house number extraction
+ problematic_address_comparison["extracted_house_number"] = problematic_address_comparison.apply(
+ lambda x: SearchEpc.get_house_number(x["ADDRESS"], x["OS_POSTCODE"]), axis=1
+ )
+
+ problematic_address_comparison["house_numbers_different"] = (
+ problematic_address_comparison["house_number"].str.lower().str.split(",").str[0].str.split(" ").str[0] !=
+ problematic_address_comparison[
+ "extracted_house_number"].str.lower()
+ )
+
+ # We perform a final check
+ # Take anything where the postcodes don't match, where the house numbers are different and the match similarity
+ # is less than 90, or the match similarity is less than 80
+ final_check = problematic_address_comparison[
+ (~problematic_address_comparison["postcodes_match"])
+ ]
+ final_check = final_check.sort_values("match_similarity_score", ascending=False)
+ final_check = final_check.reset_index(drop=True)
+
+ final_best_matches = []
+ no_matches = []
+ for _, row in final_check.iterrows():
+ os_data = problematic_os_all[row["internal_id"]]
+ os_data = pd.DataFrame(
+ [x["DPA"] if "DPA" in x else x["LPI"] for x in os_data]
+ )
+
+ if ("POSTCODE_LOCATOR" in os_data.columns) and ("POSTCODE" in os_data.columns):
+ os_data["postcode"] = np.where(
+ ~pd.isnull(os_data["POSTCODE"]),
+ os_data["POSTCODE"],
+ os_data["POSTCODE_LOCATOR"]
+ )
+ elif "POSTCODE" in os_data.columns:
+ os_data["postcode"] = os_data["POSTCODE"]
+ else:
+ os_data["postcode"] = os_data["POSTCODE_LOCATOR"]
+ os_data = os_data[os_data["postcode"].str.lower() == row["postcode"].lower()]
+ if os_data.shape[0] >= 1:
+ final_best_matches.append(
+ {
+ "internal_id": row["internal_id"],
+ **os_data.iloc[0].to_dict()
+ }
+ )
+ else:
+ no_matches.append(
+ {
+ "internal_id": row["internal_id"],
+ "full_address": row["full_address"],
+ "postcode": row["postcode"]
+ }
+ )
+
+ no_matches = pd.DataFrame(no_matches)
+
+ # Data to be confirmed
+ from etl.customers.stonewater.no_matches import no_matches
+ no_matches_to_export = pd.DataFrame(no_matches)
+ no_matches_to_export = asset_list.merge(
+ no_matches_to_export[["internal_id", "Note"]],
+ how="inner",
+ on="internal_id"
+ ).rename(
+ columns={
+ "internal_id": "Osm. ID",
+ "customer_asset_id": "Org. ref.",
+ "external_address_id": "Address ID",
+ }
+ )
+ no_matches_to_export.to_excel("Stonewater - addresses with no matches.xlsx", index=False)
+
+ # We also confirm final_best_matches
+ final_best_matches_df = pd.DataFrame(final_best_matches)[
+ ["internal_id", "ADDRESS", "UPRN"]
+ ].rename(
+ columns={
+ "ADDRESS": "Ordnance Survey Address - same postcode (best match)",
+ "UPRN": "UPRN - same postcode (best match)"
+ }
+ )
+ # We also get their original match
+ final_best_matches_df = final_best_matches_df.merge(
+ problematic[["internal_id", "ADDRESS", "UPRN"]].rename(
+ columns={
+ "ADDRESS": "Ordnance Survey Address - best possible match",
+ "UPRN": "UPRN - best possible match"
+ }
+ ),
+ how="inner",
+ on="internal_id"
+ )
+
+ # merge on the original data
+ final_best_matches_df = asset_list.merge(
+ final_best_matches_df,
+ how="inner",
+ on="internal_id"
+ ).rename(
+ columns={
+ "internal_id": "Osm. ID",
+ "customer_asset_id": "Org. ref.",
+ "external_address_id": "Address ID",
+ }
+ )
+
+ # "Osm. ID": "internal_id",
+ # "Org. ref.": "customer_asset_id",
+ # "Postcode": "postcode",
+ # "House no": "house_number",
+ # "Name": "address1",
+ # "Address line 2": "address2",
+ # "City/Town": "city_town",
+ # "County": "county",
+ # "Address ID": "external_address_id",
+
+
+def filter_os_data(p_os_data, p_os_data_all, udprn, is_flat):
+ if udprn is None:
+ p_os_data_all = pd.DataFrame([z["DPA"] if "DPA" in z else z["LPI"] for z in p_os_data_all])
+ if is_flat:
+ p_os_data_all = p_os_data_all[p_os_data_all["CLASSIFICATION_CODE"] == "RD06"]
+ return p_os_data_all.head(1)
+
+ return p_os_data_all.head(1)
+
+ final_os_data = p_os_data[p_os_data["UDPRN"] == udprn]
+ if final_os_data.empty:
+ p_os_data_all = pd.DataFrame([z["DPA"] if "DPA" in z else z["LPI"] for z in p_os_data_all])
+ final_os_data = p_os_data_all[p_os_data_all["UDPRN"].astype(str) == udprn]
+
+ return final_os_data
+
+
+def compile_data():
+ """
+ Various data sources have been produced to create the final data source for Stonewater.
+ This function combines them
+ :return:
+ """
+ ########################################################################
+ # Read in data
+ ########################################################################
+ # asset_list = read_excel_from_s3(
+ # file_key="customers/Stonewater/Stonewater SHDF_3_0_Board Triage 22.05.24.xlsx",
+ # bucket_name="retrofit-data-dev",
+ # header_row=4
+ # )
+ #
+ # udprn_data = read_excel_from_s3(
+ # file_key="customers/Stonewater/UDPRN updated RA Sample for 5 year programme.xlsx",
+ # bucket_name="retrofit-data-dev",
+ # header_row=0
+ # )[["AddressId", "UDPRN"]].rename(columns={"AddressId": "external_address_id"})
+
+ asset_list = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/Stonewater SHDF_3_0_Board Triage 22.05.24.xlsx", header=4
+ )
+
+ udprn_data = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/UDPRN updated RA Sample for 5 year programme.xlsx", header=0
+ )[["AddressId", "UDPRN"]].rename(columns={"AddressId": "Address ID"})
+ udprn_data["UDPRN"] = udprn_data["UDPRN"].astype("Int64").astype(str)
+ udprn_data["Address ID"] = udprn_data["Address ID"].astype(str)
+
+ asset_list = asset_list.merge(udprn_data, how="inner", on="Address ID")
+ asset_list = asset_list.rename(columns={"UDPRN": "udprn"})
+
+ # Read in the lookups
+ uprn_lookup_1 = pd.DataFrame(json.loads(read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup.json"
+ )))
+
+ uprn_lookup_2 = pd.DataFrame(json.loads(read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup_2.json"
+ )))
+ uprn_lookup_2 = uprn_lookup_2.rename(
+ columns={
+ "epc_address": "standardised_address",
+ "epc_postcode": "standardised_postcode"
+ }
+ )
+
+ # concat
+ uprn_lookup = pd.concat([uprn_lookup_1, uprn_lookup_2])
+
+ # TODO: Read in UPRNs or UDPRN
+ # UPRN LOOKUPS TO READ IN: address_uprn_udprn_lookup, address_uprn_udprn_lookup_2
+
+ epc_data = json.loads(
+ read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/epc_data.json"
+ )
+ )
+ epc_data = pd.DataFrame(epc_data)
+
+ # We drop come EPCS
+ epc_data = epc_data[~epc_data["internal_id"].isin(internal_id_epcs_to_drop)]
+
+ # This we can use to produce additional variables such as number of old surveys
+ # older_epc_data = json.loads(
+ # read_from_s3(
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/old_epc_data.json"
+ # )
+ # )
+ # older_epc_data = {k: v for k, v in older_epc_data.items() if k not in internal_id_epcs_to_drop}
+
+ # This is the first ordnance survey data pull
+ os_most_relevant_1 = []
+ os_all_1 = {}
+ for i in tqdm(["1", "2", "3"]):
+ most_relevant_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_most_relevant_{i}.json"
+ )
+ os_most_relevant_1.extend(json.loads(most_relevant_segment))
+ os_all_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_all_{i}.json"
+ )
+ os_all_1 = {**os_all_1, **json.loads(os_all_segment)}
+
+ os_most_relevant_1 = pd.DataFrame(os_most_relevant_1)
+
+ # This is the second ordnance survey data pull
+ os_most_relevant_2 = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/problematic_os.json"
+ )
+ os_most_relevant_2 = json.loads(os_most_relevant_2)
+ os_most_relevant_2 = pd.DataFrame(os_most_relevant_2)
+
+ os_all_2 = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/problematic_os_all.json"
+ )
+ os_all_2 = json.loads(os_all_2)
+
+ ########################################################################
+ # Prepare asset list
+ ########################################################################
+ # TODO: Merge on UPRNs
+
+ # Keep just the columns we're interested in
+ asset_list = asset_list[
+ [
+ "Osm. ID",
+ "Org. ref.",
+ "Postcode",
+ "House no",
+ "Name",
+ "Address line 2",
+ "City/Town",
+ "County",
+ "Address ID", # This is not uprn
+ "udprn"
+ ]
+ ].rename(
+ columns={
+ "Osm. ID": "internal_id",
+ "Org. ref.": "customer_asset_id",
+ "Postcode": "postcode",
+ "House no": "house_number",
+ "Name": "address1",
+ "Address line 2": "address2",
+ "City/Town": "city_town",
+ "County": "county",
+ "Address ID": "external_address_id",
+ }
+ )
+
+ # Create full address
+ asset_list["full_address"] = np.where(
+ ~pd.isnull(asset_list["address2"]),
+ (
+ asset_list["address1"] + ", " +
+ asset_list["address2"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ ),
+ asset_list["address1"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ )
+
+ if pd.isnull(asset_list["full_address"]).sum():
+ raise ValueError("Missing full addresses")
+
+ # Merge on UDPRN
+
+ asset_list = asset_list.merge(
+ uprn_lookup.drop(columns=["udprn"]), how="left", on=["internal_id", "external_address_id"]
+ )
+
+ # This is everything without a uprn
+
+ # Quick check to see if we have os data for every property that doesn't have an EPC
+ without_epc = asset_list[~asset_list["internal_id"].isin(epc_data["internal_id"].values)]
+
+ os_most_relevant_1_internal_ids = os_most_relevant_1["internal_id"].tolist()
+ os_most_relevant_2_internal_ids = os_most_relevant_2["internal_id"].tolist()
+
+ missing_os_data = []
+ for _, x in without_epc.iterrows():
+ # We would prioritise the data pulled the second time around
+
+ internal_id = x["internal_id"]
+ if internal_id in os_most_relevant_2_internal_ids:
+ continue
+
+ if internal_id in os_most_relevant_1_internal_ids:
+ continue
+
+ missing_os_data.append(internal_id)
+
+ if len(missing_os_data):
+ raise Exception("We don't have SOME data for each internal_id")
+
+ # Let's create a lookup table of internal_id, external_address_id, udprn, uprn, standardised_address
+ address_uprn_udprn_lookup = []
+ for _, x in without_epc.iterrows():
+ if pd.isnull(x["UDPRN"]):
+ continue
+ udprn = str(int(x["UDPRN"]))
+ internal_id = x["internal_id"]
+
+ is_flat = "flat" in x["address1"].lower()
+
+ # Get the OS data
+ final_os_data = pd.DataFrame()
+ if internal_id in os_most_relevant_1_internal_ids:
+ p_os_data = os_most_relevant_1[os_most_relevant_1["internal_id"] == internal_id]
+ p_os_data_all = os_all_1[str(internal_id)]
+ final_os_data = filter_os_data(p_os_data, p_os_data_all, udprn, is_flat)
+
+ if (internal_id in os_most_relevant_2_internal_ids) and final_os_data.empty:
+ p_os_data = os_most_relevant_2[os_most_relevant_2["internal_id"] == internal_id]
+ p_os_data_all = os_all_2[str(internal_id)]
+
+ final_os_data = filter_os_data(p_os_data, p_os_data_all, udprn, is_flat)
+
+ if final_os_data.empty:
+ continue
+
+ if final_os_data.shape[0] != 1:
+ if final_os_data["UPRN"].nunique() > 1:
+ raise Exception("Investigate me")
+
+ address_uprn_udprn_lookup.append(
+ {
+ "internal_id": internal_id,
+ "external_address_id": x["external_address_id"],
+ "udprn": udprn,
+ "uprn": final_os_data["UPRN"].values[0],
+ "standardised_address": final_os_data["ADDRESS"].values[0],
+ "standardised_postcode": final_os_data["POSTCODE"].values[0]
+ }
+ )
+
+ # Store this lookup
+ # save_data_to_s3(
+ # data=json.dumps(address_uprn_udprn_lookup),
+ # s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+
+ address_uprn_udprn_lookup = pd.DataFrame(address_uprn_udprn_lookup)
+ missed = asset_list[~asset_list["internal_id"].isin(address_uprn_udprn_lookup["internal_id"].values)]
+
+ address_comparison = (
+ asset_list[
+ ["internal_id", "external_address_id", "UDPRN", "full_address", "postcode", "house_number", "address1"]
+ ].merge(
+ epc_data[["internal_id", "address", "postcode", "address1", "uprn"]].rename(
+ columns={
+ "address": "epc_address",
+ "postcode": "epc_postcode",
+ "address1": "epc_address1"
+ }
+ ),
+ how="inner",
+ on="internal_id"
+ )
+ )
+
+ address_comparison["address_similarity_score"] = address_comparison.apply(
+ lambda x: fuzz.ratio(
+ remove_commas_and_full_stops(x["address1"].lower() + x["postcode"].lower()),
+ remove_commas_and_full_stops(x["epc_address1"].lower() + x["epc_postcode"].lower())
+ ),
+ axis=1
+ )
+ address_comparison = address_comparison.sort_values("address_similarity_score", ascending=False)
+ # Cond
+ confident = address_comparison[address_comparison["address_similarity_score"] >= 95]
+ low_confidence = address_comparison[address_comparison["address_similarity_score"] < 95].copy()
+
+ lookup_2 = confident[
+ [
+ 'internal_id', 'external_address_id', 'UDPRN', 'uprn',
+ 'epc_address', 'epc_postcode']
+ ].rename(columns={"UDPRN": "udprn"})
+
+ # Store in S3
+ # save_data_to_s3(
+ # data=json.dumps(lookup_2.to_dict("records")),
+ # s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup_2.json",
+ # bucket_name="retrofit-data-dev"
+ # )
+
+ # Need to deal with the low confidence records
+ low_confidence_asset_list = asset_list[asset_list["internal_id"].isin(low_confidence["internal_id"])]
+ for _, x in low_confidence_asset_list.iterrows():
+ udprn = str(int(x["UDPRN"]))
+ internal_id = x["internal_id"]
+ # Get the OS data
+ final_os_data = pd.DataFrame()
+ if internal_id in os_most_relevant_1_internal_ids:
+ p_os_data = os_most_relevant_1[os_most_relevant_1["internal_id"] == internal_id]
+ p_os_data_all = os_all_1[str(internal_id)]
+ final_os_data = filter_os_data(p_os_data, p_os_data_all, udprn)
+
+ if (internal_id in os_most_relevant_2_internal_ids) and final_os_data.empty:
+ p_os_data = os_most_relevant_2[os_most_relevant_2["internal_id"] == internal_id]
+ p_os_data_all = os_all_2[str(internal_id)]
+
+ final_os_data = filter_os_data(p_os_data, p_os_data_all, udprn)
+
+ # For the EPC data, some of them are missing UPRN
+ epc_data = epc_data.merge(missing_uprn_map, how="left", on="internal_id")
+ epc_data["uprn"] = np.where(
+ epc_data["uprn"] == "",
+ epc_data["mapped_uprn"],
+ epc_data["uprn"]
+ )
+ epc_data = epc_data.drop(columns=["mapped_uprn"])
+
+ # Once we have UPRNs, we might want to pull in the EPC data again
+ # epc_data_with_uprn = []
+ # older_epc_data_with_uprn = {}
+ #
+ # for row_number, asset in tqdm(asset_list.iterrows(), total=len(asset_list)):
+ # searcher = SearchEpc(
+ # address1=str(asset["address1"]),
+ # postcode=str(asset["postcode"]),
+ # auth_token=EPC_AUTH_TOKEN,
+ # os_api_key="",
+ # full_address=str(asset["full_address"]),
+ # uprn=asset["uprn"]
+ # )
+ # searcher.find_property(skip_os=True)
+ #
+ # if searcher.newest_epc is None:
+ # continue
+ #
+ # epc_data_with_uprn.append(
+ # {
+ # "internal_id": asset["internal_id"],
+ # **searcher.newest_epc
+ # }
+ # )
+ #
+ # if searcher.older_epcs is not None:
+ # older_epc_data_with_uprn[asset["internal_id"]] = searcher.older_epcs
+
+ # We now get the remaining properties
+ # TODO: We might want to use epc_data_with_uprn
+ remaining_properties = asset_list[~asset_list["internal_id"].isin(epc_data["internal_id"].values)]
+
+ # We estimate the data
+ final_epcs = []
+ for _, p in remaining_properties.iterrows():
+ internal_id = p["internal_id"]
+ uprn = p["UPRN"]
+
+ if internal_id in os_most_relevant_1_internal_ids:
+ p_os_data = os_most_relevant_1[os_most_relevant_1["internal_id"] == internal_id].to_dict("records")[0]
+ p_os_full = os_all_1[str(internal_id)]
+ else:
+ p_os_data = os_most_relevant_2[os_most_relevant_2["internal_id"] == internal_id].to_dict("records")[0]
+ p_os_full = os_all_2[str(internal_id)]
+ p_os_full = pd.DataFrame(
+ [x["DPA"] if "DPA" in x else x["LPI"] for x in p_os_full]
+ )
+
+ # TODO: Add this back in
+ # When we have this
+ if p["uprn"] != p_os_data["UPRN"]:
+ # Get it from the older data
+ filtered = p_os_full[p_os_full["UPRN"] == p["uprn"]]
+ p_os_data = filtered.to_dict("records")[0]
+
+ searcher = SearchEpc(
+ address1=str(p["address1"]),
+ postcode=str(p["postcode"]),
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ uprn=uprn
+ )
+ searcher.ordnance_survey_client.parse_classification_code(p_os_data["CLASSIFICATION_CODE"])
+
+ searcher.find_property(skip_os=True)
+
+ final_epcs.append(
+ {
+ "internal_id": internal_id,
+ **searcher.newest_epc
+ }
+ )
+
+ final_epcs = pd.DataFrame(final_epcs)
+
+ complete_epcs = pd.concat(
+ [
+ epc_data,
+ final_epcs
+ ]
+ )
+
+ # We now pull spatial data
+ # We get the spatial file list and loop through each EPC and determine which file it needs.
+ # We then just read in the files that we need and get the data, for each uprn from that file
+
+ uprn_filenames = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
+ )
+
+ uprn_lookup = {}
+ for uprn in complete_epcs["uprn"]:
+ if pd.isnull(uprn):
+ # TODO: Do something about this!
+ continue
+ filtered_df = uprn_filenames[
+ (uprn_filenames["lower"] <= int(uprn))
+ & (uprn_filenames["upper"] >= int(uprn))
+ ]
+ if filtered_df["filenames"].values[0] in uprn_lookup:
+ uprn_lookup[filtered_df["filenames"].values[0]].append(int(uprn))
+ else:
+ uprn_lookup[filtered_df["filenames"].values[0]] = [int(uprn)]
+
+ spatial_data_to_uprn = []
+ for filename, associated_uprn in tqdm(uprn_lookup.items(), total=len(uprn_lookup)):
+ # Read in the file
+ spatial_data = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key=f"spatial/{filename}"
+ )
+
+ spatial_df = spatial_data[spatial_data["UPRN"].isin(associated_uprn)]
+ spatial_data_to_uprn.append(spatial_df)
+
+ spatial_data_to_uprn = pd.concat(spatial_data_to_uprn)
+
+ # TODO: Let's store this in s3
+ save_data_to_s3(
+ data=json.dumps(spatial_data_to_uprn.to_dict("records")),
+ s3_file_name="scustomers/Stonewater/clustering/spatial_data_to_uprn.json",
+ bucket_name="retrofit-data-dev"
+ )
+
+ # We merge this spatial data onto final EPCS
+
+
+def concatenate_row(row):
+ return ', '.join(row.dropna().replace('', None).dropna().astype(str))
+
+
+def compile_data_final():
+ # Updated version:
+
+ """
+ Various data sources have been produced to create the final data source for Stonewater.
+ This function combines them
+ :return:
+ """
+ ########################################################################
+ # Read in data
+ ########################################################################
+
+ asset_list = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/Stonewater SHDF_3_0_Board Triage 22.05.24.xlsx", header=4
+ )
+
+ udprn_data = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/UDPRN updated RA Sample for 5 year programme.xlsx", header=0
+ )[["AddressId", "UDPRN"]].rename(columns={"AddressId": "Address ID"})
+ udprn_data["UDPRN"] = udprn_data["UDPRN"].astype("Int64").astype(str)
+ udprn_data["Address ID"] = udprn_data["Address ID"].astype(str)
+
+ asset_list = asset_list.merge(udprn_data, how="inner", on="Address ID")
+ asset_list = asset_list.rename(columns={"UDPRN": "udprn"})
+
+ # Read in the lookups
+ uprn_lookup_1 = pd.DataFrame(json.loads(read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup.json"
+ )))
+ uprn_lookup_1["match_type"] = "Exact"
+
+ uprn_lookup_2 = pd.DataFrame(json.loads(read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="scustomers/Stonewater/clustering/address_uprn_udprn_lookup_2.json"
+ )))
+ uprn_lookup_2 = uprn_lookup_2.rename(
+ columns={
+ "epc_address": "standardised_address",
+ "epc_postcode": "standardised_postcode"
+ }
+ )
+ uprn_lookup_2["match_type"] = "EPC"
+ uprn_lookup_2["uprn"] = np.where(
+ uprn_lookup_2["internal_id"] == 1091,
+ 83143766,
+ uprn_lookup_2["uprn"]
+ )
+
+ uprn_lookup_3 = pd.DataFrame(json.loads(read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/ideal-postcodes_pull_2.json"
+ )))
+ uprn_lookup_3["standardised_address"] = uprn_lookup_3[["line_1", "line_2", "line_3", "district", "postcode"]].apply(
+ concatenate_row, axis=1
+ )
+ uprn_lookup_3 = uprn_lookup_3[
+ ["udprn", "uprn", "standardised_address", "postcode"]
+ ].rename(columns={"postcode": "standardised_postcode"})
+ uprn_lookup_3["match_type"] = "Exact"
+
+ uprn_lookup_4_basis = pd.read_csv("manual_fix_uprns-populated.csv", index_col=False)
+ uprn_lookup_4_basis["os_option_1_uprn"] = uprn_lookup_4_basis["os_option_1_uprn"].astype(str)
+ uprn_lookup_4_basis["os_option_2_uprn"] = uprn_lookup_4_basis["os_option_2_uprn"].astype("Int64").astype(str)
+ # prepare lookup 4
+ uprn_lookup_4 = []
+ for _, x in uprn_lookup_4_basis.iterrows():
+
+ property_type = None
+ built_form = None
+ if x["option"] == 1:
+ uprn = x["os_option_1_uprn"]
+ standardised_address = x["os_option_1_address"]
+ postcode = x["os_option_1_postcode"]
+ elif x["option"] == 2:
+ uprn = x["os_option_2_uprn"]
+ standardised_address = x["os_option_2_address"]
+ postcode = x["os_option_2_address"].split(", ")[-1]
+ else:
+ uprn = x["manual_uprn"]
+ standardised_address = x["manual_address"]
+ postcode = x["manual_postcode"]
+
+ uprn_lookup_4.append(
+ {
+ "internal_id": x["internal_id"],
+ "external_address_id": x["external_address_id"],
+ "uprn": uprn,
+ "standardised_address": standardised_address,
+ "standardised_postcode": postcode,
+ "property_type": property_type,
+ "built_form": built_form
+ }
+ )
+ uprn_lookup_4 = pd.DataFrame(uprn_lookup_4)
+ uprn_lookup_4["match_type"] = "Fuzzy"
+
+ # concat
+ uprn_lookup = pd.concat([uprn_lookup_1, uprn_lookup_2])
+
+ # We now merge all of the UPRNs onto the asset list
+ assert len(uprn_lookup) + len(uprn_lookup_3) + len(uprn_lookup_4) == len(asset_list)
+
+ epc_data = json.loads(
+ read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/epc_data.json"
+ )
+ )
+ epc_data = pd.DataFrame(epc_data)
+
+ epc_data["uprn"] = np.where(
+ epc_data["internal_id"] == 1091,
+ 83143766,
+ epc_data["uprn"]
+ )
+
+ # We drop come EPCS
+ epc_data = epc_data[epc_data["internal_id"].isin(uprn_lookup_2["internal_id"].values)]
+
+ # This we can use to produce additional variables such as number of old surveys
+ # older_epc_data = json.loads(
+ # read_from_s3(
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/old_epc_data.json"
+ # )
+ # )
+ # older_epc_data = {k: v for k, v in older_epc_data.items() if k not in internal_id_epcs_to_drop}
+
+ ########################################################################
+ # Prepare asset list
+ ########################################################################
+
+ # Keep just the columns we're interested in
+ asset_list = asset_list[
+ [
+ "Osm. ID",
+ "Org. ref.",
+ "Postcode",
+ "House no",
+ "Name",
+ "Address line 2",
+ "City/Town",
+ "County",
+ "Address ID", # This is not uprn
+ "udprn",
+ "Owning body"
+ ]
+ ].rename(
+ columns={
+ "Osm. ID": "internal_id",
+ "Org. ref.": "customer_asset_id",
+ "Postcode": "postcode",
+ "House no": "house_number",
+ "Name": "address1",
+ "Address line 2": "address2",
+ "City/Town": "city_town",
+ "County": "county",
+ "Address ID": "external_address_id",
+ "Owning body": "owner"
+ }
+ )
+
+ # Create full address
+ asset_list["full_address"] = np.where(
+ ~pd.isnull(asset_list["address2"]),
+ (
+ asset_list["address1"] + ", " +
+ asset_list["address2"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ ),
+ asset_list["address1"] + ", " +
+ asset_list["city_town"].str.title() + ", " +
+ # asset_list["county"] + ", " +
+ asset_list["postcode"]
+ )
+
+ if pd.isnull(asset_list["full_address"]).sum():
+ raise ValueError("Missing full addresses")
+
+ # Final preps of lookups
+ uprn_lookup_3["udprn"] = uprn_lookup_3["udprn"].astype(str)
+ uprn_lookup_3 = uprn_lookup_3.merge(
+ asset_list[["udprn", "internal_id", "external_address_id"]], how="left", on="udprn"
+ )
+ uprn_lookup = pd.concat([
+ uprn_lookup,
+ uprn_lookup_3,
+ uprn_lookup_4
+ ])
+ uprn_lookup["external_address_id"] = uprn_lookup["external_address_id"].astype(str)
+
+ asset_list = asset_list.merge(
+ uprn_lookup.drop(columns=["udprn"]),
+ how="inner",
+ on=["internal_id", "external_address_id"]
+ )
+
+ # Store locally
+ # asset_list.to_excel("Stonewater asset list with uprn.xlsx")
+
+ # We take just domestic properties
+
+ # This is the first ordnance survey data pull
+ os_most_relevant_1 = []
+ os_all_1 = {}
+ for i in tqdm(["1", "2", "3"]):
+ most_relevant_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_most_relevant_{i}.json"
+ )
+ os_most_relevant_1.extend(json.loads(most_relevant_segment))
+ os_all_segment = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name=f"customers/Stonewater/clustering/os_all_{i}.json"
+ )
+ os_all_1 = {**os_all_1, **json.loads(os_all_segment)}
+
+ os_most_relevant_1 = pd.DataFrame(os_most_relevant_1)
+
+ # This is the second ordnance survey data pull
+ os_most_relevant_2 = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/problematic_os.json"
+ )
+ os_most_relevant_2 = json.loads(os_most_relevant_2)
+ os_most_relevant_2 = pd.DataFrame(os_most_relevant_2)
+
+ os_all_2 = read_from_s3(
+ bucket_name="retrofit-data-dev",
+ s3_file_name="customers/Stonewater/clustering/problematic_os_all.json"
+ )
+ os_all_2 = json.loads(os_all_2)
+
+ needs_epc_data = asset_list[~asset_list["internal_id"].isin(epc_data["internal_id"])]
+
+ os_1_ids = os_most_relevant_1["internal_id"].values
+ os_2_ids = os_most_relevant_2["internal_id"].values
+
+ epc_data_batch_2 = []
+ older_epcs_batch_2 = {}
+ for _, property in tqdm(needs_epc_data.iterrows(), total=len(needs_epc_data)):
+ if pd.isnull(property["uprn"]):
+ continue
+ searcher = SearchEpc(
+ address1=", ".join(property["standardised_address"].split(", ")[:-1]),
+ postcode=property["standardised_postcode"],
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ full_address=property["standardised_address"],
+ uprn=property["uprn"]
+ )
+ searcher.find_property(skip_os=True)
+
+ if searcher.newest_epc is None and property["match_type"] == "Exact":
+ # Estimate!
+ # Get the OS data
+ p_os_df = pd.DataFrame()
+ if property["internal_id"] in os_1_ids:
+ p_os_df = pd.DataFrame(
+ [x["DPA"] if "DPA" in x else x["LPI"] for x in os_all_1[str(property["internal_id"])]]
+ )
+ p_os_df = p_os_df[p_os_df["UPRN"].astype(str) == property["uprn"]]
+
+ if p_os_df.empty:
+ p_os_df = pd.DataFrame(
+ [x["DPA"] if "DPA" in x else x["LPI"] for x in os_all_2[str(property["internal_id"])]]
+ )
+ p_os_df = p_os_df[p_os_df["UPRN"] == property["uprn"]]
+
+ if not p_os_df.empty:
+ searcher.ordnance_survey_client.parse_classification_code(p_os_df["CLASSIFICATION_CODE"].values[0])
+ else:
+ searcher.ordnance_survey_client.property_type = ""
+ # Now we estimate
+ searcher.newest_epc = searcher.estimate_epc(
+ property_type=searcher.ordnance_survey_client.property_type,
+ built_form=searcher.ordnance_survey_client.built_form,
+ lmks_to_drop=None,
+ exclude_old=True
+ )
+
+ elif searcher.newest_epc is None and property["match_type"] == "Fuzzy":
+
+ if "flat" in property["standardised_address"].lower():
+ searcher.newest_epc = searcher.estimate_epc(
+ property_type="Flat",
+ built_form=None,
+ lmks_to_drop=None,
+ exclude_old=True
+ )
+ else:
+ searcher.newest_epc = searcher.estimate_epc(
+ property_type="House",
+ built_form=None,
+ lmks_to_drop=None,
+ exclude_old=True
+ )
+
+ epc_data_batch_2.append(
+ {
+ "internal_id": property["internal_id"],
+ **searcher.newest_epc
+ }
+ )
+
+ if searcher.older_epcs is not None:
+ older_epcs_batch_2[property["internal_id"]] = searcher.older_epcs
+ # Store in S3
+ # TODO - read in instead of running
+ save_pickle_to_s3(
+ data=epc_data_batch_2,
+ s3_file_name="customers/Stonewater/clustering/epc_data_batch_2.pkl",
+ bucket_name="retrofit-data-dev"
+ )
+
+ save_pickle_to_s3(
+ data=older_epcs_batch_2,
+ s3_file_name="customers/Stonewater/clustering/older_epcs_batch_2.pkl",
+ bucket_name="retrofit-data-dev"
+ )
+
+ epc_data_batch_2 = pd.DataFrame(epc_data_batch_2)
+ complete_epcs = pd.concat([epc_data, epc_data_batch_2])
+
+ # We now prepare the final data for clustering
+ uprn_filenames = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
+ )
+
+ uprn_map = {}
+ for uprn in complete_epcs["uprn"]:
+ filtered_df = uprn_filenames[
+ (uprn_filenames["lower"] <= int(uprn))
+ & (uprn_filenames["upper"] >= int(uprn))
+ ]
+ if filtered_df["filenames"].values[0] in uprn_map:
+ uprn_map[filtered_df["filenames"].values[0]].append(int(uprn))
+ else:
+ uprn_map[filtered_df["filenames"].values[0]] = [int(uprn)]
+
+ spatial_data_to_uprn = []
+ for filename, associated_uprn in tqdm(uprn_map.items(), total=len(uprn_map)):
+ # Read in the file
+ spatial_data = read_dataframe_from_s3_parquet(
+ bucket_name="retrofit-data-dev", file_key=f"spatial/{filename}"
+ )
+
+ spatial_df = spatial_data[spatial_data["UPRN"].isin(associated_uprn)]
+ spatial_data_to_uprn.append(spatial_df)
+
+ # TODO: Let's store this in s3
+ # save_pickle_to_s3(
+ # data=spatial_data_to_uprn,
+ # s3_file_name="scustomers/Stonewater/clustering/spatial_data_to_uprn.pkl",
+ # bucket_name="retrofit-data-dev"
+ # )
+
+ spatial_data_to_uprn = pd.concat(spatial_data_to_uprn)
+
+ spatial_data_to_uprn = spatial_data_to_uprn.drop(
+ columns=["partition", "filename"]
+ ).rename(columns={"UPRN": "uprn"})
+ spatial_data_to_uprn["uprn"] = spatial_data_to_uprn["uprn"].astype(str)
+
+ property_attributes = complete_epcs.merge(
+ spatial_data_to_uprn,
+ how="inner",
+ on="uprn"
+ )
+
+ property_attributes = property_attributes.merge(
+ asset_list[["internal_id", "owner", "match_type"]], how="left", on="internal_id"
+ )
+
+ # TODO: Add on data from the asset list such as ownership
+
+ # We drop the columns we don't care about for clustering
+ property_attributes = property_attributes.drop(
+ columns=[
+ "address",
+ "uprn-source",
+ "heating-cost-potential",
+ "hot-water-cost-potential",
+ "potential-energy-rating",
+ "environment-impact-potential",
+ "address3",
+ "local-authority-label",
+ "sheating-energy-eff",
+ "local-authority-label",
+ "county",
+ "postcode",
+ "constituency",
+ "co2-emissions-potential",
+ "energy-consumption-potential",
+ "local-authority",
+ "inspection-date",
+ "address1",
+ "constituency-label",
+ "building-reference-number",
+ "floor-energy-eff",
+ "address2",
+ "posttown",
+ "floor-env-eff",
+ "sheating-env-eff",
+ "lighting-cost-potential",
+ "main-heating-controls",
+ "transaction-type",
+ "uprn",
+ "lodgement-date",
+ "lmk-key",
+ "wind-turbine-count",
+ "tenure",
+ "potential-energy-efficiency",
+ "glazed-area"
+ ]
+ )
+
+ # Fields to transform: lodgement-datetime
+ property_attributes["days_since_last_epc"] = (
+ datetime.now() - pd.to_datetime(property_attributes["lodgement-datetime"], errors="coerce")
+ ).dt.days
+
+ property_attributes = property_attributes.drop(columns=["lodgement-datetime"])
+
+ # Up to:
+ # Round averages to nearest integer
+ fill_with_average = [
+ "low-energy-fixed-light-count",
+ "floor-height",
+ "heating-cost-current",
+ "fixed-lighting-outlets-count",
+ "hot-water-cost-current",
+ "number-heated-rooms",
+ "co2-emiss-curr-per-floor-area",
+ "total-floor-area",
+ "environment-impact-current",
+ "co2-emissions-current",
+ "number-habitable-rooms",
+ "energy-consumption-current",
+ 'lighting-cost-current',
+ "low-energy-lighting",
+ ]
+
+ fill_with_mode = [
+ "multi-glaze-proportion",
+ "extension-count",
+ ]
+
+ fill_with_zero = [
+ "unheated-corridor-length",
+ "number-open-fireplaces",
+ "photo-supply",
+ ]
+
+ fill_with_categorical = {
+ "construction-age-band": "unknown",
+ "mainheat-energy-eff": "N/A",
+ "windows-env-eff": "N/A",
+ "lighting-energy-eff": "N/A",
+ "energy-tariff": 'NO DATA!',
+ "mechanical-ventilation": 'NO DATA!',
+ "solar-water-heating-flag": "N",
+ "mains-gas-flag": "N",
+ "heat-loss-corridor": "unknown",
+ "flat-storey-count": "Not a flat",
+ "roof-energy-eff": "N/A",
+ "hot-water-env-eff": "N/A",
+ "mainheatc-energy-eff": "N/A",
+ "main-fuel": 'NO DATA!',
+ "lighting-env-eff": "N/A",
+ "windows-energy-eff": "N/A",
+ "roof-env-eff": "N/A",
+ "walls-env-eff": "N/A",
+ "mainheat-env-eff": "N/A",
+ "flat-top-storey": "N",
+ "mainheatc-env-eff": "N",
+ "floor-level": "NODATA!",
+ "hot-water-energy-eff": "N/A",
+ "glazed-type": "unknown"
+ }
+
+ # Consolidation columns to single value
+ consolidation_columns = {
+ "glazed-type": {"from": ['', 'NO DATA!', 'not defined', 'INVALID!'], "to": "unknown"},
+ "mechanical-ventilation": {"from": ['', 'NO DATA!', 'not defined', 'INVALID!'], "to": "unknown"},
+ "solar-water-heating-flag": {"from": [''], "to": "N"},
+ "mains-gas-flag": {"from": [''], "to": "N"},
+ "heat-loss-corridor": {"from": ['NO DATA!', ''], "to": "N"},
+ "flat-top-storey": {"from": [''], "to": "N"},
+ "floor-level": {"from": [""], "to": "NODATA!"}
+ }
+
+ # Perform the cleaning
+ for col in fill_with_average:
+ property_attributes[col] = property_attributes[col].replace('', None)
+ avg_val = np.mean([float(x) for x in property_attributes[col].values if x not in [None, "", np.nan]])
+ if pd.isnull(avg_val):
+ raise Exception("something went wrong")
+ property_attributes[col] = property_attributes[col].fillna(round(avg_val))
+ property_attributes[col] = property_attributes[col].astype(float)
+
+ for c in fill_with_zero:
+ property_attributes[c] = property_attributes[c].replace('', 0)
+ property_attributes[c] = property_attributes[c].fillna(0)
+ property_attributes[c] = property_attributes[c].astype(float)
+
+ for col in fill_with_mode:
+ property_attributes[col] = property_attributes[col].replace('', None)
+ mode_val = stats.mode([float(x) for x in property_attributes[col].values if x not in [None, "", np.nan]])[0]
+ if pd.isnull(mode_val):
+ raise Exception("something went wrong")
+ property_attributes[col] = property_attributes[col].fillna(mode_val)
+ property_attributes[col] = property_attributes[col].astype(float)
+
+ for c, fill_val in fill_with_categorical.items():
+ property_attributes[c] = property_attributes[c].replace('', fill_val)
+ property_attributes[c] = property_attributes[c].fillna(fill_val)
+
+ # Finally, consolidate
+ for c, consolidate_config in consolidation_columns.items():
+ for v in consolidate_config["from"]:
+ property_attributes[c] = property_attributes[c].replace(v, consolidate_config["to"])
+
+ property_attributes["estimated"] = property_attributes["estimated"].fillna(False)
+ property_attributes["conservation_status"] = property_attributes["conservation_status"].fillna(False)
+ property_attributes["days_since_last_epc"] = property_attributes["days_since_last_epc"].fillna(
+ property_attributes["days_since_last_epc"].mean()
+ )
+
+ missings = pd.isnull(property_attributes).sum()
+ missings = missings[missings > 0]
+
+ # Save this
+ # save_pickle_to_s3(
+ # data=property_attributes,
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/clustering_dataframe.pkl"
+ # )
+
+ # from utils.s3 import read_pickle_from_s3
+ # data = read_pickle_from_s3(
+ # bucket_name="retrofit-data-dev",
+ # s3_file_name="customers/Stonewater/clustering/clustering_dataframe.pkl"
+ # )
+
+ # CLUSTERING!!
+
+ # from sklearn.cluster import KMeans
+ # from sklearn.preprocessing import OneHotEncoder
+ # from scipy.spatial.distance import cdist
+ #
+ # property_attributes.set_index('internal_id', inplace=True)
+ #
+ # # Step 1: Prepare the data
+ # # Identify categorical columns (you might need to adjust this)
+ # categorical_cols = property_attributes.select_dtypes(include=['object', 'category']).columns.tolist()
+ # for col in categorical_cols:
+ # property_attributes[col] = property_attributes[col].astype(str)
+ #
+ # # Applying OneHotEncoder
+ # encoder = OneHotEncoder(sparse=False)
+ # encoded_cats = encoder.fit_transform(property_attributes[categorical_cols])
+ #
+ # # Creating a new DataFrame with encoded categorical data and original numerical data
+ # numerical_data = property_attributes.select_dtypes(include=[np.number])
+ # data_for_clustering = pd.concat([numerical_data, pd.DataFrame(encoded_cats, index=numerical_data.index)], axis=1)
+ #
+ # # Convert all column names to strings to satisfy KMeans requirements
+ # data_for_clustering.columns = data_for_clustering.columns.astype(str)
+ #
+ # # Step 2: K-Means Clustering
+ # k = 450 # number of clusters
+ # kmeans = KMeans(n_clusters=k, random_state=0)
+ # property_attributes['cluster'] = kmeans.fit_predict(data_for_clustering)
+ #
+ # # Extracting centroids
+ # centroids = kmeans.cluster_centers_
+ #
+ # # Step 3: Assign clusters and rank rows
+ # # Calculating distances from each point to its cluster's centroid
+ # distances = cdist(data_for_clustering, centroids, 'euclidean')
+ # min_distances = distances.min(axis=1)
+ # property_attributes['distance_to_centroid'] = min_distances
+ #
+ # # Ranking rows by distance within each cluster
+ # property_attributes['rank'] = property_attributes.groupby('cluster')['distance_to_centroid'].rank(method='first')
+ #
+ # # Sorting to verify
+ # property_attributes.sort_values(by=['cluster', 'rank'], inplace=True)
+ #
+ # # Optional: Displaying the dataframe
+ # print(property_attributes.head())
+
+ from sklearn.cluster import KMeans
+ from sklearn.preprocessing import StandardScaler, OneHotEncoder
+ from sklearn.compose import ColumnTransformer
+ from sklearn.pipeline import Pipeline
+ from scipy.spatial.distance import cdist
+ id_column = 'internal_id'
+ property_attributes.set_index(id_column, inplace=True)
+
+ # Define the preprocessing for numerical and categorical features
+ numerical_features = property_attributes.select_dtypes(include=['int64', 'float64']).columns.tolist()
+ categorical_features = property_attributes.select_dtypes(include=['object', 'category']).columns.tolist()
+
+ for col in categorical_features:
+ property_attributes[col] = property_attributes[col].astype(str)
+
+ preprocessor = ColumnTransformer(
+ transformers=[
+ ('num', StandardScaler(), numerical_features),
+ ('cat', OneHotEncoder(), categorical_features)
+ ]
+ )
+
+ pipeline = Pipeline(steps=[('preprocessor', preprocessor),
+ ('kmeans', KMeans(n_clusters=450, random_state=0))])
+
+ # Fit the pipeline to the data
+ pipeline.fit(property_attributes)
+
+ # Transform the data using the fitted pipeline
+ processed_data = pipeline.named_steps['preprocessor'].transform(property_attributes)
+
+ # Get cluster labels
+ property_attributes['cluster'] = pipeline.named_steps['kmeans'].labels_
+
+ # Get centroids (already in the same transformed space)
+ centroids = pipeline.named_steps['kmeans'].cluster_centers_
+
+ processed_data = processed_data.toarray()
+
+ # Calculate distances from each point to the centroid of its cluster
+ distances_to_centroids = [
+ cdist(processed_data[i].reshape(1, -1), centroids[label].reshape(1, -1)).flatten()[0]
+ for i, label in enumerate(property_attributes['cluster'])
+ ]
+
+ property_attributes['distance_to_centroid'] = distances_to_centroids
+
+ for cluster_id in property_attributes['cluster'].unique():
+ cluster_data = property_attributes[property_attributes['cluster'] == cluster_id]
+ min_distance = cluster_data['distance_to_centroid'].min()
+ print(f"Cluster {cluster_id} minimum distance to centroid: {min_distance}")
+ if min_distance != 0:
+ print(f"No point with zero distance found in cluster {cluster_id}")
+
+ # Ranking rows by distance within each cluster
+ property_attributes['rank'] = property_attributes.groupby('cluster')['distance_to_centroid'].rank(
+ method='first')
+
+ # Sorting to verify
+ property_attributes.sort_values(by=['cluster', 'rank'], inplace=True)
+
+ ################################################
+ # Prepare outputs!!!!
+ ################################################
+ property_attributes.reset_index(inplace=True)
+ property_attributes["archetype_representative"] = property_attributes["rank"] == 1
+
+ asset_list_with_archetypes = asset_list.merge(
+ property_attributes[["internal_id", "cluster", "archetype_representative", "rank"]], how="left",
+ on="internal_id"
+ )
+
+ asset_list_with_archetypes["cluster"] = asset_list_with_archetypes["cluster"].fillna(-999)
+ asset_list_with_archetypes["cluster"] = asset_list_with_archetypes["cluster"].astype(int).astype(str)
+ asset_list_with_archetypes["cluster"] = asset_list_with_archetypes["cluster"].replace("-999", "NO ARCHETYPE")
+
+ asset_list_with_archetypes["rank"] = asset_list_with_archetypes["rank"].fillna(-999)
+ asset_list_with_archetypes["rank"] = asset_list_with_archetypes["rank"].astype(int).astype(str)
+ asset_list_with_archetypes["rank"] = asset_list_with_archetypes["rank"].replace("-999", "NO ARCHETYPE")
+
+ asset_list_with_archetypes["archetype_representative"] = asset_list_with_archetypes[
+ "archetype_representative"].fillna(False)
+
+ asset_list_with_archetypes.to_csv("Stonewater asset list with archetypes.csv", index=False)
+
+ stonewater_uprn_lookup = asset_list_with_archetypes[
+ ["external_address_id", "udprn", "uprn", "match_type", "standardised_address", "standardised_postcode"]
+ ]
+
+ stonewater_uprn_lookup.to_excel("Stonewater uprn lookup table.xlsx")
+
+ ################################################
+ # Agglomertive Clustering
+ ################################################
+
+ # from sklearn.cluster import KMeans, AgglomerativeClustering
+ # from sklearn.preprocessing import StandardScaler, OneHotEncoder
+ # from sklearn.compose import ColumnTransformer
+ # from sklearn.pipeline import Pipeline
+ # from scipy.spatial.distance import cdist
+ # import numpy as np
+ # from collections import Counter
+ #
+ # id_column = 'internal_id'
+ # property_attributes.set_index(id_column, inplace=True)
+ #
+ # # Define the preprocessing for numerical and categorical features
+ # numerical_features = property_attributes.select_dtypes(include=['int64', 'float64']).columns.tolist()
+ # categorical_features = property_attributes.select_dtypes(include=['object', 'category']).columns.tolist()
+ #
+ # for col in categorical_features:
+ # property_attributes[col] = property_attributes[col].astype(str)
+ #
+ # preprocessor = ColumnTransformer(
+ # transformers=[
+ # ('num', StandardScaler(), numerical_features),
+ # ('cat', OneHotEncoder(sparse_output=False), categorical_features)
+ # ]
+ # )
+ #
+ # # Function to perform clustering and merge small clusters
+ # def cluster_with_min_size(data, preprocessor, n_clusters=10, min_size=5):
+ # while True:
+ # # Preprocess the data
+ # processed_data = preprocessor.fit_transform(data)
+ #
+ # # Initial clustering
+ # clustering = AgglomerativeClustering(n_clusters=n_clusters)
+ # labels = clustering.fit_predict(processed_data)
+ #
+ # # Check cluster sizes
+ # cluster_counts = Counter(labels)
+ #
+ # # Find clusters smaller than min_size
+ # small_clusters = {cluster for cluster, count in cluster_counts.items() if count < min_size}
+ #
+ # if not small_clusters:
+ # break
+ #
+ # # Merge small clusters
+ # for cluster in small_clusters:
+ # # Find the nearest cluster to merge with
+ # cluster_data = processed_data[labels == cluster]
+ # other_clusters = [i for i in range(n_clusters) if i not in small_clusters]
+ # other_cluster_data = [processed_data[labels == i] for i in other_clusters]
+ # other_centroids = np.vstack([data.mean(axis=0) for data in other_cluster_data])
+ #
+ # distances = cdist(cluster_data, other_centroids).mean(axis=0)
+ # closest_cluster = other_clusters[np.argmin(distances)]
+ #
+ # labels[labels == cluster] = closest_cluster
+ #
+ # n_clusters -= len(small_clusters)
+ #
+ # return labels
+ #
+ # # Perform clustering with minimum size constraint
+ # n_clusters = 10
+ # min_size = 5
+ # property_attributes['cluster'] = cluster_with_min_size(property_attributes, preprocessor, n_clusters, min_size)
+ #
+ # # Filter out empty clusters
+ # valid_clusters = property_attributes['cluster'].unique()
+ #
+ # # Get centroids for the resulting clusters
+ # processed_data = preprocessor.transform(property_attributes.drop(columns=["cluster"]))
+ # centroids = np.vstack([processed_data[property_attributes['cluster'] == i].mean(axis=0) for i in valid_clusters])
+ #
+ # # Calculate distances from each point to the centroid of its cluster
+ # distances_to_centroids = [
+ # cdist(processed_data[i].reshape(1, -1),
+ # centroids[valid_clusters.tolist().index(label)].reshape(1, -1)).flatten()[0]
+ # for i, label in enumerate(property_attributes['cluster'])
+ # ]
+ #
+ # property_attributes['distance_to_centroid'] = distances_to_centroids
+ #
+ # # Verify that at least one point in each cluster has zero distance to the centroid
+ # for cluster_id in valid_clusters:
+ # cluster_data = property_attributes[property_attributes['cluster'] == cluster_id]
+ # min_distance = cluster_data['distance_to_centroid'].min()
+ # print(f"Cluster {cluster_id} minimum distance to centroid: {min_distance}")
+ # if min_distance != 0:
+ # print(f"No point with zero distance found in cluster {cluster_id}")
+ #
+ # # Rank the distances within each cluster
+ # property_attributes['rank_within_cluster'] = property_attributes.groupby('cluster')['distance_to_centroid'] \
+ # .rank(method='first')
+ #
+ # # Reset index to get 'internal_id' back
+ # property_attributes.reset_index(inplace=True)
+ #
+ # # Display the DataFrame
+ # print(property_attributes)
+
+
+def pull_ideal_postcodes(missing_uprn_with_udprn):
+ api_key = "" # Log into the platform the get the API key: https://account.ideal-postcodes.co.uk/
+ import requests
+ import time
+ completed_id = 0
+
+ uprn_to_udprn = []
+ for row_index, data in tqdm(missing_uprn_with_udprn.iterrows(), total=len(missing_uprn_with_udprn)):
+ if row_index < completed_id:
+ continue
+ time.sleep(0.5)
+
+ # Call the API
+ udprn = data["udprn"]
+
+ url = f"https://api.ideal-postcodes.co.uk/v1/udprn/{udprn}?api_key={api_key}"
+
+ payload = {
+ "api_key": api_key
+ }
+ headers = {
+ 'Accept': 'application/json'
+ }
+
+ response = requests.request("GET", url, headers=headers, data=payload)
+ if response.status_code != 200:
+ raise ValueError("API call dead")
+
+ result = response.json()
+ uprn_to_udprn.append(
+ result["result"]
+ )
+ completed_id += 1
diff --git a/etl/customers/unitas/20_may_2024_data_pull.py b/etl/customers/unitas/20_may_2024_data_pull.py
new file mode 100644
index 00000000..21686ef4
--- /dev/null
+++ b/etl/customers/unitas/20_may_2024_data_pull.py
@@ -0,0 +1,148 @@
+import os
+
+import pandas as pd
+from tqdm import tqdm
+
+from dotenv import load_dotenv
+from backend.SearchEpc import SearchEpc
+from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
+
+from recommendations.recommendation_utils import (
+ estimate_perimeter,
+ estimate_external_wall_area,
+ estimate_number_of_floors
+)
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+ """
+ This app is EPC pulling data for some properties owned by Unitas
+ :return:
+ """
+ # asset_list = read_excel_from_s3(
+ # bucket_name="retrofit-datalake-dev",
+ # file_key="customers/guiness/TGP CW Properties PV.xlsx",
+ # header_row=0
+ # )
+ asset_list = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/UNITAS BUNGALOWS - EPC DATA PULL.xlsx", header=0
+ )
+
+ epc_data = []
+ for _, home in tqdm(asset_list.iterrows(), total=len(asset_list)):
+
+ searcher = SearchEpc(
+ address1=str(home["Address Line 1"]),
+ postcode=home["Post Code"],
+ uprn=home["Property Reference"],
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=True
+ )
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+ if searcher.newest_epc is None:
+ continue
+
+ epc = {
+ "asset_list_address": home["Address Line 1"],
+ "asset_list_postcode": home["Post Code"],
+ **searcher.newest_epc.copy()
+ }
+
+ epc_data.append(epc)
+
+ epc_df = pd.DataFrame(epc_data)
+
+ # Retrieve just the data we need
+ epc_df = epc_df[
+ [
+ "asset_list_address",
+ "uprn",
+ "property-type",
+ "built-form",
+ "inspection-date",
+ "current-energy-rating",
+ "current-energy-efficiency",
+ "roof-description",
+ "walls-description",
+ "transaction-type",
+ # New fields needed
+ "secondheat-description",
+ "total-floor-area",
+ "construction-age-band",
+ "floor-height",
+ "number-habitable-rooms",
+ "mainheat-description"
+ ]
+ ]
+
+ asset_list = asset_list.merge(
+ epc_df,
+ how="left",
+ left_on=["Address Line 1"],
+ right_on=["asset_list_address"]
+ )
+
+ asset_list = asset_list.drop(columns=["asset_list_address"])
+
+ # Rename the columns
+ asset_list = asset_list.rename(columns={
+ "inspection-date": "Date of last EPC",
+ "current-energy-efficiency": "SAP score on register",
+ "current-energy-rating": "EPC rating on register",
+ "property-type": "EPC Property Type",
+ "built-form": "Archetype",
+ "total-floor-area": "Property Floor Area",
+ "construction-age-band": "Property Age Band",
+ "floor-height": "Property Floor Height",
+ "number-habitable-rooms": "Number of Habitable Rooms",
+ "walls-description": "Wall Construction",
+ "roof-description": "Roof Construction",
+ "mainheat-description": "Heating Type",
+ "secondheat-description": "Secondary Heating",
+ "transaction-type": "Reason for last EPC"
+ })
+
+ asset_list["Estimated Number of Floors"] = asset_list.apply(
+ lambda x: estimate_number_of_floors(property_type=x["EPC Property Type"]) if not pd.isnull(
+ x["EPC Property Type"]) else None,
+ axis=1
+ )
+
+ asset_list["Property Floor Area"] = asset_list["Property Floor Area"].astype(float)
+ asset_list["Number of Habitable Rooms"] = asset_list["Number of Habitable Rooms"].astype(float)
+
+ asset_list["Estimated Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_perimeter(
+ floor_area=x["Property Floor Area"] / x["Estimated Number of Floors"],
+ num_rooms=x["Number of Habitable Rooms"] / x["Estimated Number of Floors"],
+ ) if not pd.isnull(x["uprn"]) else None, axis=1
+ )
+
+ asset_list["Estimated Heat Loss Perimeter (m)"] = asset_list.apply(
+ lambda x: estimate_external_wall_area(
+ num_floors=x["Estimated Number of Floors"],
+ floor_height=float(x["Property Floor Height"]) if x["Property Floor Height"] else 2.5,
+ perimeter=x["Estimated Perimeter (m)"],
+ built_form=x["Archetype"]
+ ) if not pd.isnull(x["uprn"]) else None,
+ axis=1
+ )
+
+ asset_list["Roof Insulation Thickness"] = asset_list.apply(
+ lambda x: RoofAttributes(description=x["Roof Construction"]).process()["insulation_thickness"] if not pd.isnull(
+ x["uprn"]) else None,
+ axis=1
+ )
+
+ # Store as an excel
+ filename = "UNITAS BUNGALOWS - EPC DATA PULL - May 30tg 2024.xlsx"
+ asset_list.to_excel(filename, index=False)
diff --git a/etl/customers/unitas/Audit_check.py b/etl/customers/unitas/Audit_check.py
new file mode 100644
index 00000000..ad5361d4
--- /dev/null
+++ b/etl/customers/unitas/Audit_check.py
@@ -0,0 +1,182 @@
+import pandas as pd
+import os
+
+from tqdm import tqdm
+
+from dotenv import load_dotenv
+from backend.SearchEpc import SearchEpc
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+ # Read in rolling master
+ master = pd.read_csv(
+ "/Users/khalimconn-kowlessar/Downloads/UNITAS ( STOKE) MASTER ROLLING SHEET UPDATED 16.5.24 K - PASSWORD "
+ "PROTECTED/ECO 4 - PHASE 1-Table 1.csv"
+ )
+
+ master = master[master["INSTALLER"] == "SCIS"]
+
+ master = master[
+ [
+ 'UPRN', 'NO.', 'Street / Block Name', 'Town/Area', 'Post Code', 'Surveyor', "SUBMISSION DATE"
+ ]
+ ]
+
+ master = master[~pd.isnull(master["UPRN"])]
+ master = master[master["UPRN"] != "NOT ON ASSET LIST"]
+
+ heights = []
+ eco_assessment_epcs = []
+ for _, row in tqdm(master.iterrows(), total=len(master)):
+ searcher = SearchEpc(
+ address1="",
+ postcode="",
+ uprn=str(int(row["UPRN"])),
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=False,
+ )
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+ if searcher.newest_epc is None:
+ continue
+
+ # Look for eco assessment epcs
+ eco_epc = [x for x in [searcher.newest_epc] + searcher.older_epcs if x['transaction-type'] == 'ECO assessment']
+ # Take the newest
+ eco_epc = sorted(eco_epc, key=lambda x: x['inspection-date'], reverse=True)
+ if eco_epc:
+ eco_assessment_epcs.append(eco_epc[0])
+
+ height = {
+ "uprn": row["UPRN"],
+ "floor_height": searcher.newest_epc["floor-height"]
+ }
+ heights.append(height)
+
+ heights_df = pd.DataFrame(heights)
+
+ eco_assessment_epcs_df = pd.DataFrame(eco_assessment_epcs)
+
+ merged_heights_df = master.merge(heights_df, left_on="UPRN", right_on="uprn", how="inner")
+ merged_heights_df = merged_heights_df[merged_heights_df["floor_height"] != ""]
+ merged_eco_assessment_epcs_df = master.merge(eco_assessment_epcs_df[["uprn", "floor-height"]], left_on="UPRN",
+ right_on="uprn", how="inner")
+ merged_eco_assessment_epcs_df["floor-height"] = merged_eco_assessment_epcs_df["floor-height"].astype(float)
+
+ merged_eco_assessment_epcs_df.groupby("Surveyor")["floor-height"].mean()
+
+ # Store
+ merged_heights_df.to_csv("Unitas 2022 heights - based on newest EPC.csv", index=False)
+ merged_eco_assessment_epcs_df.to_csv("Unitas 2022 heights - based on ECO assessment EPC.csv", index=False)
+
+ # Read in a diferent sheet
+ master = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Downloads/COMMUNITY HOUSING SURVEYS WITH A POST EPC.xlsx"
+ )
+
+ master["row_number"] = master.index
+
+ heights = []
+ eco_assessment_epcs = []
+ expected_pre = []
+ expected_post = []
+ biggest_floor_height = []
+ for _, row in tqdm(master.iterrows(), total=len(master)):
+
+ full_address = ", ".join([
+ str(row["NO."]), row["Street / Block Name"], row["Town/Area"], row["Post Code"]
+ ])
+ searcher = SearchEpc(
+ address1=str(row["NO."]),
+ postcode=str(row["Post Code"]),
+ auth_token=EPC_AUTH_TOKEN,
+ os_api_key="",
+ property_type=None,
+ fast=False,
+ full_address=full_address
+ )
+
+ # Force the skipping of estimating the EPC
+ searcher.ordnance_survey_client.property_type = None
+ searcher.ordnance_survey_client.built_form = None
+
+ searcher.find_property(skip_os=True)
+
+ if searcher.newest_epc is None:
+ continue
+
+ all_epcs = [searcher.newest_epc] + searcher.older_epcs
+ # Search for SAP 54s
+ sap_54s = [x for x in all_epcs if x["current-energy-efficiency"] == "54"]
+ sap_69s = [x for x in all_epcs if x["current-energy-efficiency"] == "69"]
+ heights = [float(x["floor-height"]) for x in all_epcs if x["floor-height"] != ""]
+
+ # Look for eco assessment epcs
+ eco_epc = [x for x in [searcher.newest_epc] + searcher.older_epcs if x['transaction-type'] == 'ECO assessment']
+ # Take the newest
+ eco_epc = sorted(eco_epc, key=lambda x: x['inspection-date'], reverse=True)
+ if eco_epc:
+ eco_assessment_epcs.append(
+ {
+ "row_number": row["row_number"],
+ **eco_epc[0]
+ }
+ )
+
+ if heights:
+ floor_height_max = max(heights)
+ biggest_floor_height.append(
+ {
+ "row_number": row["row_number"],
+ "floor_height": floor_height_max
+ }
+ )
+
+ if sap_54s:
+ expected_pre.append(
+ {
+ "row_number": row["row_number"],
+ **sap_54s[0]
+ }
+ )
+
+ if sap_69s:
+ expected_post.append(
+ {
+ "row_number": row["row_number"],
+ **sap_69s[0]
+ }
+ )
+
+ expected_pre_df = pd.DataFrame(expected_pre)
+ expected_post_df = pd.DataFrame(expected_post)
+
+ heights_df = pd.DataFrame(biggest_floor_height)
+ eco_assessment_epcs_df = pd.DataFrame(eco_assessment_epcs)
+
+ merged_heights_df = master.merge(heights_df, on="row_number", how="inner")
+ merged_heights_df = merged_heights_df[merged_heights_df["floor_height"] != ""]
+
+ merged_eco_assessment_epcs_df = master.merge(
+ eco_assessment_epcs_df[["row_number", "floor-height"]], on="row_number", how="inner"
+ )
+ merged_eco_assessment_epcs_df["floor-height"] = merged_eco_assessment_epcs_df["floor-height"].astype(float)
+
+ merged_eco_assessment_epcs_df.groupby("Surveyor")["floor-height"].mean()
+
+ # Check average floor height for social housing properties with ECO assessment EPCs in Birmingham
+ sample = pd.read_csv("local_data/all-domestic-certificates/domestic-E08000025-Birmingham/certificates.csv")
+ sample = sample[sample["TRANSACTION_TYPE"] == "ECO assessment"]
+ sample = sample[sample["TENURE"].isin(["rental (social)", "Rented (social)"])]
+ sample["FLOOR_HEIGHT"] = sample["FLOOR_HEIGHT"].astype(float)
+ sample["FLOOR_HEIGHT"].mean()
+
+ sample[pd.to_datetime(sample["LODGEMENT_DATE"]) >= "2022-01-01"]["FLOOR_HEIGHT"].mean()
diff --git a/etl/epc/Dataset.py b/etl/epc/Dataset.py
index ee3e357c..83a85b78 100644
--- a/etl/epc/Dataset.py
+++ b/etl/epc/Dataset.py
@@ -203,11 +203,11 @@ class TrainingDataset(BaseDataset):
common_cols = [[col + "_starting", col + "_ending"] for col in common_cols]
self.df = self.df.loc[
- :,
- no_suffix_cols
- + only_ending_cols
- + [col for cols in common_cols for col in cols],
- ]
+ :,
+ no_suffix_cols
+ + only_ending_cols
+ + [col for cols in common_cols for col in cols],
+ ]
def _remove_abnormal_change_in_floor_area(self):
"""
@@ -511,7 +511,7 @@ class TrainingDataset(BaseDataset):
expanded_df["is_sandstone_or_limestone"]
== expanded_df["is_sandstone_or_limestone_ending"]
)
- ]
+ ]
elif component == "floor":
expanded_df = expanded_df[
(expanded_df["is_suspended"] == expanded_df["is_suspended_ending"])
@@ -528,7 +528,7 @@ class TrainingDataset(BaseDataset):
expanded_df["is_to_external_air"]
== expanded_df["is_to_external_air_ending"]
)
- ]
+ ]
elif component == "roof":
expanded_df = expanded_df[
(expanded_df["is_pitched"] == expanded_df["is_pitched_ending"])
@@ -541,7 +541,7 @@ class TrainingDataset(BaseDataset):
expanded_df["has_dwelling_above"]
== expanded_df["has_dwelling_above_ending"]
)
- ]
+ ]
return expanded_df
@@ -742,7 +742,7 @@ class TrainingDataset(BaseDataset):
self.df[col] = self.df[col].fillna("Unknown")
def _null_validation(self, information: str):
- print(f"Null validation after {information}")
+ # print(f"Null validation after {information}")
if pd.isnull(self.df).sum().sum():
raise ValueError(f"Null values found in dataset, after step {information}")
diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py
index 76f99f09..84d1f3e9 100644
--- a/etl/epc_clean/epc_attributes/RoofAttributes.py
+++ b/etl/epc_clean/epc_attributes/RoofAttributes.py
@@ -45,7 +45,7 @@ class RoofAttributes(Definitions):
"""
self.description: str = description.lower().strip()
- self.nodata = not description or description in self.DATA_ANOMALY_MATCHES
+ self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or self.description == "sap05:roof"
self.welsh_translation_search()
diff --git a/etl/property_valuation/requirements.txt b/etl/property_valuation/requirements.txt
new file mode 100644
index 00000000..8a4a1924
--- /dev/null
+++ b/etl/property_valuation/requirements.txt
@@ -0,0 +1,7 @@
+seleniumbase
+beautifulsoup4
+requests
+pandas
+tqdm
+openpyxl
+undetected_chromedriver
\ No newline at end of file
diff --git a/etl/property_valuation/scrape_valuations.py b/etl/property_valuation/scrape_valuations.py
new file mode 100644
index 00000000..434168ca
--- /dev/null
+++ b/etl/property_valuation/scrape_valuations.py
@@ -0,0 +1,88 @@
+import requests
+import random
+import time
+import pandas as pd
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+from seleniumbase import Driver
+from seleniumbase import page_actions
+
+import undetected_chromedriver as webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+import time
+import pandas as pd
+
+BASE_URL = "https://www.zoopla.co.uk/property/uprn/{uprn}/"
+
+
+def initialize_driver():
+ driver = Driver(headless=True, uc=True) # Set headless to True if you want headless mode
+ return driver
+
+
+def app():
+ # Read in the starting asset list
+ asset_list = pd.read_excel("portfolio_epc_data_50m 28th May.xlsx")
+ asset_list = asset_list[["UPRN", "ADDRESS", "POSTCODE"]]
+
+ # asset_list.to_excel("property value.xlsx", index=False)
+
+ # Generate the list of urls
+ urls = [BASE_URL.format(uprn=uprn) for uprn in asset_list["UPRN"]]
+
+ driver = webdriver.Chrome()
+
+ driver = initialize_driver()
+ driver.set_page_load_timeout(30) # Increase page load timeout
+
+ result = []
+ for i, (url, uprn) in tqdm(enumerate(zip(urls, asset_list["UPRN"].tolist())), total=len(urls)):
+
+ # Every 10 requests sleep for an extra 7 seconds
+ if len(result) % 10 == 0 and len(result) != 0:
+ time.sleep(7)
+
+ try:
+
+ driver.get(url)
+ page_actions.wait_for_element_visible(driver, "p[data-testid='estimate-blurred']", timeout=30)
+
+ price_element = driver.find_element("css selector", "p[data-testid='estimate-blurred']")
+ price = price_element.get_text(strip=True)
+
+ low_price_element = driver.find_element("css selector", "span[data-testid='low-estimate-blurred']")
+ low_price = low_price_element.get_text(strip=True)
+
+ high_price_element = driver.find_element("css selector", "span[data-testid='high-estimate-blurred']")
+ high_price = high_price_element.get_text(strip=True)
+
+ result.append(
+ {
+ "UPRN": uprn,
+ "price": price,
+ "lower_estimate": low_price,
+ "upper_estimate": high_price
+ }
+ )
+
+ # Sleep a random amount of time between 5 and 20 seconds
+ sleep_time = 5 + (15 * random.random())
+ time.sleep(sleep_time)
+
+ except Exception as e:
+ print(f"Failed to retrieve data for UPRN {uprn} at iteration {i}: {e}")
+
+ # Store the result depending on where we are
+ savepoint = pd.DataFrame(result)
+ savepoint.to_csv(f"savepoint_index_{i}.csv", index=False)
+
+ # TODO: Testing Jina AI - didn't work but maybe one of the alternatives might work:
+ # https://www.youtube.com/watch?v=QxHE4af5BQE
+ response = requests.get("https://r.jina.ai/https://www.zoopla.co.uk/property/uprn/41222761/")
+ response.text
+
+
+if __name__ == "__main__":
+ app()
diff --git a/recommendations/Costs.py b/recommendations/Costs.py
index 03190727..5f752730 100644
--- a/recommendations/Costs.py
+++ b/recommendations/Costs.py
@@ -20,21 +20,21 @@ regional_labour_variations = [
# This data is based on the MCS database
MCS_SOLAR_PV_COST_DATA = {
- "last_updated": "2024-01-04",
- "average_cost_per_kwh": 2013.94,
- "average_cost_per_kwh-Outer London": 2618.75,
- "average_cost_per_kwh-Inner London": 2618.75,
- "average_cost_per_kwh-South East England": 2083.33,
- "average_cost_per_kwh-South West England": 2113,
- "average_cost_per_kwh-East of England": 1973.86,
- "average_cost_per_kwh-East Midlands": 1981.86,
- "average_cost_per_kwh-West Midlands": 1926.55,
- "average_cost_per_kwh-North East England": 2028.49,
- "average_cost_per_kwh-North West England": 1620.42,
- "average_cost_per_kwh-Yorkshire and the Humber": 2060.9,
- "average_cost_per_kwh-Wales": 1898.83,
- "average_cost_per_kwh-Scotland": 1967.97,
- "average_cost_per_kwh-Northern Ireland": 2126.09,
+ "last_updated": "2024-06-10",
+ "average_cost_per_kwh": 1750,
+ "average_cost_per_kwh-Outer London": 1776,
+ "average_cost_per_kwh-Inner London": 1776,
+ "average_cost_per_kwh-South East England": 1672,
+ "average_cost_per_kwh-South West England": 1732,
+ "average_cost_per_kwh-East of England": 1721,
+ "average_cost_per_kwh-East Midlands": 1730,
+ "average_cost_per_kwh-West Midlands": 1761,
+ "average_cost_per_kwh-North East England": 1669,
+ "average_cost_per_kwh-North West England": 1764,
+ "average_cost_per_kwh-Yorkshire and the Humber": 1705,
+ "average_cost_per_kwh-Wales": 1896,
+ "average_cost_per_kwh-Scotland": 1767,
+ "average_cost_per_kwh-Northern Ireland": 1767,
}
# This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average,
diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py
index 2041f783..ac8c4973 100644
--- a/recommendations/HeatingRecommender.py
+++ b/recommendations/HeatingRecommender.py
@@ -8,6 +8,14 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender
class HeatingRecommender:
+ ELECTRIC_HEATING_DESCRIPTIONS = [
+ "Room heaters, electric",
+ "Electric storage heaters",
+ "Electric storage heaters, radiators",
+ "Portable electric heaters assumed for most rooms",
+ ]
+
+ high_heat_retention_contols_desc = "Controls for high heat retention storage heaters"
def __init__(self, property_instance: Property):
self.property = property_instance
@@ -16,6 +24,24 @@ class HeatingRecommender:
self.heating_recommendations = []
self.heating_control_recommendations = []
+ self.has_electric_heating_description = (
+ self.property.main_heating["clean_description"] in self.ELECTRIC_HEATING_DESCRIPTIONS
+ )
+
+ def is_high_heat_retention_valid(self):
+ """
+ Check conditions if high heat retention storage is valid
+ :return:
+ """
+
+ # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we
+ # can consider hhr storage heaters
+ electric_heating_assumed = (
+ self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"]
+ )
+
+ return self.has_electric_heating_description or electric_heating_assumed
+
def recommend(self, has_cavity_or_loft_recommendations, phase=0):
"""
Produces heating recommendations
@@ -34,17 +60,10 @@ class HeatingRecommender:
# This first iteration of the recommender will provide very basic recommendation
# We recommend heating controls based on the main heating system
- has_electric_heating_description = self.property.main_heating["clean_description"] in [
- "Room heaters, electric", "Electric storage heaters", "Electric storage heaters, radiators"
- ]
-
- no_heating_no_mains = (
- self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"] and
- not self.property.data["mains-gas-flag"]
- )
-
- if has_electric_heating_description or no_heating_no_mains:
+ if self.is_high_heat_retention_valid():
# Recommend high heat retention storage heaters
+ # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just
+ # needs the controls
self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False)
# if the property has mains heating with boiler and radiators, we recommend optimal heating controls
@@ -61,7 +80,7 @@ class HeatingRecommender:
)
# We also check if the property has electric heating, but it has access to the mains gas
- electic_heating_has_mains = has_electric_heating_description and self.property.data["mains-gas-flag"]
+ electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"]
portable_heaters_has_mains = (
self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"] and
@@ -93,16 +112,19 @@ class HeatingRecommender:
# In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions
# and either allow or prevent the recommendation of an air source heat pump
- suitable_property_type = self.property.data["property-type"] in ["House", "Bungalow"]
- has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"]
-
- if suitable_property_type and not has_air_source_heat_pump:
+ if self.is_ashp_valid():
self.recommend_air_source_heat_pump(
phase=phase, has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations
)
return
+ def is_ashp_valid(self):
+ suitable_property_type = self.property.data["property-type"] in ["House", "Bungalow"]
+ has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"]
+
+ return suitable_property_type and not has_air_source_heat_pump
+
def recommend_air_source_heat_pump(self, phase, has_cavity_or_loft_recommendations, _return=False):
"""
This method will implement the recommendation for an air source heat pump
@@ -314,6 +336,27 @@ class HeatingRecommender:
return output
+ def is_hhr_already_installed(self):
+ """
+ Check if the property already has high heat retention storage heaters
+ :return:
+ """
+
+ already_has_hhr = "Electric storage heaters" in self.property.main_heating["clean_description"]
+
+ # Some electric storage heaters will show that the controls are "Manual charge controls" which are indicative
+ # of the old model of electric storage heaters, originating from 1961.
+ # Newer HHR storage heaters will charge up over night but will retain the heat durin the day for when warmth
+ # is actually needed, unlike traditional storage heaters that charge up at night and release heat during the day
+ # which isn't always ideal for the occupants.
+ already_has_hhr_contols = (
+ self.property.main_heating_controls[
+ "clean_description"
+ ].lower() == self.high_heat_retention_contols_desc.lower()
+ )
+
+ return already_has_hhr and already_has_hhr_contols
+
def recommend_hhr_storage_heaters(self, phase, system_change, heating_controls_only, _return=False):
"""
We will recommend upgrading to a high heat retention storage system, if the current system is not already
@@ -330,19 +373,14 @@ class HeatingRecommender:
controls_recommender = HeatingControlRecommender(self.property)
# The heating controls we're recommending for are based on the recommended heating system
- high_heat_retention_contols_desc = "Controls for high heat retention storage heaters"
+
# We only recommend Celect-type controls if the current heating system is not Celect-type controls
- if self.property.main_heating_controls["clean_description"] != high_heat_retention_contols_desc:
+ if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc:
controls_recommender.recommend(heating_description="Electric storage heaters, radiators")
- # Conditions for not needing this recommendation
- already_installed_hh_retention = (
- "Electric storage heaters" in self.property.main_heating["clean_description"] and
- self.property.main_heating_controls["clean_description"].lower() == high_heat_retention_contols_desc.lower()
- )
-
+ has_hhr = self.is_hhr_already_installed()
# Conditions for not recommending electric storage heaters
- if already_installed_hh_retention:
+ if has_hhr:
# No recommendation needed
return
diff --git a/recommendations/Mds.py b/recommendations/Mds.py
index 7453e5e9..4c417447 100644
--- a/recommendations/Mds.py
+++ b/recommendations/Mds.py
@@ -1,3 +1,5 @@
+import itertools
+from utils.logger import setup_logger
from backend.Property import Property
from recommendations.FloorRecommendations import FloorRecommendations
from recommendations.WallRecommendations import WallRecommendations
@@ -12,13 +14,25 @@ from recommendations.HotwaterRecommendations import HotwaterRecommendations
from recommendations.SecondaryHeating import SecondaryHeating
from recommendations.Recommendations import Recommendations
+logger = setup_logger()
+
class Mds:
"""
Handles the contruction of the MDS report
"""
- def __init__(self, property_instance: Property, materials):
+ format_map = {
+ "external_wall_insulation": "EWI (Trad Const)",
+ "internal_wall_insualtion": "IWI",
+ "cavity_wall_insulation": "CWI",
+ "loft_insulation": "LI",
+ "air_source_heat_pump": "ASHP Htg",
+ "high_heat_retention_storage_heaters": "High Heat Retention Storage Heaters",
+ "solar_pv": "Solar PV",
+ }
+
+ def __init__(self, property_instance: Property, materials, optimise_measures: bool = False):
self.property_instance = property_instance
self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials)
@@ -35,14 +49,169 @@ class Mds:
self.hotwater_recommender = HotwaterRecommendations(property_instance=property_instance)
self.secondary_heating_recommender = SecondaryHeating(property_instance=property_instance)
- def build(self):
- if self.property_instance.measures is None:
- raise NotImplementedError("No measures in the property - implement me")
+ # This flag indicates that we wish to optimise the measures, to the property, depending on the set of measures
+ # we have been provided
+ self.optimise_measures = optimise_measures
- measures = self.property_instance.measures
+ def select_optimal_measure_set(self, measures):
- measure_config_list = [list(m.keys())[0] for m in measures]
+ # This is the set
+ all_considered_measures = [
+ 'external_wall_insulation',
+ 'cavity_wall_insulation',
+ 'loft_insulation',
+ 'air_source_heat_pump',
+ 'high_heat_retention_storage_heaters',
+ 'solar_pv'
+ ]
+ # Check if our measures are within the ones we've handled
+ new = [m for m in measures if m not in all_considered_measures]
+ if new:
+ raise NotImplementedError("New measures - handle me")
+
+ def prune_options(options, measures):
+ options_pruned = []
+ for _group in options:
+ group_pruned = [m for m in _group if m in measures]
+ if not group_pruned:
+ continue
+ options_pruned.append(group_pruned)
+
+ return options_pruned
+
+ # For options in here, a property could only possibly have one of these
+ one_choice_options = [
+ ["external_wall_insulation", "cavity_wall_insulation", "internal_wall_insulation"],
+ ["loft_insulation", "flat_roof_insulation", "room_in_roof_insulation"],
+ ["solid_floor_insulation", "suspended_floor_insulation"],
+ ]
+ # prune one_choice_options based on the measure set considered for this property
+ one_choice_options_pruned = prune_options(one_choice_options, measures)
+
+ # For options in here, a property could have one or the other so all should be considered
+ multi_path_options = [
+ ["air_source_heat_pump", "high_heat_retention_storage_heaters", "gas_boiler"]
+ ]
+
+ multi_path_options_pruned = prune_options(multi_path_options, measures)
+
+ one_choice_combinations = [list(itertools.product(*one_choice_options_pruned))]
+ one_choice_combinations = [list(x) for sublist in one_choice_combinations for x in sublist]
+ multi_path_combinations = [list(itertools.product(*multi_path_options_pruned))]
+ multi_path_combinations = [list(x) for sublist in multi_path_combinations for x in sublist]
+
+ one_choice_flat = [item for sublist in one_choice_options_pruned for item in sublist]
+ multi_path_flat = [item for sublist in multi_path_options_pruned for item in sublist]
+
+ remaining_measures = [
+ measure for measure in measures
+ if measure not in one_choice_flat and measure not in multi_path_flat
+ ]
+
+ # Combine one_choice and multi_path combinations with remaining measures
+ final_combinations = []
+ for one_choice in one_choice_combinations:
+ for multi_path in multi_path_combinations:
+ final_combinations.append([m for m in one_choice + multi_path + remaining_measures])
+
+ pruned_combinations = []
+ # TODO: We can do these checks once, outside of the loop and prune the combinations
+ for combination in final_combinations:
+ pruned_measures = []
+ for measure in combination:
+ if measure not in measures:
+ continue
+ # There are certain measures where we need to
+ if measure == "external_wall_insulation":
+ # Check if the wall is not cavity since the other wall types can take external wall insulation
+ if (
+ self.wall_recommender.ewi_valid() and
+ not self.property_instance.walls["insulation_thickness"] in ["average", "above average"]
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "cavity_wall_insulation":
+ # Check if the wall is cavity
+ if (
+ self.property_instance.walls['is_cavity_wall'] and
+ not self.property_instance.walls['is_filled_cavity']
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "loft_insulation":
+ # Check if the roof is suitable for loft insulation and the loft isn't already done
+ # Or, if the home had a u-value for the roof, we don't recommend loft insulation
+ if (
+ self.property_instance.roof["is_pitched"] and
+ not self.roof_recommender.is_loft_already_insulated() and
+ self.property_instance.roof["thermal_transmittance_unit"] is None
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "solid_floor_insulation":
+ # Check if the floor is solid
+ if (
+ self.property_instance.floor["is_solid"] and
+ self.property_instance.floor["insulation_thickness"] not in ["average", "above average"] and
+ self.property_instance.floor["thermal_transmittance_unit"] is not None
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "suspended_floor_insulation":
+ # Check if the floor is suspended
+ if (
+ self.property_instance.floor["is_suspended"] and
+ self.property_instance.floor["insulation_thickness"] not in ["average", "above average"] and
+ self.property_instance.floor["thermal_transmittance_unit"] is not None
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "high_heat_retention_storage_heaters":
+
+ # For the moment, we recommend storage heaters if the property doesn't already
+ # and don't make it contngent on controls
+ already_has_hhr = self.heating_recommender.is_hhr_already_installed()
+
+ if (
+ self.heating_recommender.is_high_heat_retention_valid() and
+ not already_has_hhr
+ ):
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "air_source_heat_pump":
+ if self.heating_recommender.is_ashp_valid():
+ pruned_measures.append(measure)
+ continue
+
+ if measure == "solar_pv":
+ if self.solar_recommender.is_solar_pv_valid():
+ pruned_measures.append(measure)
+ continue
+
+ raise NotImplementedError("Implement me")
+
+ if not pruned_measures:
+ continue
+
+ pruned_measures_formatted = []
+ for pm in pruned_measures:
+ pruned_measures_formatted.append({pm: self.format_map[pm]})
+
+ pruned_combinations.append(pruned_measures_formatted)
+
+ # We're left with the subset of measures that are possible for this property
+ # These are the possible groups of measures that could be applied to this home
+
+ return pruned_combinations
+
+ def _build(self, measure_config_list, measures):
not_implemented_measures = [
"party_wall_insulation",
"ground_source_heat_pump",
@@ -60,114 +229,164 @@ class Mds:
mds_recommendations = []
errors = []
+ phase = 0
# TODO: Could use a decarator to reduce the boilerplate code - insert_recommendation_id and then the append
if "external_wall_insulation" in measure_config_list:
- recs = self.wall_recommender.mds_recommend_ewi(phase=0)
+ recs = self.wall_recommender.mds_recommend_ewi(phase=phase)
if not recs:
raise Exception("No recommendations for external wall insulation")
recs = self.insert_recommendation_id(recs, measures, "external_wall_insulation")
mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
if "cavity_wall_insulation" in measure_config_list:
- recs = self.wall_recommender.mds_recommend_cavity_wall_insulation(phase=0)
+ recs = self.wall_recommender.mds_recommend_cavity_wall_insulation(phase=phase)
recs = self.insert_recommendation_id(recs, measures, "cavity_wall_insulation")
mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
if "loft_insulation" in measure_config_list:
# Check if the roof is suitable for loft insulation
if self.property_instance.roof['is_roof_room']:
errors.append("Roof is a room")
else:
- recs = self.roof_recommender.mds_loft_insulation(phase=0)
+ recs = self.roof_recommender.mds_loft_insulation(phase=phase)
if not recs:
raise Exception("No recommendations for loft insulation")
recs = self.insert_recommendation_id(recs, measures, "loft_insulation")
mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
if "internal_wall_insulation" in measure_config_list:
raise Exception("check me out 4")
- self.wall_recommender.recommend(phase=0)
+ self.wall_recommender.recommend(phase=phase)
if "suspended_floor_insulation" in measure_config_list:
raise Exception("check me out 5")
- self.floor_recommender.recommend(phase=0)
+ self.floor_recommender.recommend(phase=phase)
if "solid_floor_insulation" in measure_config_list:
raise Exception("check me out 6")
- self.floor_recommender.recommend(phase=0)
+ self.floor_recommender.recommend(phase=phase)
if "air_source_heat_pump" in measure_config_list:
recs = self.heating_recommender.recommend_air_source_heat_pump(
- phase=0, has_cavity_or_loft_recommendations=False, _return=True
+ phase=phase, has_cavity_or_loft_recommendations=False, _return=True
)
recs = self.insert_recommendation_id(recs, measures, "air_source_heat_pump")
mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
- if "electric_storage_heaters" in measure_config_list:
+ if "high_heat_retention_storage_heaters" in measure_config_list:
recs = self.heating_recommender.recommend_hhr_storage_heaters(
- phase=0, system_change=True, heating_controls_only=False, _return=True
+ phase=phase, system_change=True, heating_controls_only=False, _return=True
)
- recs = self.insert_recommendation_id(recs, measures, "electric_storage_heaters")
- mds_recommendations.append(recs)
+ if recs is None:
+ logger.info(
+ f"No recommendations for high heat retention storage heaters, current heating "
+ f"{self.property_instance.main_heating['clean_description']}"
+ )
+ else:
+ recs = self.insert_recommendation_id(recs, measures, "high_heat_retention_storage_heaters")
+ mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
if "low_energy_lighting" in measure_config_list:
raise Exception("check me out 9")
- self.lighting_recommender.recommend(phase=0)
+ self.lighting_recommender.recommend(phase=phase)
if "cylinder_insulation" in measure_config_list:
raise Exception("check me out 10")
- self.hotwater_recommender.recommend(phase=0)
+ self.hotwater_recommender.recommend(phase=phase)
if "smart_controls" in measure_config_list:
raise Exception("check me out 11")
- self.heating_recommender.recommend(phase=0)
+ self.heating_recommender.recommend(phase=phase)
if "zone_controls" in measure_config_list:
raise Exception("check me out 12")
- self.heating_recommender.recommend(phase=0)
+ self.heating_recommender.recommend(phase=phase)
if "trvs" in measure_config_list:
raise Exception("check me out 13")
- self.heating_recommender.recommend(phase=0)
+ self.heating_recommender.recommend(phase=phase)
if "solar_pv" in measure_config_list:
- recs = self.solar_recommender.mds_recommend(phase=0, solar_pv_percentage=0.5)
+ recs = self.solar_recommender.mds_recommend(phase=phase, solar_pv_percentage=0.5)
recs = self.insert_recommendation_id(recs, measures, "solar_pv")
mds_recommendations.append(recs)
+ if self.optimise_measures and len(recs):
+ phase += 1
if "double_glazing" in measure_config_list:
raise Exception("check me out 15")
- self.windows_recommender.recommend(phase=0)
+ self.windows_recommender.recommend(phase=phase)
if "mechanical_ventilation" in measure_config_list:
raise Exception("check me out 16")
- self.ventilation_recomender.recommend(phase=0)
+ self.ventilation_recomender.recommend(phase=phase)
if "gas_boiler" in measure_config_list:
raise Exception("check me out 17")
- self.heating_recommender.recommend(phase=0)
+ self.heating_recommender.recommend(phase=phase)
if "flat_roof_insulation" in measure_config_list:
raise Exception("check me out 18")
- self.roof_recommender.recommend(phase=0)
+ self.roof_recommender.recommend(phase=phase)
if "room_in_roof_insulation" in measure_config_list:
raise Exception("check me out 19")
- self.roof_recommender.recommend(phase=0)
+ self.roof_recommender.recommend(phase=phase)
property_representative_recommendations = Recommendations.create_representative_recommendations(
mds_recommendations, non_invasive_recommendations=[]
)
- return property_representative_recommendations, errors
+ return mds_recommendations, property_representative_recommendations, errors
+
+ def build(self):
+ if self.property_instance.measures is None:
+ raise NotImplementedError("No measures in the property - implement me")
+
+ if self.optimise_measures:
+ measures_set = self.select_optimal_measure_set(self.property_instance.measures)
+ mds_recommendations_map = {}
+ representative_recommendations_map = {}
+ errors_map = {}
+ for measures in measures_set:
+ measure_config_list = [list(x.keys())[0] for x in measures]
+ mds_recommendations, rep_recommendations, errors = self._build(
+ measure_config_list=measure_config_list,
+ measures=measures
+ )
+ if errors:
+ logger.info(f"Errors: {errors}")
+
+ mds_recommendations_map[str(measure_config_list)] = mds_recommendations
+ representative_recommendations_map[str(measure_config_list)] = rep_recommendations
+ errors_map[str(measure_config_list)] = errors
+
+ return mds_recommendations_map, representative_recommendations_map, errors_map
+
+ else:
+ measure_config_list = [list(m.keys())[0] for m in self.property_instance.measures]
+ return self._build(measure_config_list=measure_config_list, measures=self.property_instance.measures)
@staticmethod
def insert_recommendation_id(recommendations, measures, measure_name):
# Insert the recommendation identifier into this recommendation
measure_config = [m for m in measures if measure_name in m][0]
+
+ idx = 0
for r in recommendations:
- r["recommendation_id"] = list(measure_config.values())[0]
+ r["recommendation_id"] = list(measure_config.values())[0] + "-" + str(idx)
+ idx += 1
return recommendations
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index c8113cdc..c9ac1072 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -227,7 +227,7 @@ class Recommendations:
recommendations_by_type = sorted(recommendations_by_type, key=lambda x: x["type"])
representative_recommendations = []
- for type, recommendations in groupby(recommendations_by_type, key=lambda x: x["type"]):
+ for _type, recommendations in groupby(recommendations_by_type, key=lambda x: x["type"]):
recommendations = list(recommendations)
# We also create an efficiency key, which is used to sort the recommendations
if has_u_value:
@@ -311,14 +311,6 @@ class Recommendations:
# This is the unadjusted resulting heat demand
predicted_heat_demand_change = starting_heat_demand - expected_heat_demand
- # We don't want to adjust the heat demand for mechanical ventilation so we add it back on
-
- # We adjust the heat demand figures to align to the UCL paper
- current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
- epc_energy_consumption=starting_heat_demand,
- current_epc_rating=property_instance.data["current-energy-rating"],
- )
-
# TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are
# actually implemented
expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
@@ -327,11 +319,10 @@ class Recommendations:
)
adjusted_heat_demand_change = (
- current_adjusted_energy - expected_adjusted_energy
+ property_instance.current_adjusted_energy - expected_adjusted_energy
)
# TODO: We should determine if the home is gas & electricity or just electricity
- current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy)
expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy)
for recommendations_by_type in property_recommendations:
@@ -410,8 +401,6 @@ class Recommendations:
return (
property_recommendations,
- current_adjusted_energy,
expected_adjusted_energy,
- current_energy_bill,
expected_energy_bill
)
diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py
index 538d90e4..81f514b1 100644
--- a/recommendations/RoofRecommendations.py
+++ b/recommendations/RoofRecommendations.py
@@ -54,6 +54,13 @@ class RoofRecommendations:
]
]
+ # Extract the insulation thickness from the roof, which is used throughout this method
+ self.insulation_thickness = convert_thickness_to_numeric(
+ self.property.roof["insulation_thickness"],
+ self.property.roof["is_pitched"],
+ self.property.roof["is_flat"]
+ )
+
def mds_loft_insulation(self, phase):
"""
For usages within the mds report
@@ -62,18 +69,18 @@ class RoofRecommendations:
"""
self.recommendations = []
- insulation_thickness = convert_thickness_to_numeric(
- self.property.roof["insulation_thickness"],
- self.property.roof["is_pitched"],
- self.property.roof["is_flat"]
- )
-
u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band})
- self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
+ self.recommend_roof_insulation(u_value, self.insulation_thickness, self.property.roof, phase)
return self.recommendations
+ def is_loft_already_insulated(self):
+ """
+ Check if the loft is already insulated
+ """
+ return (self.insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]
+
def recommend(self, phase):
if self.property.roof["has_dwelling_above"]:
@@ -81,21 +88,15 @@ class RoofRecommendations:
u_value = self.property.roof["thermal_transmittance"]
- insulation_thickness = convert_thickness_to_numeric(
- self.property.roof["insulation_thickness"],
- self.property.roof["is_pitched"],
- self.property.roof["is_flat"]
- )
-
# We check if the roof is already insulated and if so, we exit
# Building regulations part L recommend installing at least 270mm of insulation, however generally we
# experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation
# This only holds true for pitched roofs.
- if (insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]:
+ if self.is_loft_already_insulated():
return
- if (insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]:
+ if (self.insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]:
return
if self.property.roof["is_roof_room"]:
@@ -119,7 +120,7 @@ class RoofRecommendations:
return
if self.property.roof["is_pitched"] or self.property.roof["is_flat"]:
- self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
+ self.recommend_roof_insulation(u_value, self.insulation_thickness, self.property.roof, phase)
return
if self.property.roof["is_roof_room"]:
diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py
index 14161da3..458eae12 100644
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@@ -4,10 +4,13 @@ from recommendations.recommendation_utils import override_costs
class SolarPvRecommendations:
+ # Solar panel specs based on Eurener 400s solar panels
+ # https://midsummerwholesale.co.uk/buy/eurener/eurener-400w-mepv-zebra-ab-half-cut-mono
# Approximate area of the solar panels
- SOLAR_PANEL_AREA = 1.6
+ SOLAR_PANEL_AREA = 1.79
# Wattage per panel - this is based on the average wattage of a solar panel being between 250w and 420w
- SOLAR_PANEL_WATTAGE = 250
+ # This was previously set to 250w, but has been upped to 400 based on the systems used by Cotswolrd Energy Group
+ SOLAR_PANEL_WATTAGE = 400
MAX_SYSTEM_WATTAGE = 6000
MIN_SYSTEM_WATTAGE = 1000
@@ -75,15 +78,7 @@ class SolarPvRecommendations:
}
]
- def recommend(self, phase):
- """
- We check if a property is potentially suitable for solar PV based on the following criteria:
- - The property is a house or bungalow
- - The property has a flat or pitched roof
- - The property does not have existing solar pv
- :return:
- """
-
+ def is_solar_pv_valid(self):
is_valid_property_type = self.property.data["property-type"] in ["House", "Bungalow", "Maisonette"]
is_valid_roof_type = (
self.property.roof["is_flat"] or self.property.roof["is_pitched"] or self.property.roof["is_roof_room"]
@@ -93,7 +88,18 @@ class SolarPvRecommendations:
None, 0, self.property.DATA_ANOMALY_MATCHES
]
- if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv:
+ return is_valid_property_type and is_valid_roof_type and has_no_existing_solar_pv
+
+ def recommend(self, phase):
+ """
+ We check if a property is potentially suitable for solar PV based on the following criteria:
+ - The property is a house or bungalow
+ - The property has a flat or pitched roof
+ - The property does not have existing solar pv
+ :return:
+ """
+
+ if not self.is_solar_pv_valid():
return
solar_pv_percentage = self.property.solar_pv_percentage
diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py
index ea58c4e3..fb228b49 100644
--- a/recommendations/WallRecommendations.py
+++ b/recommendations/WallRecommendations.py
@@ -55,23 +55,26 @@ class WallRecommendations(Definitions):
NEW_BUILD_INSULATED = 0.75
# These are the ending descriptions we consider for walls with external insulation
+ # This maps the clean descriptions to the ending descriptions
EXTERNALLY_INSULATED_WALL_DESCRIPTIONS = {
- "solid_brick": "Solid brick, with external insulation",
- "cob": "Cob, with external insulation",
- "system_built": "System built, with external insulation",
- "granite_or_whinstone": 'Granite or whinstone, with external insulation',
- "sandstone_or_limestone": 'Sandstone or limestone, with external insulation',
- "timber_frame": "Timber frame, with external insulation"
+ "Cavity wall, as built, insulated": "Cavity wall, filled cavity and external insulation",
+ "Solid brick, as built, no insulation": "Solid brick, with external insulation",
+ "Solid brick, as built, insulated": "Solid brick, with external insulation",
+ "Cob, as built": "Cob, with external insulation",
+ "System built, as built, no insulation": "System built, with external insulation",
+ "Granite or whinstone, as built, no insulation": 'Granite or whinstone, with external insulation',
+ "Timber frame, as built, no insulation": "Timber frame, with external insulation",
}
# These are the ending descriptions we consider for walls with internal insulation
INTERNALLY_INSULATED_WALL_DESCRIPTIONS = {
- "solid_brick": "Solid brick, with internal insulation",
- "cob": "Cob, with internal insulation",
- "system_built": "System built, with internal insulation",
- "granite_or_whinstone": 'Granite or whinstone, with internal insulation',
- "sandstone_or_limestone": 'Sandstone or limestone, with internal insulation',
- "timber_frame": "Timber frame, with internal insulation"
+ "Cavity wall, as built, insulated": "Cavity wall, filled cavity and internal insulation",
+ "Solid brick, as built, no insulation": "Solid brick, with internal insulation",
+ "Solid brick, as built, insulated": "Solid brick, with internal insulation",
+ "Cob, as built": "Cob, with internal insulation",
+ "System built, as built, no insulation": "System built, with internal insulation",
+ "Granite or whinstone, as built, no insulation": 'Granite or whinstone, with internal insulation',
+ "Timber frame, as built, no insulation": "Timber frame, with internal insulation",
}
def __init__(
@@ -99,7 +102,7 @@ class WallRecommendations(Definitions):
part
for part in materials
if part["type"]
- in ["iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"]
+ in ["iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"]
]
self.external_wall_insulation_materials = [
@@ -109,11 +112,9 @@ class WallRecommendations(Definitions):
self.external_wall_non_insulation_materials = [
part
for part in materials
- if part["type"]
- in ["ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"]
+ if part["type"] in ["ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"]
]
- @property
def ewi_valid(self):
"""
This method check available data, to determine if a property is suitable for external wall insulation
@@ -123,11 +124,24 @@ class WallRecommendations(Definitions):
# it is not suitable for EWI
if self.property.restricted_measures or (
self.property.data["property-type"].lower() == "flat"
+ ) or (
+ self.property.walls['is_cob'] or
+ self.property.walls['is_sandstone_or_limestone'] or
+ self.property.walls["is_cavity_wall"]
):
return False
return True
+ def is_suitable_for_solid_insulation(self):
+ """
+ Checks if the wall is of a suitable type for internal/external wall insulation
+ """
+ if self.property.walls["is_cavity_wall"] or self.property.walls["is_cob"]:
+ return False
+
+ return True
+
def mds_recommend_cavity_wall_insulation(self, phase=None):
# Function specifically for cavity wall insulation, for usage in the mds report
self.recommendations = []
@@ -175,7 +189,7 @@ class WallRecommendations(Definitions):
# recommend internal wall insulation as a possible measure
u_value = self.property.walls["thermal_transmittance"]
- u_value = None if math.isnan(u_value) else u_value
+ u_value = None if pd.isnull(u_value) else u_value
is_cavity_wall = self.property.walls["is_cavity_wall"]
insulation_thickness = self.property.walls["insulation_thickness"]
@@ -246,7 +260,7 @@ class WallRecommendations(Definitions):
return
# Remaining wall types are treated with IWI or EWI
- if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
+ if (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) and self.is_suitable_for_solid_insulation():
self.find_insulation(u_value, phase)
return
@@ -332,18 +346,19 @@ class WallRecommendations(Definitions):
wall_ending_config = WallAttributes("Cavity wall, filled cavity").process()
- simulation_config = {}
- if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
- simulation_config = {
- "walls_energy_eff_ending": "Good",
- "walls_thermal_transmittance_ending": new_u_value
- }
-
walls_simulation_config = check_simulation_difference(
new_config=wall_ending_config, old_config=self.property.walls, prefix="walls_"
)
- simulation_config = {**simulation_config, **walls_simulation_config}
+ simulation_config = self.set_starting_simulation_config(
+ wall_ending_config=wall_ending_config
+ )
+
+ simulation_config = {
+ **simulation_config,
+ **walls_simulation_config,
+ "walls_thermal_transmittance_ending": new_u_value
+ }
recommendations.append(
{
@@ -370,30 +385,35 @@ class WallRecommendations(Definitions):
self.recommendations = recommendations
def get_internal_external_wall_description(self, description_map, new_u_value):
- if self.property.walls["is_solid_brick"]:
- return description_map["solid_brick"]
-
- if self.property.walls["is_cob"]:
- return description_map["cob"]
-
- if self.property.walls["is_system_built"]:
- return description_map["system_built"]
-
- if self.property.walls["is_granite_or_whinstone"]:
- return description_map["granite_or_whinstone"]
-
- if self.property.walls["is_sandstone_or_limestone"]:
- return description_map["sandstone_or_limestone"]
-
- if self.property.walls["is_timber_frame"]:
- return description_map["timber_frame"]
if "Average thermal transmittance" in self.property.walls["clean_description"]:
if new_u_value is None:
raise ValueError("New u value is None")
return f'Average thermal transmittance {new_u_value} W/m-¦K'
- raise NotImplementedError("Not implemented yet")
+ return description_map[self.property.walls["clean_description"]]
+
+ def set_starting_simulation_config(self, wall_ending_config):
+ """
+ Helper function to set the starting simulation config
+ """
+
+ simulation_config = {}
+ if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
+ simulation_config = {
+ "walls_energy_eff_ending": "Good"
+ }
+
+ # We check if we have double insulation in any instances
+ double_insulation = (
+ (wall_ending_config["is_filled_cavity"] and wall_ending_config["external_insulation"]) or
+ (wall_ending_config["is_filled_cavity"] and wall_ending_config["internal_insulation"]) or
+ (wall_ending_config["external_insulation"] and wall_ending_config["internal_insulation"])
+ )
+ if double_insulation:
+ simulation_config["walls_energy_eff_ending"] = "Very Good"
+
+ return simulation_config
def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase):
@@ -468,16 +488,14 @@ class WallRecommendations(Definitions):
wall_ending_config = WallAttributes(new_description).process()
- simulation_config = {}
- if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
- simulation_config = {
- "walls_energy_eff_ending": "Good"
- }
-
walls_simulation_config = check_simulation_difference(
new_config=wall_ending_config, old_config=self.property.walls, prefix="walls_"
)
+ simulation_config = self.set_starting_simulation_config(
+ wall_ending_config=wall_ending_config
+ )
+
simulation_config = {
**walls_simulation_config,
**simulation_config,
@@ -521,7 +539,7 @@ class WallRecommendations(Definitions):
# consider diminishing returns between the two as they are considered to be separate measures
ewi_recommendations = []
- if self.ewi_valid:
+ if self.ewi_valid():
ewi_recommendations = self._find_insulation(
u_value=u_value,
insulation_materials=pd.DataFrame(
diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py
index 996f5c9c..07a861dc 100644
--- a/recommendations/recommendation_utils.py
+++ b/recommendations/recommendation_utils.py
@@ -756,17 +756,23 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned):
return cavity_age
-def check_simulation_difference(old_config, new_config, prefix=""):
+def check_simulation_difference(old_config, new_config, prefix="", keys_with_prefix=None):
"""
Given two dictionaries, that describe the heating control configurations, this method will compare the two
and pick out the differences. These differences will be things that have been added and things that have been
removed. This will be used to determine how we should be updating the configuration in the simulation
:return:
"""
+
+ keys_with_prefix = (
+ ["is_assumed", "thermal_transmittance", "insulation_thickness"] if keys_with_prefix is None
+ else keys_with_prefix
+ )
+
differences = {}
for key in new_config:
if old_config[key] != new_config[key]:
- new_key = prefix + key + "_ending" if key in ["is_assumed", "thermal_transmittance"] else key + "_ending"
+ new_key = prefix + key + "_ending" if key in keys_with_prefix else key + "_ending"
differences[new_key] = new_config[key]
return differences