junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix merge conflict

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-04-03 20:24:42 +01:00
commit f5961ff6c7
5 changed files with 299 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

319
backend/Property.py
View file

@ -11,11 +11,17 @@ from utils.s3 import read_dataframe_from_s3_parquet
from etl.epc.settings import DATA_ANOMALY_MATCHES from etl.epc.settings import DATA_ANOMALY_MATCHES
from recommendations.rdsap_tables import FLOOR_LEVEL_MAP from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
from recommendations.recommendation_utils import ( from recommendations.recommendation_utils import (
estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area, estimate_windows estimate_perimeter,
get_wall_type,
estimate_external_wall_area,
esimtate_pitched_roof_area,
estimate_windows,
) )
ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev")
DATA_BUCKET = os.environ.get('DATA_BUCKET', 'retrofit-data-dev' if ENVIRONMENT == 'dev' else None) DATA_BUCKET = os.environ.get(
"DATA_BUCKET", "retrofit-data-dev" if ENVIRONMENT == "dev" else None
)
logger = setup_logger() logger = setup_logger()
@ -30,7 +36,7 @@ class Property:
"roof-description": "roof", "roof-description": "roof",
"walls-description": "walls", "walls-description": "walls",
"windows-description": "windows", "windows-description": "windows",
"lighting-description": "lighting" "lighting-description": "lighting",
} }
floor = None floor = None
@ -57,7 +63,9 @@ class Property:
self.address = address self.address = address
self.postcode = postcode self.postcode = postcode
self.data = {k.replace("_", "-"): v for k, v in epc_record.get("prepared_epc").items()} self.data = {
k.replace("_", "-"): v for k, v in epc_record.get("prepared_epc").items()
}
self.old_data = epc_record.get("old_data") self.old_data = epc_record.get("old_data")
self.property_dimensions = None self.property_dimensions = None
@ -92,7 +100,9 @@ class Property:
"wind_turbine": epc_record.prepared_epc.get("wind_turbine_count"), "wind_turbine": epc_record.prepared_epc.get("wind_turbine_count"),
} }
self.number_of_open_fireplaces = { self.number_of_open_fireplaces = {
"number_of_open_fireplaces": epc_record.prepared_epc.get("number_open_fireplaces"), "number_of_open_fireplaces": epc_record.prepared_epc.get(
"number_open_fireplaces"
),
} }
self.number_of_extensions = { self.number_of_extensions = {
"number_of_extensions": epc_record.prepared_epc.get("extension_count"), "number_of_extensions": epc_record.prepared_epc.get("extension_count"),
@ -105,13 +115,15 @@ class Property:
"length": epc_record.prepared_epc.get("unheated_corridor_length"), "length": epc_record.prepared_epc.get("unheated_corridor_length"),
"heat_loss_corridor_boolean": epc_record.get("heat_loss_corridor_bool"), "heat_loss_corridor_boolean": epc_record.get("heat_loss_corridor_bool"),
} }
self.mains_gas = epc_record.prepared_epc.get('mains_gas_flag') self.mains_gas = epc_record.prepared_epc.get("mains_gas_flag")
self.floor_height = epc_record.prepared_epc.get('floor_height') self.floor_height = epc_record.prepared_epc.get("floor_height")
self.insulation_wall_area = None self.insulation_wall_area = None
self.floor_area = epc_record.prepared_epc.get('total_floor_area') self.floor_area = epc_record.prepared_epc.get("total_floor_area")
self.pitched_roof_area = None self.pitched_roof_area = None
self.insulation_floor_area = None self.insulation_floor_area = None
self.number_lighting_outlets = epc_record.prepared_epc.get("fixed_lighting_outlets_count") self.number_lighting_outlets = epc_record.prepared_epc.get(
"fixed_lighting_outlets_count"
)
self.floor_level = None self.floor_level = None
self.number_of_windows = None self.number_of_windows = None
self.solar_pv_percentage = None self.solar_pv_percentage = None
@ -128,18 +140,33 @@ class Property:
It will be the same starting and ending EPC, as we don't have the expected EPC yet It will be the same starting and ending EPC, as we don't have the expected EPC yet
""" """
difference_record = self.epc_record - self.epc_record # difference_record = self.epc_record - self.epc_record
# TODO: change these lower and replace in the settings file # TODO: change these lower and replace in the settings file
print(
"CHANGE THE LATEST FIELD TO REMOVE NUMBER HABITABLE ROOMS IF WE WANT TO USE STARTING/ENDING"
)
fixed_data_col_names = MANDATORY_FIXED_FEATURES + LATEST_FIELD fixed_data_col_names = MANDATORY_FIXED_FEATURES + LATEST_FIELD
print("NEED TO CHANGE THE DASH TO LOWER CASE") print("NEED TO CHANGE THE DASH TO LOWER CASE")
fixed_data_col_names = [x.lower().replace("_", "-") for x in fixed_data_col_names] fixed_data_col_names = [
x.lower().replace("_", "-") for x in fixed_data_col_names
]
fixed_data = {k.replace("-", "_"): v for k, v in self.data.items() if k in fixed_data_col_names} fixed_data = {
k.replace("-", "_"): v
for k, v in self.data.items()
if k in fixed_data_col_names
}
difference_record.append_fixed_data(fixed_data) # difference_record.append_fixed_data(fixed_data)
self.base_difference_record = TrainingDataset(datasets=[difference_record], cleaned_lookup=cleaned_lookup) difference_record = self.epc_record.create_EPCDifferenceRecord(
self.epc_record, fixed_data
)
self.base_difference_record = TrainingDataset(
datasets=[difference_record], cleaned_lookup=cleaned_lookup
)
# TODO: adjust the base difference record with the previously calculated u values + features # TODO: adjust the base difference record with the previously calculated u values + features
# estimated_perimeter is different to the perimeter in the epc record # estimated_perimeter is different to the perimeter in the epc record
@ -147,9 +174,7 @@ class Property:
# self.base_difference_record.df # self.base_difference_record.df
def adjust_difference_record_with_recommendations( def adjust_difference_record_with_recommendations(
self, self, property_recommendations, property_representative_recommendations
property_recommendations,
property_representative_recommendations
): ):
""" """
This method will adjust the difference record, based on the recommendations made for the property This method will adjust the difference record, based on the recommendations made for the property
@ -161,13 +186,23 @@ class Property:
""" """
self.recommendations_scoring_data = [] self.recommendations_scoring_data = []
phases = sorted([r[0]["phase"] for r in property_recommendations if r[0]["phase"] is not None]) phases = sorted(
[
r[0]["phase"]
for r in property_recommendations
if r[0]["phase"] is not None
]
)
for phase in phases: for phase in phases:
property_recommendations_by_phase = [r for r in property_recommendations if r[0]["phase"] == phase][0] property_recommendations_by_phase = [
r for r in property_recommendations if r[0]["phase"] == phase
][0]
previous_phases = [p for p in phases if p < phase] previous_phases = [p for p in phases if p < phase]
previous_phase_representatives = [ previous_phase_representatives = [
r for r in property_representative_recommendations if r["phase"] in previous_phases r
for r in property_representative_recommendations
if r["phase"] in previous_phases
] ]
# For solid wall insulation, we will actually have 2 representative recommendations, since we consider # For solid wall insulation, we will actually have 2 representative recommendations, since we consider
# both internal and external wall insulation as possible measures. We will use the representative that # both internal and external wall insulation as possible measures. We will use the representative that
@ -175,15 +210,20 @@ class Property:
# Take the representative with the lowest efficiency, by phase # Take the representative with the lowest efficiency, by phase
# To be safe, we sort by phase # To be safe, we sort by phase
previous_phase_representatives = sorted(previous_phase_representatives, key=lambda x: x['phase']) previous_phase_representatives = sorted(
previous_phase_representatives, key=lambda x: x["phase"]
)
previous_phase_representatives = [ previous_phase_representatives = [
min(group, key=lambda x: x['efficiency']) for _, group in groupby( min(group, key=lambda x: x["efficiency"])
previous_phase_representatives, key=lambda x: x['phase'] for _, group in groupby(
previous_phase_representatives, key=lambda x: x["phase"]
) )
] ]
recommendation_record = self.base_difference_record.df.to_dict("records")[0].copy() recommendation_record = self.base_difference_record.df.to_dict("records")[
0
].copy()
for rec in property_recommendations_by_phase: for rec in property_recommendations_by_phase:
# We simulate the impact of the recommendation at this current phase, and all of the prior phases # We simulate the impact of the recommendation at this current phase, and all of the prior phases
@ -195,13 +235,16 @@ class Property:
property_id=self.id, property_id=self.id,
recommendation_record=recommendation_record, recommendation_record=recommendation_record,
recommendations=previous_phase_representatives + [rec], recommendations=previous_phase_representatives + [rec],
primary_recommendation_id=rec["recommendation_id"] primary_recommendation_id=rec["recommendation_id"],
) )
self.recommendations_scoring_data.append(scoring_dict) self.recommendations_scoring_data.append(scoring_dict)
@staticmethod @staticmethod
def create_recommendation_scoring_data( def create_recommendation_scoring_data(
property_id, recommendation_record, recommendations: list, primary_recommendation_id: int property_id,
recommendation_record,
recommendations: list,
primary_recommendation_id: int,
): ):
""" """
This function will iterate through a list of recommendations and apply a simulation for each recommendation This function will iterate through a list of recommendations and apply a simulation for each recommendation
@ -216,7 +259,9 @@ class Property:
output = recommendation_record.copy() output = recommendation_record.copy()
for col in [ for col in [
"walls_insulation_thickness", "floor_insulation_thickness", "roof_insulation_thickness" "walls_insulation_thickness",
"floor_insulation_thickness",
"roof_insulation_thickness",
]: ]:
if output[col] is None: if output[col] is None:
output[col] = "none" output[col] = "none"
@ -226,11 +271,15 @@ class Property:
# We update the description to indicate it's insulated # We update the description to indicate it's insulated
if recommendation["type"] in [ if recommendation["type"] in [
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation" "internal_wall_insulation",
"external_wall_insulation",
"cavity_wall_insulation",
]: ]:
# The upgrade made here is to the u-value of the walls and the description of the # The upgrade made here is to the u-value of the walls and the description of the
# insulation thickness # insulation thickness
output["walls_thermal_transmittance_ending"] = recommendation["new_u_value"] output["walls_thermal_transmittance_ending"] = recommendation[
"new_u_value"
]
# Setting the insulation thickness here to above average should be tested further because we # Setting the insulation thickness here to above average should be tested further because we
# don't see a high volume of instances for this # don't see a high volume of instances for this
output["walls_insulation_thickness_ending"] = "average" output["walls_insulation_thickness_ending"] = "average"
@ -263,10 +312,14 @@ class Property:
# Update description to indicate it's insulate # Update description to indicate it's insulate
if recommendation["type"] in [ if recommendation["type"] in [
"solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation" "solid_floor_insulation",
"suspended_floor_insulation",
"exposed_floor_insulation",
]: ]:
if len(recommendation["parts"]) > 1: if len(recommendation["parts"]) > 1:
raise NotImplementedError("Have more than 1 floor insulation part - handle this case") raise NotImplementedError(
"Have more than 1 floor insulation part - handle this case"
)
# output["floor_thermal_transmittance_ending"] = recommendation["new_u_value"] # output["floor_thermal_transmittance_ending"] = recommendation["new_u_value"]
# We don't really see above average for this in the training data # We don't really see above average for this in the training data
@ -280,22 +333,43 @@ class Property:
if output["floor_insulation_thickness_ending"] is None: if output["floor_insulation_thickness_ending"] is None:
output["floor_insulation_thickness_ending"] = "none" output["floor_insulation_thickness_ending"] = "none"
if recommendation["type"] in ["loft_insulation", "room_roof_insulation", "flat_roof_insulation"]: if recommendation["type"] in [
output["roof_thermal_transmittance_ending"] = recommendation["new_u_value"] "loft_insulation",
"room_roof_insulation",
"flat_roof_insulation",
]:
output["roof_thermal_transmittance_ending"] = recommendation[
"new_u_value"
]
parts = recommendation["parts"] parts = recommendation["parts"]
if len(parts) != 1: if len(parts) != 1:
raise ValueError("More than one part for roof insulation - investiage me") raise ValueError(
"More than one part for roof insulation - investiage me"
)
# This is based on the values we have in the training data # This is based on the values we have in the training data
valid_numeric_values = [ valid_numeric_values = [
12, 25, 50, 75, 100, 150, 200, 250, 270, 300, 350, 400 12,
25,
50,
75,
100,
150,
200,
250,
270,
300,
350,
400,
] ]
proposed_depth = int(parts[0]["depth"]) proposed_depth = int(parts[0]["depth"])
if proposed_depth not in valid_numeric_values: if proposed_depth not in valid_numeric_values:
# Take the nearest value for scoring # Take the nearest value for scoring
proposed_depth = min(valid_numeric_values, key=lambda x: abs(x - proposed_depth)) proposed_depth = min(
valid_numeric_values, key=lambda x: abs(x - proposed_depth)
)
output["roof_insulation_thickness_ending"] = str(proposed_depth) output["roof_insulation_thickness_ending"] = str(proposed_depth)
if recommendation["type"] == "loft_insulation": if recommendation["type"] == "loft_insulation":
@ -329,11 +403,17 @@ class Property:
if output["glazing_type_ending"] == "multiple": if output["glazing_type_ending"] == "multiple":
pass pass
elif output["glazing_type_ending"] == "single": elif output["glazing_type_ending"] == "single":
output["glazing_type_ending"] = "secondary" if is_secondary_glazing else "double" output["glazing_type_ending"] = (
"secondary" if is_secondary_glazing else "double"
)
elif output["glazing_type_ending"] == "double": elif output["glazing_type_ending"] == "double":
output["glazing_type_ending"] = "multiple" if is_secondary_glazing else "double" output["glazing_type_ending"] = (
"multiple" if is_secondary_glazing else "double"
)
elif output["glazing_type_ending"] == "secondary": elif output["glazing_type_ending"] == "secondary":
output["glazing_type_ending"] = "secondary" if is_secondary_glazing else "multiple" output["glazing_type_ending"] = (
"secondary" if is_secondary_glazing else "multiple"
)
elif output["glazing_type_ending"] in ["triple", "high performance"]: elif output["glazing_type_ending"] in ["triple", "high performance"]:
output["glazing_type_ending"] = "multiple" output["glazing_type_ending"] = "multiple"
else: else:
@ -342,7 +422,9 @@ class Property:
if is_secondary_glazing: if is_secondary_glazing:
output["glazed_type_ending"] = "secondary glazing" output["glazed_type_ending"] = "secondary glazing"
else: else:
output["glazed_type_ending"] = "double glazing installed during or after 2002" output["glazed_type_ending"] = (
"double glazing installed during or after 2002"
)
if recommendation["type"] in ["heating", "hot_water_tank_insulation", "heating_control"]: if recommendation["type"] in ["heating", "hot_water_tank_insulation", "heating_control"]:
# We update the data, as defined in the recommendaton # We update the data, as defined in the recommendaton
@ -367,13 +449,17 @@ class Property:
"windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation", "windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation",
"heating_control", "heating_control",
]: ]:
raise NotImplementedError("Implement me, given type %s" % recommendation["type"]) raise NotImplementedError(
"Implement me, given type %s" % recommendation["type"]
)
output['id'] = "+".join([str(property_id), str(primary_recommendation_id)]) output["id"] = "+".join([str(property_id), str(primary_recommendation_id)])
return output return output
def get_components(self, cleaned, photo_supply_lookup, floor_area_decile_thresholds): def get_components(
self, cleaned, photo_supply_lookup, floor_area_decile_thresholds
):
""" """
Given the cleaning that has been performed, we'll use this to identify the property Given the cleaning that has been performed, we'll use this to identify the property
components, from roof to walls to windows, heating and hot water components, from roof to walls to windows, heating and hot water
@ -398,10 +484,12 @@ class Property:
if self.data[description] in self.DATA_ANOMALY_MATCHES: if self.data[description] in self.DATA_ANOMALY_MATCHES:
template = cleaned[description][0] template = cleaned[description][0]
fill_dict = dict(zip(template.keys(), [None] * len(template))) fill_dict = dict(zip(template.keys(), [None] * len(template)))
fill_dict.update({ fill_dict.update(
"original_description": self.data[description], {
"clean_description": self.data[description], "original_description": self.data[description],
}) "clean_description": self.data[description],
}
)
setattr( setattr(
self, self,
self.ATTRIBUTE_MAP[description], self.ATTRIBUTE_MAP[description],
@ -410,11 +498,15 @@ class Property:
continue continue
attributes = [ attributes = [
x for x in cleaned[description] if x["original_description"] == self.data[description] x
for x in cleaned[description]
if x["original_description"] == self.data[description]
] ]
if len(attributes) > 1: if len(attributes) > 1:
raise ValueError("Either No attributes or multiple found for %s" % description) raise ValueError(
"Either No attributes or multiple found for %s" % description
)
if len(attributes) == 0: if len(attributes) == 0:
# We attempt to perform the clean on the fly # We attempt to perform the clean on the fly
@ -422,8 +514,12 @@ class Property:
cleaner_cls = cleaner_cls(self.data[description]) cleaner_cls = cleaner_cls(self.data[description])
processed = { processed = {
"original_description": self.data[description], "original_description": self.data[description],
"clean_description": cleaner_cls.description.replace("(assumed)", "").rstrip().capitalize(), "clean_description": cleaner_cls.description.replace(
**cleaner_cls.process() "(assumed)", ""
)
.rstrip()
.capitalize(),
**cleaner_cls.process(),
} }
attributes = [processed] attributes = [processed]
@ -435,7 +531,8 @@ class Property:
self.set_floor_level() self.set_floor_level()
self.set_windows_count() self.set_windows_count()
self.set_solar_panel_area( self.set_solar_panel_area(
photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds,
) )
self.set_energy_source() self.set_energy_source()
@ -452,7 +549,11 @@ class Property:
self.is_heritage = spatial["is_heritage_building"].values[0] self.is_heritage = spatial["is_heritage_building"].values[0]
# We do an equals True, in the case of one of these variables being True # We do an equals True, in the case of one of these variables being True
if (self.in_conservation_area == True) | (self.is_listed == True) | (self.is_heritage == True): if (
(self.in_conservation_area == True)
| (self.is_listed == True)
| (self.is_heritage == True)
):
self.restricted_measures = True self.restricted_measures = True
spatial_dict = spatial.to_dict("records")[0] spatial_dict = spatial.to_dict("records")[0]
@ -494,7 +595,7 @@ class Property:
"tenure": self.data["tenure"], "tenure": self.data["tenure"],
"current_epc_rating": self.data["current-energy-rating"], "current_epc_rating": self.data["current-energy-rating"],
"current_sap_points": self.data["current-energy-efficiency"], "current_sap_points": self.data["current-energy-efficiency"],
"current_valuation": current_valuation "current_valuation": current_valuation,
} }
property_data = self._clean_upload_data(property_data) property_data = self._clean_upload_data(property_data)
@ -506,7 +607,11 @@ class Property:
""" """
Utility function for usage in the lambda, for preparing the _rating fields Utility function for usage in the lambda, for preparing the _rating fields
""" """
return rating_lookup[field].value if (field not in cls.DATA_ANOMALY_MATCHES) and (field is not None) else None return (
rating_lookup[field].value
if (field not in cls.DATA_ANOMALY_MATCHES) and (field is not None)
else None
)
def get_property_details_epc(self, portfolio_id: int, rating_lookup): def get_property_details_epc(self, portfolio_id: int, rating_lookup):
@ -516,21 +621,37 @@ class Property:
"full_address": self.data["address"], "full_address": self.data["address"],
"total_floor_area": float(self.data["total-floor-area"]), "total_floor_area": float(self.data["total-floor-area"]),
"walls": self.walls["clean_description"], "walls": self.walls["clean_description"],
"walls_rating": self._prepare_rating_field(self.data["walls-energy-eff"], rating_lookup), "walls_rating": self._prepare_rating_field(
self.data["walls-energy-eff"], rating_lookup
),
"roof": self.roof["clean_description"], "roof": self.roof["clean_description"],
"roof_rating": self._prepare_rating_field(self.data["roof-energy-eff"], rating_lookup), "roof_rating": self._prepare_rating_field(
self.data["roof-energy-eff"], rating_lookup
),
"floor": self.floor["clean_description"], "floor": self.floor["clean_description"],
"floor_rating": self._prepare_rating_field(self.data["floor-energy-eff"], rating_lookup), "floor_rating": self._prepare_rating_field(
self.data["floor-energy-eff"], rating_lookup
),
"windows": self.windows["clean_description"], "windows": self.windows["clean_description"],
"windows_rating": self._prepare_rating_field(self.data["windows-energy-eff"], rating_lookup), "windows_rating": self._prepare_rating_field(
self.data["windows-energy-eff"], rating_lookup
),
"heating": self.main_heating["clean_description"], "heating": self.main_heating["clean_description"],
"heating_rating": self._prepare_rating_field(self.data["mainheat-energy-eff"], rating_lookup), "heating_rating": self._prepare_rating_field(
self.data["mainheat-energy-eff"], rating_lookup
),
"heating_controls": self.main_heating_controls["clean_description"], "heating_controls": self.main_heating_controls["clean_description"],
"heating_controls_rating": self._prepare_rating_field(self.data["mainheatc-energy-eff"], rating_lookup), "heating_controls_rating": self._prepare_rating_field(
self.data["mainheatc-energy-eff"], rating_lookup
),
"hot_water": self.hotwater["clean_description"], "hot_water": self.hotwater["clean_description"],
"hot_water_rating": self._prepare_rating_field(self.data["hot-water-energy-eff"], rating_lookup), "hot_water_rating": self._prepare_rating_field(
self.data["hot-water-energy-eff"], rating_lookup
),
"lighting": self.lighting["clean_description"], "lighting": self.lighting["clean_description"],
"lighting_rating": self._prepare_rating_field(self.data["lighting-energy-eff"], rating_lookup), "lighting_rating": self._prepare_rating_field(
self.data["lighting-energy-eff"], rating_lookup
),
"mainfuel": self.main_fuel["clean_description"], "mainfuel": self.main_fuel["clean_description"],
"ventilation": self.ventilation["ventilation"], "ventilation": self.ventilation["ventilation"],
"solar_pv": self.solar_pv["solar_pv"], "solar_pv": self.solar_pv["solar_pv"],
@ -539,7 +660,9 @@ class Property:
"floor_height": self.floor_height, "floor_height": self.floor_height,
"heat_loss_corridor": self.heat_loss_corridor["heat_loss_corridor_boolean"], "heat_loss_corridor": self.heat_loss_corridor["heat_loss_corridor_boolean"],
"unheated_corridor_length": self.heat_loss_corridor["length"], "unheated_corridor_length": self.heat_loss_corridor["length"],
"number_of_open_fireplaces": self.number_of_open_fireplaces["number_of_open_fireplaces"], "number_of_open_fireplaces": self.number_of_open_fireplaces[
"number_of_open_fireplaces"
],
"number_of_extensions": self.number_of_extensions["number_of_extensions"], "number_of_extensions": self.number_of_extensions["number_of_extensions"],
"number_of_storeys": self.number_of_storeys["number_of_storeys"], "number_of_storeys": self.number_of_storeys["number_of_storeys"],
"mains_gas": self.mains_gas, "mains_gas": self.mains_gas,
@ -547,20 +670,21 @@ class Property:
"primary_energy_consumption": self.energy["primary_energy_consumption"], "primary_energy_consumption": self.energy["primary_energy_consumption"],
"co2_emissions": self.energy["co2_emissions"], "co2_emissions": self.energy["co2_emissions"],
"adjusted_energy_consumption": self.current_adjusted_energy, "adjusted_energy_consumption": self.current_adjusted_energy,
"estimated": self.data.get("estimated", False) "estimated": self.data.get("estimated", False),
} }
return property_details_epc return property_details_epc
def get_spatial_data(self, uprn_filenames): def get_spatial_data(self, uprn_filenames):
""" """
Given a property's UPRN, this method will pull the associated spatial data from s3 Given a property's UPRN, this method will pull the associated spatial data from s3
:return: :return:
""" """
if self.uprn is None: if self.uprn is None:
logger.warning("We do not have a UPRN for this property - this needs to be implemented") logger.warning(
"We do not have a UPRN for this property - this needs to be implemented"
)
self.in_conservation_area = False self.in_conservation_area = False
self.is_listed = False self.is_listed = False
self.is_heritage = False self.is_heritage = False
@ -568,12 +692,15 @@ class Property:
return return
# We get the file name for the uprn # We get the file name for the uprn
filtered_df = uprn_filenames[(uprn_filenames['lower'] <= self.uprn) & (uprn_filenames['upper'] >= self.uprn)] filtered_df = uprn_filenames[
(uprn_filenames["lower"] <= self.uprn)
& (uprn_filenames["upper"] >= self.uprn)
]
if filtered_df.empty: if filtered_df.empty:
logger.warning("Could not find file containing UPRNS") logger.warning("Could not find file containing UPRNS")
return None return None
filename = filtered_df.iloc[0]['filenames'] filename = filtered_df.iloc[0]["filenames"]
spatial_data = read_dataframe_from_s3_parquet( spatial_data = read_dataframe_from_s3_parquet(
bucket_name=DATA_BUCKET, file_key=f"spatial/{filename}" bucket_name=DATA_BUCKET, file_key=f"spatial/{filename}"
@ -591,15 +718,27 @@ class Property:
:return: filtered property dimensions dataframe :return: filtered property dimensions dataframe
""" """
result = property_dimensions[(property_dimensions["PROPERTY_TYPE"] == self.data["property-type"])] result = property_dimensions[
(property_dimensions["PROPERTY_TYPE"] == self.data["property-type"])
]
if self.construction_age_band is not None and self.construction_age_band not in self.DATA_ANOMALY_MATCHES: if (
result = result[(result["CONSTRUCTION_AGE_BAND"] == self.construction_age_band)] self.construction_age_band is not None
and self.construction_age_band not in self.DATA_ANOMALY_MATCHES
):
result = result[
(result["CONSTRUCTION_AGE_BAND"] == self.construction_age_band)
]
if self.data["built-form"] not in self.DATA_ANOMALY_MATCHES and self.data["built-form"] in result["BUILT_FORM"]: if (
self.data["built-form"] not in self.DATA_ANOMALY_MATCHES
and self.data["built-form"] in result["BUILT_FORM"]
):
result = result[(result["BUILT_FORM"] == self.data["built-form"])] result = result[(result["BUILT_FORM"] == self.data["built-form"])]
return result[["NUMBER_HABITABLE_ROOMS", "TOTAL_FLOOR_AREA", "FLOOR_HEIGHT"]].mean() return result[
["NUMBER_HABITABLE_ROOMS", "TOTAL_FLOOR_AREA", "FLOOR_HEIGHT"]
].mean()
def set_basic_property_dimensions(self): def set_basic_property_dimensions(self):
""" """
@ -618,7 +757,8 @@ class Property:
# They could also be added as attributes to the EPC Record # They could also be added as attributes to the EPC Record
self.perimeter = estimate_perimeter( self.perimeter = estimate_perimeter(
self.floor_area / self.number_of_floors, self.number_of_rooms / self.number_of_floors self.floor_area / self.number_of_floors,
self.number_of_rooms / self.number_of_floors,
) )
self.insulation_wall_area = estimate_external_wall_area( self.insulation_wall_area = estimate_external_wall_area(
@ -636,8 +776,9 @@ class Property:
def set_floor_level(self): def set_floor_level(self):
self.floor_level = ( self.floor_level = (
FLOOR_LEVEL_MAP[self.data["floor-level"]] if FLOOR_LEVEL_MAP[self.data["floor-level"]]
self.data["floor-level"] not in self.DATA_ANOMALY_MATCHES and self.data['floor-level'] is not None if self.data["floor-level"] not in self.DATA_ANOMALY_MATCHES
and self.data["floor-level"] is not None
else None else None
) )
@ -699,12 +840,16 @@ class Property:
raise NotImplementedError("Implement this floor type") raise NotImplementedError("Implement this floor type")
@staticmethod @staticmethod
def _extract_component(component_data, component_rename_cols, component_drop_cols, rename_prefix=None): def _extract_component(
component_data, component_rename_cols, component_drop_cols, rename_prefix=None
):
for k in component_rename_cols: for k in component_rename_cols:
component_data[f"{rename_prefix}_{k}"] = component_data.get(k) component_data[f"{rename_prefix}_{k}"] = component_data.get(k)
component_data = { component_data = {
k: v for k, v in component_data.items() if k not in component_drop_cols + component_rename_cols k: v
for k, v in component_data.items()
if k not in component_drop_cols + component_rename_cols
} }
return component_data return component_data
@ -752,7 +897,7 @@ class Property:
is_flat=self.roof["is_flat"], is_flat=self.roof["is_flat"],
is_pitched=self.roof["is_pitched"], is_pitched=self.roof["is_pitched"],
is_roof_room=self.roof["is_roof_room"], is_roof_room=self.roof["is_roof_room"],
floor_area=self.floor_area floor_area=self.floor_area,
) )
percentage_of_roof = photo_supply_matched["photo_supply_median"].mean() percentage_of_roof = photo_supply_matched["photo_supply_median"].mean()
@ -768,8 +913,9 @@ class Property:
""" """
return ( return (
self.insulation_floor_area * percentage_of_roof if self.roof["is_flat"] else self.insulation_floor_area * percentage_of_roof
self.pitched_roof_area * percentage_of_roof if self.roof["is_flat"]
else self.pitched_roof_area * percentage_of_roof
) )
def set_energy_source(self): def set_energy_source(self):
@ -782,7 +928,12 @@ class Property:
# If the tariff explicitly indicates electricity use without a dual indication and mains_gas_flag is not True # If the tariff explicitly indicates electricity use without a dual indication and mains_gas_flag is not True
# We check for the common electricity tariffs # We check for the common electricity tariffs
if not self.data["mains-gas-flag"] and self.data["energy-tariff"] in [ if not self.data["mains-gas-flag"] and self.data["energy-tariff"] in [
"Single", "off-peak 7 hour", "off-peak 10 hour", "off-peak 18 hour", "standard tariff", "24 hour" "Single",
"off-peak 7 hour",
"off-peak 10 hour",
"off-peak 18 hour",
"standard tariff",
"24 hour",
]: ]:
energy_source = "electricity" energy_source = "electricity"

2
etl/epc/Dataset.py
View file

@ -563,7 +563,6 @@ class TrainingDataset(BaseDataset):
"original_description_ending", "original_description_ending",
"thermal_transmittance_unit_ending", "thermal_transmittance_unit_ending",
"is_cavity_wall_ending", "is_cavity_wall_ending",
"is_filled_cavity_ending",
"is_solid_brick_ending", "is_solid_brick_ending",
"is_system_built_ending", "is_system_built_ending",
"is_timber_frame_ending", "is_timber_frame_ending",
@ -607,7 +606,6 @@ class TrainingDataset(BaseDataset):
"is_loft_ending", "is_loft_ending",
"is_flat_ending", "is_flat_ending",
"is_thatched_ending", "is_thatched_ending",
"is_at_rafters_ending",
"has_dwelling_above_ending", "has_dwelling_above_ending",
"is_assumed_ending", "is_assumed_ending",
"is_valid_ending", "is_valid_ending",

19
etl/epc/Pipeline.py
View file

@ -1,5 +1,6 @@
import msgpack import msgpack
import pandas as pd import pandas as pd
from datetime import datetime
from typing import List from typing import List
from pathlib import Path from pathlib import Path
@ -25,7 +26,8 @@ from etl.epc.settings import (
# TODO: change in setting file # TODO: change in setting file
MANDATORY_FIXED_FEATURES = [x.lower() for x in MANDATORY_FIXED_FEATURES] MANDATORY_FIXED_FEATURES = [x.lower() for x in MANDATORY_FIXED_FEATURES]
LATEST_FIELD = [x.lower() for x in LATEST_FIELD if x.lower() not in ROOM_FEATURES] # LATEST_FIELD = [x.lower() for x in LATEST_FIELD if x.lower() not in ROOM_FEATURES]
LATEST_FIELD = [x.lower() for x in LATEST_FIELD]
COMPONENT_FEATURES = [x.lower() for x in COMPONENT_FEATURES] COMPONENT_FEATURES = [x.lower() for x in COMPONENT_FEATURES]
RDSAP_RESPONSE = RDSAP_RESPONSE.lower() RDSAP_RESPONSE = RDSAP_RESPONSE.lower()
HEAT_DEMAND_RESPONSE = HEAT_DEMAND_RESPONSE.lower() HEAT_DEMAND_RESPONSE = HEAT_DEMAND_RESPONSE.lower()
@ -81,9 +83,9 @@ class EPCPipeline:
run_mode="training", run_mode="training",
epc_local_file="certificates.csv", epc_local_file="certificates.csv",
epc_bucket_name="retrofit-data-dev", epc_bucket_name="retrofit-data-dev",
epc_cleaning_dataset_key="sap_change_model/cleaning_dataset_rooms.parquet", epc_cleaning_dataset_key="sap_change_model/{}/cleaning_dataset_rooms.parquet",
epc_all_equal_rows_key="sap_change_model/all_equal_rows_rooms.parquet", epc_all_equal_rows_key="sap_change_model/{}/all_equal_rows_rooms.parquet",
epc_compiled_dataset_key="sap_change_model/dataset_rooms.parquet", epc_compiled_dataset_key="sap_change_model/{}/dataset_rooms.parquet",
use_parallel=False, use_parallel=False,
): ):
""" """
@ -106,10 +108,13 @@ class EPCPipeline:
self.run_mode = run_mode self.run_mode = run_mode
self.epc_local_file = epc_local_file self.epc_local_file = epc_local_file
self.epc_bucket_name = epc_bucket_name self.epc_bucket_name = epc_bucket_name
self.epc_cleaning_dataset_key = epc_cleaning_dataset_key
self.epc_all_equal_rows_key = epc_all_equal_rows_key
self.epc_compiled_dataset_key = epc_compiled_dataset_key
self.use_parallel = use_parallel self.use_parallel = use_parallel
self.timeprefix = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
self.epc_cleaning_dataset_key = epc_cleaning_dataset_key.format(self.timeprefix)
self.epc_all_equal_rows_key = epc_all_equal_rows_key.format(self.timeprefix)
self.epc_compiled_dataset_key = epc_compiled_dataset_key.format(self.timeprefix)
def run(self): def run(self):
""" """

62
etl/epc/generate_scenarios_data.py
View file

@ -20,6 +20,10 @@ from recommendations.Recommendations import Recommendations
from utils.logger import setup_logger from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet, save_dataframe_to_s3_parquet from utils.s3 import read_dataframe_from_s3_parquet, save_dataframe_to_s3_parquet
from datetime import datetime
now = datetime.now().strftime("%d-%m-%Y-%H-%M-%S")
logger = setup_logger() logger = setup_logger()
logger.info("Connecting to db") logger.info("Connecting to db")
@ -50,9 +54,19 @@ scenario_properties = [
"postcode": "NN1 5JY", "postcode": "NN1 5JY",
"lmk-key": "1459796789102016070507274146560098", "lmk-key": "1459796789102016070507274146560098",
"measures": [ "measures": [
[["internal_wall_insulation"], "11", None, [0]], [
[["external_wall_insulation"], "10", None, [0]], ["internal_wall_insulation"],
[["solar", "windows"], "12-15", {"photo_supply_ending": 50}, [0, 1]], "11",
{"walls_insulation_thickness_ending": "average"},
[0],
],
[
["external_wall_insulation"],
"10",
{"walls_insulation_thickness_ending": "average"},
[0],
],
[["solar", "windows"], "15", {"photo_supply_ending": 50}, [0, 1]],
], ],
}, },
{ {
@ -60,7 +74,12 @@ scenario_properties = [
"postcode": "HP1 2HA", "postcode": "HP1 2HA",
"lmk-key": "c14029235739827d5f627dc8aa9bb567d026b267e851e0db0001db24638667b1", "lmk-key": "c14029235739827d5f627dc8aa9bb567d026b267e851e0db0001db24638667b1",
"measures": [ "measures": [
[["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], [
["cavity_wall_insulation", "loft_insulation"],
"15",
{"walls_insulation_thickness_ending": "average"},
[0, 1],
],
], ],
}, },
{ {
@ -68,7 +87,12 @@ scenario_properties = [
"postcode": "HP1 2HE", "postcode": "HP1 2HE",
"lmk-key": "99296a6dda21314fef3a61cda59e441e9a2aacf115eb96f4a0fa85696bf7b117", "lmk-key": "99296a6dda21314fef3a61cda59e441e9a2aacf115eb96f4a0fa85696bf7b117",
"measures": [ "measures": [
[["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], [
["cavity_wall_insulation", "loft_insulation"],
"15",
{"walls_insulation_thickness_ending": "average"},
[0, 1],
],
], ],
}, },
{ {
@ -76,7 +100,12 @@ scenario_properties = [
"postcode": "HP1 2AN", "postcode": "HP1 2AN",
"lmk-key": "d1e0534be3a44c33003323b21d0e322e3daddc65b5ee71936f89c59ddab96b50", "lmk-key": "d1e0534be3a44c33003323b21d0e322e3daddc65b5ee71936f89c59ddab96b50",
"measures": [ "measures": [
[["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], [
["cavity_wall_insulation", "loft_insulation"],
"15",
{"walls_insulation_thickness_ending": "average"},
[0, 1],
],
], ],
}, },
{ {
@ -84,11 +113,17 @@ scenario_properties = [
"postcode": "HP1 2HX", "postcode": "HP1 2HX",
"lmk-key": "1eae354db522a95188018d9cd0502ed8c609910b6c88f8797d3a25f59b11770a", "lmk-key": "1eae354db522a95188018d9cd0502ed8c609910b6c88f8797d3a25f59b11770a",
"measures": [ "measures": [
[["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], [
["cavity_wall_insulation", "loft_insulation"],
"15",
{"walls_insulation_thickness_ending": "average"},
[0, 1],
],
], ],
}, },
] ]
recommendations_scoring_data = [] recommendations_scoring_data = []
for scenario_property in scenario_properties: for scenario_property in scenario_properties:
@ -132,7 +167,7 @@ for scenario_property in scenario_properties:
p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
recommender = Recommendations(property_instance=p, materials=materials) recommender = Recommendations(property_instance=p, materials=materials)
property_recommendations = recommender.recommend() property_recommendations = recommender.recommend("0")
wall_recommendations = recommender.wall_recomender.recommendations wall_recommendations = recommender.wall_recomender.recommendations
loft_recommendations = recommender.roof_recommender.recommendations loft_recommendations = recommender.roof_recommender.recommendations
@ -183,7 +218,7 @@ for scenario_property in scenario_properties:
if "windows" in measure: if "windows" in measure:
for rec in windows_recommendations: for rec in windows_recommendations:
if rec["type"] == "windows": if rec["type"] == "windows_glazing":
windows_recs.append(rec) windows_recs.append(rec)
combi_list = [wall_recs, loft_recs, solar_recs, windows_recs] combi_list = [wall_recs, loft_recs, solar_recs, windows_recs]
@ -213,6 +248,9 @@ for scenario_property in scenario_properties:
recommendations_scoring_data.extend(scoring_list) recommendations_scoring_data.extend(scoring_list)
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
recommendations_scoring_data["impact"] = recommendations_scoring_data["impact"].astype(
int
)
recommendations_scoring_data = recommendations_scoring_data.drop( recommendations_scoring_data = recommendations_scoring_data.drop(
columns=[ columns=[
"rdsap_change", "rdsap_change",
@ -240,12 +278,12 @@ all_predictions = model_api.predict_all(
prediction_buckets={ prediction_buckets={
"sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET,
"heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET,
"carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET,
} },
) )
save_dataframe_to_s3_parquet( save_dataframe_to_s3_parquet(
recommendations_scoring_data, recommendations_scoring_data,
"retrofit-data-dev", "retrofit-data-dev",
"scenario_data/recommendations_scoring_data.parquet", f"scenario_data/{now}/recommendations_scoring_data.parquet",
) )

3
etl/epc/requirements.txt
View file

@ -1,4 +1,5 @@
pandas==2.1.3 pandas==2.1.3
tqdm==4.66.1 tqdm==4.66.1
msgpack==1.0.7 msgpack==1.0.7
boto3==1.29.6 boto3==1.29.6
pyarrow==15.0.2