junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix weird cases for now

Browse source
This commit is contained in:
Michael Duong 2024-02-22 20:22:11 +00:00
parent 955e72f0bb
commit ed407bc98b
3 changed files with 145 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

215
etl/epc/Dataset.py
View file

@ -809,6 +809,7 @@ class TrainingDataset(BaseDataset):
# else: # else:
# return self.__add__(other) # return self.__add__(other)
class RecordDataset(BaseDataset): class RecordDataset(BaseDataset):
""" """
A collection of EPCRecrods can be combined into a Dataset. A collection of EPCRecrods can be combined into a Dataset.
@ -824,25 +825,25 @@ class RecordDataset(BaseDataset):
self._expand_description_to_features(cleaned_lookup) self._expand_description_to_features(cleaned_lookup)
self._adjust_assumed_values_in_wall_descriptions() self._adjust_assumed_values_in_wall_descriptions()
self._generate_u_values_from_features() self._generate_u_values_from_features()
# # TODO: For some of the features that we clean, we have either a true, false or possibly null value # # # TODO: For some of the features that we clean, we have either a true, false or possibly null value
# # Those nulls should be False. clean_missings_after_description_process handles this but shouldn't # # # Those nulls should be False. clean_missings_after_description_process handles this but shouldn't
# # need to # # # need to
self._clean_missing_values() self._clean_missing_values()
self._null_validation(information="Clean Missing Values") self._null_validation(information="Clean Missing Values")
# self._remove_abnormal_change_in_floor_area() # # self._remove_abnormal_change_in_floor_area()
self._ensure_numeric() self._ensure_numeric()
def _ensure_numeric(self): def _ensure_numeric(self):
""" """
Ensure that all columns are numeric Ensure that all columns are numeric
""" """
# TODO: move into EPCRecord record # TODO: move into EPCRecord record
uvalue_columns = [col for col in self.df.columns if "thermal_transmittance" in col] uvalue_columns = [
col for col in self.df.columns if "thermal_transmittance" in col
]
for uvalue_col in uvalue_columns: for uvalue_col in uvalue_columns:
self.df[uvalue_col] = pd.to_numeric(self.df[uvalue_col]) self.df[uvalue_col] = pd.to_numeric(self.df[uvalue_col])
def _clean_missing_values(self, ignore_cols=None): def _clean_missing_values(self, ignore_cols=None):
missings = pd.isnull(self.df).sum() missings = pd.isnull(self.df).sum()
missings = missings[missings > 0] missings = missings[missings > 0]
@ -859,17 +860,22 @@ class RecordDataset(BaseDataset):
else: else:
self.df[col] = self.df[col].fillna("Unknown") self.df[col] = self.df[col].fillna("Unknown")
@staticmethod @staticmethod
def _lambda_function_to_generate_roof_uvalue(row, is_end=False): def _lambda_function_to_generate_roof_uvalue(row, is_end=False):
""" """
Using the apply method, use the get_roof_u_value method to generate the u-value Using the apply method, use the get_roof_u_value method to generate the u-value
""" """
col_name = "roof_insulation_thickness" if not is_end else "roof_insulation_thickness_ending" col_name = (
"roof_insulation_thickness"
if not is_end
else "roof_insulation_thickness_ending"
)
if row["has_dwelling_above"]: if row["has_dwelling_above"]:
if row["roof_thermal_transmittance"] != 0: if (row["roof_thermal_transmittance"] != 0) & (
not pd.isnull(row["roof_thermal_transmittance"])
):
raise ValueError("Should have 0 u-value for roof") raise ValueError("Should have 0 u-value for roof")
return get_roof_u_value( return get_roof_u_value(
@ -881,16 +887,24 @@ class RecordDataset(BaseDataset):
is_flat=row["is_flat"], is_flat=row["is_flat"],
is_pitched=row["is_pitched"], is_pitched=row["is_pitched"],
is_at_rafters=row["is_at_rafters"], is_at_rafters=row["is_at_rafters"],
age_band=england_wales_age_band_lookup[row["construction_age_band"]] age_band=england_wales_age_band_lookup[row["construction_age_band"]],
) )
@staticmethod @staticmethod
def _lambda_function_to_generate_wall_uvalue(row, is_end=False): def _lambda_function_to_generate_wall_uvalue(row, is_end=False):
""" """
Using the apply method, use the get_wall_u_value method to generate the u-value Using the apply method, use the get_wall_u_value method to generate the u-value
""" """
description_col_name = "walls_clean_description" if not is_end else "walls_clean_description_ending" description_col_name = (
thermal_transistance_col_name = "walls_thermal_transmittance" if not is_end else "walls_thermal_transmittance_ending" "walls_clean_description"
if not is_end
else "walls_clean_description_ending"
)
thermal_transistance_col_name = (
"walls_thermal_transmittance"
if not is_end
else "walls_thermal_transmittance_ending"
)
if pd.isnull(row[thermal_transistance_col_name]): if pd.isnull(row[thermal_transistance_col_name]):
output = get_wall_u_value( output = get_wall_u_value(
@ -903,17 +917,23 @@ class RecordDataset(BaseDataset):
output = row[thermal_transistance_col_name] output = row[thermal_transistance_col_name]
return output return output
@staticmethod @staticmethod
def _lambda_function_to_generate_floor_uvalue(row, is_end=False): def _lambda_function_to_generate_floor_uvalue(row, is_end=False):
""" """
Using the apply method, use the get_floor_u_value method to generate the u-value Using the apply method, use the get_floor_u_value method to generate the u-value
""" """
floor_thermal_col_name = "floor_thermal_transmittance" if not is_end else "floor_thermal_transmittance_ending" floor_thermal_col_name = (
"floor_thermal_transmittance"
if not is_end
else "floor_thermal_transmittance_ending"
)
if row["another_property_below"]: if row["another_property_below"]:
if row["floor_thermal_transmittance"] != 0: if (row["floor_thermal_transmittance"] != 0) & (
not pd.isnull(row["floor_thermal_transmittance"])
):
raise ValueError("Should have 0 u-value for floor") raise ValueError("Should have 0 u-value for floor")
return 0 return 0
@ -922,19 +942,27 @@ class RecordDataset(BaseDataset):
if pd.isnull(uvalue): if pd.isnull(uvalue):
insulation_col_name = "floor_insulation_thickness" if not is_end else "floor_insulation_thickness_ending" insulation_col_name = (
floor_area_col_name = "estimated_perimeter" if not is_end else "estimated_perimeter_ending" "floor_insulation_thickness"
perimeter_col_name = "total_floor_area" if not is_end else "total_floor_area_ending" if not is_end
else "floor_insulation_thickness_ending"
)
floor_area_col_name = (
"estimated_perimeter" if not is_end else "estimated_perimeter_ending"
)
perimeter_col_name = (
"total_floor_area" if not is_end else "total_floor_area_ending"
)
uvalue = get_floor_u_value( uvalue = get_floor_u_value(
floor_type=row["floor_type"], floor_type=row["floor_type"],
perimeter=row[floor_area_col_name], perimeter=row[floor_area_col_name],
area=row[perimeter_col_name], area=row[perimeter_col_name],
insulation_thickness=row[insulation_col_name], insulation_thickness=row[insulation_col_name],
wall_type=row["wall_type"], wall_type=row["wall_type"],
age_band=england_wales_age_band_lookup[row["construction_age_band"]] age_band=england_wales_age_band_lookup[row["construction_age_band"]],
) )
return uvalue return uvalue
def _generate_u_values_from_features(self): def _generate_u_values_from_features(self):
@ -947,58 +975,63 @@ class RecordDataset(BaseDataset):
# ~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~
walls_uvalue = self.df.apply( walls_uvalue = self.df.apply(
lambda row: self._lambda_function_to_generate_wall_uvalue(row), lambda row: self._lambda_function_to_generate_wall_uvalue(row), axis=1
axis=1
) )
walls_uvalue = self.df['walls_thermal_transmittance'].fillna(walls_uvalue) walls_uvalue = self.df["walls_thermal_transmittance"].fillna(walls_uvalue)
# ~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~
# Roof # Roof
# ~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~
roof_uvalue = self.df.apply( roof_uvalue = self.df.apply(
lambda row: self._lambda_function_to_generate_roof_uvalue(row), lambda row: self._lambda_function_to_generate_roof_uvalue(row), axis=1
axis=1
) )
roof_uvalue = self.df['roof_thermal_transmittance'].fillna(roof_uvalue) roof_uvalue = self.df["roof_thermal_transmittance"].fillna(roof_uvalue)
# ~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~
# Floor # Floor
# ~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~
self.df['estimated_perimeter'] = self.df.apply( self.df["estimated_perimeter"] = self.df.apply(
lambda row: estimate_perimeter(row["total_floor_area"], row["number_habitable_rooms"]), lambda row: estimate_perimeter(
axis=1 row["total_floor_area"], row["number_habitable_rooms"]
),
axis=1,
) )
self.df["floor_type"] = self.df["is_suspended"].replace({True: "suspended", False: "solid"}) self.df["floor_type"] = self.df["is_suspended"].replace(
{True: "suspended", False: "solid"}
)
self.df["wall_type"] = self.df.apply( self.df["wall_type"] = self.df.apply(
lambda row: get_wall_type( lambda row: get_wall_type(
is_cavity_wall=row["is_cavity_wall"], is_cavity_wall=row["is_cavity_wall"],
is_solid_brick=row["is_solid_brick"], is_solid_brick=row["is_solid_brick"],
is_timber_frame=row["is_timber_frame"], is_timber_frame=row["is_timber_frame"],
is_granite_or_whinstone=row["is_granite_or_whinstone"], is_granite_or_whinstone=row["is_granite_or_whinstone"],
is_cob=row["is_cob"], is_cob=row["is_cob"],
is_sandstone_or_limestone=row["is_sandstone_or_limestone"], is_sandstone_or_limestone=row["is_sandstone_or_limestone"],
is_system_built=row["is_system_built"], is_system_built=row["is_system_built"],
is_park_home=row["is_park_home"] is_park_home=row["is_park_home"],
), ),
axis=1 axis=1,
)
floor_uvalue = self.df.apply(
lambda row: self._lambda_function_to_generate_floor_uvalue(row),
axis=1
) )
floor_uvalue = self.df['floor_thermal_transmittance'].fillna(floor_uvalue) floor_uvalue = self.df.apply(
lambda row: self._lambda_function_to_generate_floor_uvalue(row), axis=1
)
floor_uvalue = self.df["floor_thermal_transmittance"].fillna(floor_uvalue)
for component in ["walls", "roof", "floor"]: for component in ["walls", "roof", "floor"]:
self.df[f"{component}_thermal_transmittance"] = self.df[f"{component}_thermal_transmittance"].fillna(eval(f"{component}_uvalue")) self.df[f"{component}_thermal_transmittance"] = self.df[
f"{component}_thermal_transmittance"
].fillna(eval(f"{component}_uvalue"))
self.df = self.df.drop(columns=["floor_type", "wall_type", "walls_clean_description"]) self.df = self.df.drop(
columns=["floor_type", "wall_type", "walls_clean_description"]
)
def _adjust_assumed_values_in_wall_descriptions(self): def _adjust_assumed_values_in_wall_descriptions(self):
""" """
@ -1007,7 +1040,6 @@ class RecordDataset(BaseDataset):
for col in ["walls_clean_description"]: for col in ["walls_clean_description"]:
self.df[col] = self.df[col].str.replace("(assumed)", "").str.rstrip() self.df[col] = self.df[col].str.replace("(assumed)", "").str.rstrip()
def _clean_efficiency_variables(self): def _clean_efficiency_variables(self):
""" """
These is scope to clean this by the model per corresponding description. These is scope to clean this by the model per corresponding description.
@ -1023,7 +1055,7 @@ class RecordDataset(BaseDataset):
missings = missings[missings >= 1] missings = missings[missings >= 1]
if len(missings) == 0: if len(missings) == 0:
return return
# Make sure they are all efficiency columns # Make sure they are all efficiency columns
if any(~missings.index.str.contains("energy_eff")): if any(~missings.index.str.contains("energy_eff")):
@ -1033,13 +1065,11 @@ class RecordDataset(BaseDataset):
column_index = self.df[m].isna() column_index = self.df[m].isna()
self.df.loc[column_index, m] = "NO_RATING" self.df.loc[column_index, m] = "NO_RATING"
def _null_validation(self, information: str): def _null_validation(self, information: str):
print(f"Null validation after {information}") print(f"Null validation after {information}")
if pd.isnull(self.df).sum().sum(): if pd.isnull(self.df).sum().sum():
raise ValueError(f"Null values found in dataset, after step {information}") raise ValueError(f"Null values found in dataset, after step {information}")
def _expand_description_to_features(self, cleaned_lookup: dict): def _expand_description_to_features(self, cleaned_lookup: dict):
""" """
This method will merge on the cleaned lookup table and ensure that the building fabric in the This method will merge on the cleaned lookup table and ensure that the building fabric in the
@ -1050,49 +1080,63 @@ class RecordDataset(BaseDataset):
# remove this record, as it indicates that the quality of the EPC conducted in the first instance # remove this record, as it indicates that the quality of the EPC conducted in the first instance
# is low # is low
# We also replace descriptions with their cleaned variants # We also replace descriptions with their cleaned variants
""" """
cols_to_drop = { cols_to_drop = {
"walls": [ "walls": [
# We need to cleaned descriptions for pulling out u-values # We need to cleaned descriptions for pulling out u-values
'original_description', 'thermal_transmittance_unit', "original_description",
"thermal_transmittance_unit",
# Re remove the is_assumed columns # Re remove the is_assumed columns
"is_assumed" "is_assumed",
], ],
"floor": [ "floor": [
"original_description", "clean_description", "thermal_transmittance_unit", "original_description",
"no_data", "clean_description",
"is_assumed" "thermal_transmittance_unit",
"no_data",
"is_assumed",
], ],
"roof": [ "roof": [
"original_description", "clean_description", "thermal_transmittance_unit", "original_description",
"is_assumed", "is_valid" "clean_description",
"thermal_transmittance_unit",
"is_assumed",
"is_valid",
], ],
"hotwater": [ "hotwater": [
"original_description", "clean_description", "assumed", "original_description",
"clean_description",
"assumed",
], ],
"mainheat": [ "mainheat": [
"original_description", "clean_description", "original_description",
"clean_description",
"has_assumed", "has_assumed",
], ],
"mainheatcont": [ "mainheatcont": [
"original_description", "clean_description", "original_description",
"clean_description",
], ],
"windows": [ "windows": [
"original_description", "clean_description", "original_description",
"clean_description",
# We don't need many of the glazing coverage features because we have the multi_glaze_proportion feature # We don't need many of the glazing coverage features because we have the multi_glaze_proportion feature
"has_glazing", "glazing_coverage", "no_data", "has_glazing",
"glazing_coverage",
"no_data",
], ],
"main-fuel": [ "main-fuel": [
"original_description", "clean_description", "original_description",
"clean_description",
], ],
} }
components_to_expand = cols_to_drop.keys() components_to_expand = cols_to_drop.keys()
for component in components_to_expand: for component in components_to_expand:
# TODO: change cleaned dataframe to have underscores instead of dashes # TODO: change cleaned dataframe to have underscores instead of dashes
if component == "main-fuel": if component == "main-fuel":
cleaned_key = "main-fuel" cleaned_key = "main-fuel"
left_on_key = "main_fuel" left_on_key = "main_fuel"
@ -1108,11 +1152,13 @@ class RecordDataset(BaseDataset):
cleaned_lookup_df_for_key, cleaned_lookup_df_for_key,
how="left", how="left",
left_on=left_on_key, left_on=left_on_key,
right_on="original_description" right_on="original_description",
) )
# Drop original cols and cols to drop # Drop original cols and cols to drop
expanded_df = expanded_df.drop(columns=cols_to_drop[component] + original_cols) expanded_df = expanded_df.drop(
columns=cols_to_drop[component] + original_cols
)
# Rename columns to component specific names, if they have not been dropped # Rename columns to component specific names, if they have not been dropped
expanded_df = expanded_df.rename( expanded_df = expanded_df.rename(
@ -1124,17 +1170,16 @@ class RecordDataset(BaseDataset):
} }
) )
self.df = expanded_df self.df = expanded_df
# We don't need any lighting specific cleaning, we just drop the original description as we use # We don't need any lighting specific cleaning, we just drop the original description as we use
# LOW_ENERGY_LIGHTING_STARTING, LOW_ENERGY_LIGHTING_ENDING # LOW_ENERGY_LIGHTING_STARTING, LOW_ENERGY_LIGHTING_ENDING
self.df = self.df.drop(columns=["lighting_description"]) self.df = self.df.drop(columns=["lighting_description"])
# def __add__(self, other) -> "NewDataset": # def __add__(self, other) -> "NewDataset":
# if not isinstance(other, NewDataset): # if not isinstance(other, NewDataset):
# raise TypeError("Addition can only be performed with another instance of ScoringDataset") # raise TypeError("Addition can only be performed with another instance of ScoringDataset")
# return NewDataset(self.datasets + other.datasets) # return NewDataset(self.datasets + other.datasets)
# def __radd__(self, other): # def __radd__(self, other):
# """ # """
# Required for sum() to work # Required for sum() to work
@ -1142,4 +1187,4 @@ class RecordDataset(BaseDataset):
# if isinstance(other, int): # if isinstance(other, int):
# return self # return self
# else: # else:
# return self.__add__(other) # return self.__add__(other)

19
etl/epc/Pipeline.py
View file

@ -87,9 +87,9 @@ class EPCPipeline:
run_mode="training", run_mode="training",
epc_local_file="certificates.csv", epc_local_file="certificates.csv",
epc_bucket_name="retrofit-data-dev", epc_bucket_name="retrofit-data-dev",
epc_cleaning_dataset_key="sap_change_model/cleaning_dataset_rooms.parquet", epc_cleaning_dataset_key="sap_change_model/cleaning_dataset_record.parquet",
epc_all_equal_rows_key="sap_change_model/all_equal_rows_rooms.parquet", epc_all_equal_rows_key="sap_change_model/all_equal_rows_record.parquet",
epc_compiled_dataset_key="sap_change_model/dataset_rooms.parquet", epc_compiled_dataset_key="sap_change_model/dataset_record.parquet",
): ):
""" """
:param directories: List of directories to process :param directories: List of directories to process
@ -127,7 +127,6 @@ class EPCPipeline:
self.run_record_dataset_pipeline() self.run_record_dataset_pipeline()
else: else:
raise ValueError("Run mode defined needs to be in 'training' or 'newdata'") raise ValueError("Run mode defined needs to be in 'training' or 'newdata'")
def run_record_dataset_pipeline(self): def run_record_dataset_pipeline(self):
""" """
@ -150,9 +149,17 @@ class EPCPipeline:
) )
# TODO: integrate with EPCRecord # TODO: integrate with EPCRecord
record_dataset = constituency_data[['uprn'] + VARIABLE_DATA_FEATURES + MANDATORY_FIXED_FEATURES + LATEST_FIELD] record_dataset = constituency_data[
["uprn"]
+ [RDSAP_RESPONSE]
+ VARIABLE_DATA_FEATURES
+ MANDATORY_FIXED_FEATURES
+ LATEST_FIELD
].rename(columns={RDSAP_RESPONSE: "sap"})
constituency_dataset = RecordDataset(datasets=record_dataset, cleaned_lookup=clean_lookup) constituency_dataset = RecordDataset(
datasets=record_dataset, cleaned_lookup=clean_lookup
)
self.compiled_dataset = pd.concat( self.compiled_dataset = pd.concat(
[self.compiled_dataset, constituency_dataset.df] [self.compiled_dataset, constituency_dataset.df]

3
etl/epc/property_change_app.py
View file

@ -12,10 +12,11 @@ def main():
""" """
directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()] directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()]
# directories = directories[0:3] # directories = directories[202:203]
epc_pipeline = EPCPipeline( epc_pipeline = EPCPipeline(
directories=directories, directories=directories,
run_mode="record",
epc_data_processor=EPCDataProcessor(run_mode="training"), epc_data_processor=EPCDataProcessor(run_mode="training"),
) )