junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added try except mechanism

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-02-27 15:15:05 +00:00
parent 730ad0fd71
commit d573c4d8a0
2 changed files with 38 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
etl/eligibility/ha_15_32/ha_analysis_batch_3.py
View file

@ -1089,6 +1089,9 @@ def get_epc_data(
outputs = {}
for ha_name, data_assets in loader.data.items():
if ha_name == "HA39":
continue
if not pull_data:
# Then we retrieve the data from S3
processed_ha_results = read_pickle_from_s3(
@ -1114,6 +1117,7 @@ def get_epc_data(
results = []
scoring_data = []
nodata = []
failed_model_rows = []
for index, property_meta in tqdm(asset_list.iterrows(), total=len(asset_list)):
if property_meta["matching_postcode"] is None:
@ -1225,19 +1229,24 @@ def get_epc_data(
if eligibility.eco4_warmfront["eligible"]:
if eligibility.epc["uprn"] == "":
eligibility.epc["uprn"] = int(property_meta["asset_list_row_id"].split(ha_name)[1])
scoring_dictionary = prepare_model_data_row(
property_id=property_meta["asset_list_row_id"],
modelling_epc=eligibility.epc,
cleaned=cleaned,
cleaning_data=cleaning_data,
created_at=created_at,
old_data=older_epcs,
full_sap_epc=full_sap_epc,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
scoring_data.extend(scoring_dictionary)
try:
scoring_dictionary = prepare_model_data_row(
property_id=property_meta["asset_list_row_id"],
modelling_epc=eligibility.epc,
cleaned=cleaned,
cleaning_data=cleaning_data,
created_at=created_at,
old_data=older_epcs,
full_sap_epc=full_sap_epc,
photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds
)
scoring_data.extend(scoring_dictionary)
except Exception as e:
# If we fail, we just keep a record of it
failed_model_rows.append(
property_meta["asset_list_row_id"]
)
results.append(
{

32
etl/epc/Record.py
View file

@ -725,26 +725,26 @@ class EPCRecord:
if self.prepared_epc["construction-age-band"] in DATA_ANOMALY_MATCHES:
if self.old_data:
# Take the most recent
max_datetime = max(
[
old_record["lodgement-datetime"]
for old_record in self.old_data
if old_record["construction-age-band"]
not in DATA_ANOMALY_MATCHES
]
)
most_recent = [
old_record
old_age_bands = [
old_record["lodgement-datetime"]
for old_record in self.old_data
if old_record["lodgement-datetime"] == max_datetime
if old_record["construction-age-band"] not in DATA_ANOMALY_MATCHES
]
self.prepared_epc["construction-age-band"] = (
EPCDataProcessor.clean_construction_age_band(
most_recent[0]["construction-age-band"]
if old_age_bands:
max_datetime = max(old_age_bands)
most_recent = [
old_record
for old_record in self.old_data
if old_record["lodgement-datetime"] == max_datetime
]
self.prepared_epc["construction-age-band"] = (
EPCDataProcessor.clean_construction_age_band(
most_recent[0]["construction-age-band"]
)
)
)
self.construction_age_band = self.prepared_epc["construction-age-band"]
self.age_band = england_wales_age_band_lookup.get(self.construction_age_band)