junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added try except mechanism

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-02-27 15:15:05 +00:00
parent 730ad0fd71
commit d573c4d8a0
2 changed files with 38 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
etl/eligibility/ha_15_32/ha_analysis_batch_3.py
View file

@ -1089,6 +1089,9 @@ def get_epc_data(
outputs = {} outputs = {}
for ha_name, data_assets in loader.data.items(): for ha_name, data_assets in loader.data.items():
if ha_name == "HA39":
continue
if not pull_data: if not pull_data:
# Then we retrieve the data from S3 # Then we retrieve the data from S3
processed_ha_results = read_pickle_from_s3( processed_ha_results = read_pickle_from_s3(
@ -1114,6 +1117,7 @@ def get_epc_data(
results = [] results = []
scoring_data = [] scoring_data = []
nodata = [] nodata = []
failed_model_rows = []
for index, property_meta in tqdm(asset_list.iterrows(), total=len(asset_list)): for index, property_meta in tqdm(asset_list.iterrows(), total=len(asset_list)):
if property_meta["matching_postcode"] is None: if property_meta["matching_postcode"] is None:
@ -1225,19 +1229,24 @@ def get_epc_data(
if eligibility.eco4_warmfront["eligible"]: if eligibility.eco4_warmfront["eligible"]:
if eligibility.epc["uprn"] == "": if eligibility.epc["uprn"] == "":
eligibility.epc["uprn"] = int(property_meta["asset_list_row_id"].split(ha_name)[1]) eligibility.epc["uprn"] = int(property_meta["asset_list_row_id"].split(ha_name)[1])
try:
scoring_dictionary = prepare_model_data_row( scoring_dictionary = prepare_model_data_row(
property_id=property_meta["asset_list_row_id"], property_id=property_meta["asset_list_row_id"],
modelling_epc=eligibility.epc, modelling_epc=eligibility.epc,
cleaned=cleaned, cleaned=cleaned,
cleaning_data=cleaning_data, cleaning_data=cleaning_data,
created_at=created_at, created_at=created_at,
old_data=older_epcs, old_data=older_epcs,
full_sap_epc=full_sap_epc, full_sap_epc=full_sap_epc,
photo_supply_lookup=photo_supply_lookup, photo_supply_lookup=photo_supply_lookup,
floor_area_decile_thresholds=floor_area_decile_thresholds floor_area_decile_thresholds=floor_area_decile_thresholds
) )
scoring_data.extend(scoring_dictionary) scoring_data.extend(scoring_dictionary)
except Exception as e:
# If we fail, we just keep a record of it
failed_model_rows.append(
property_meta["asset_list_row_id"]
)
results.append( results.append(
{ {

32
etl/epc/Record.py
View file

@ -725,26 +725,26 @@ class EPCRecord:
if self.prepared_epc["construction-age-band"] in DATA_ANOMALY_MATCHES: if self.prepared_epc["construction-age-band"] in DATA_ANOMALY_MATCHES:
if self.old_data: if self.old_data:
# Take the most recent # Take the most recent
max_datetime = max( old_age_bands = [
[ old_record["lodgement-datetime"]
old_record["lodgement-datetime"]
for old_record in self.old_data
if old_record["construction-age-band"]
not in DATA_ANOMALY_MATCHES
]
)
most_recent = [
old_record
for old_record in self.old_data for old_record in self.old_data
if old_record["lodgement-datetime"] == max_datetime if old_record["construction-age-band"] not in DATA_ANOMALY_MATCHES
] ]
self.prepared_epc["construction-age-band"] = ( if old_age_bands:
EPCDataProcessor.clean_construction_age_band( max_datetime = max(old_age_bands)
most_recent[0]["construction-age-band"]
most_recent = [
old_record
for old_record in self.old_data
if old_record["lodgement-datetime"] == max_datetime
]
self.prepared_epc["construction-age-band"] = (
EPCDataProcessor.clean_construction_age_band(
most_recent[0]["construction-age-band"]
)
) )
)
self.construction_age_band = self.prepared_epc["construction-age-band"] self.construction_age_band = self.prepared_epc["construction-age-band"]
self.age_band = england_wales_age_band_lookup.get(self.construction_age_band) self.age_band = england_wales_age_band_lookup.get(self.construction_age_band)