mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
Added try catch block
This commit is contained in:
Khalim Conn-Kowlessar
parent
eef6fd27dd
commit
c413482bf4
1 changed files with 223 additions and 214 deletions
|
|
@ -134,241 +134,250 @@ async def trigger_plan(body: PlanTriggerRequest):
|
||||||
Session = sessionmaker(bind=db_engine)
|
Session = sessionmaker(bind=db_engine)
|
||||||
session = Session()
|
session = Session()
|
||||||
|
|
||||||
logger.info("Getting the inputs")
|
try:
|
||||||
# Read in the trigger file from s3
|
|
||||||
bucket_name = get_settings().PLAN_TRIGGER_BUCKET
|
|
||||||
epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN)
|
|
||||||
|
|
||||||
plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path)
|
logger.info("Getting the inputs")
|
||||||
|
# Read in the trigger file from s3
|
||||||
|
bucket_name = get_settings().PLAN_TRIGGER_BUCKET
|
||||||
|
epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN)
|
||||||
|
|
||||||
input_properties = []
|
plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path)
|
||||||
for config in plan_input:
|
|
||||||
# We validate each record in the file. If the record is NOT valid, we need to handle this accordingly
|
|
||||||
# TODO: implment validation
|
|
||||||
|
|
||||||
# Create a record in db
|
input_properties = []
|
||||||
property_id, is_new = create_property(
|
for config in plan_input:
|
||||||
session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode']
|
# We validate each record in the file. If the record is NOT valid, we need to handle this accordingly
|
||||||
)
|
# TODO: implment validation
|
||||||
|
|
||||||
# if a new record was not created, we don't produduce recommendations
|
# Create a record in db
|
||||||
if not is_new:
|
property_id, is_new = create_property(
|
||||||
continue
|
session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode']
|
||||||
|
|
||||||
# TODO: Need to add heat demand target
|
|
||||||
create_property_targets(
|
|
||||||
session,
|
|
||||||
property_id=property_id,
|
|
||||||
portfolio_id=body.portfolio_id,
|
|
||||||
epc_target=body.goal_value,
|
|
||||||
heat_demand_target=None
|
|
||||||
)
|
|
||||||
|
|
||||||
input_properties.append(
|
|
||||||
Property(
|
|
||||||
postcode=config['postcode'],
|
|
||||||
address1=config['address'],
|
|
||||||
epc_client=epc_client,
|
|
||||||
id=property_id
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if not input_properties:
|
|
||||||
return Response(status_code=204)
|
|
||||||
|
|
||||||
logger.info("Getting EPC data")
|
|
||||||
for p in input_properties:
|
|
||||||
p.search_address_epc()
|
|
||||||
p.set_year_built()
|
|
||||||
|
|
||||||
logger.info("Getting coordinates")
|
|
||||||
# This is placeholder, until the full dataset is loaded into the database
|
|
||||||
for p in input_properties:
|
|
||||||
coordinate_data = [x for x in open_uprn_data if x['UPRN'] == int(p.data['uprn'])][0]
|
|
||||||
p.set_coordinates(coordinate_data)
|
|
||||||
|
|
||||||
logger.info("Check if property is in conservation area")
|
|
||||||
for p in input_properties:
|
|
||||||
in_conservation_area = [x for x in in_conservation_area_data if x['uprn'] == int(p.data['uprn'])][0].get(
|
|
||||||
"is_in_conservation_area"
|
|
||||||
)
|
|
||||||
p.set_is_in_conservation_area(in_conservation_area)
|
|
||||||
|
|
||||||
# The materials data could be cached or local so we don't need to make
|
|
||||||
# consistent requrests to the backend for
|
|
||||||
# the same data
|
|
||||||
# TODO: It might not be the best choice to store the materials data in a database table since thi
|
|
||||||
# table probably won't be very large and won't be updated that often. It might be better to
|
|
||||||
# store this data in s3 load it into memory when the app starts up. We will test this
|
|
||||||
|
|
||||||
materials = get_materials(session)
|
|
||||||
materials_by_type = filter_materials(materials)
|
|
||||||
|
|
||||||
logger.info("Getting components and properties recommendations")
|
|
||||||
|
|
||||||
# TODO: Move this to a class. We probably was a Recommender class which takes the injects the optimisers
|
|
||||||
# in as a dependency and then the optimisers can take the input measures in as part of the setup() method
|
|
||||||
recommendations = {}
|
|
||||||
for p in input_properties:
|
|
||||||
property_recommendations = []
|
|
||||||
|
|
||||||
# For each property, classiy floor area decide
|
|
||||||
total_floor_area_group_decile = classify_decile_newvalues(
|
|
||||||
decile_boundaries=floors_decile_data["decile_boundaries"],
|
|
||||||
decile_labels=floors_decile_data["decile_labels"],
|
|
||||||
new_values=[float(p.data["total-floor-area"])],
|
|
||||||
)[0]
|
|
||||||
|
|
||||||
# Property recommendations
|
|
||||||
p.get_components(cleaned)
|
|
||||||
|
|
||||||
# This is placeholder, until the full dataset is loaded into the database and we just make a read to the
|
|
||||||
# database
|
|
||||||
floors_u_value_estimate = [
|
|
||||||
x for x in uvalue_estimates_floors
|
|
||||||
if (x['local-authority'] == p.data["local-authority"]) &
|
|
||||||
(x['property-type'] == p.data["property-type"]) &
|
|
||||||
(x['built-form'] == p.data["built-form"]) &
|
|
||||||
(x['floor-energy-eff'] == p.data["floor-energy-eff"] if p.data["floor-energy-eff"] != 'N/A' else True) &
|
|
||||||
(x['floor-env-eff'] == p.data["floor-env-eff"] if p.data["floor-env-eff"] != 'N/A' else True)
|
|
||||||
]
|
|
||||||
|
|
||||||
# Floor recommendations
|
|
||||||
floor_recommender = FloorRecommendations(
|
|
||||||
property_instance=p,
|
|
||||||
uvalue_estimates=floors_u_value_estimate,
|
|
||||||
total_floor_area_group_decile=total_floor_area_group_decile,
|
|
||||||
materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"],
|
|
||||||
)
|
|
||||||
floor_recommender.recommend()
|
|
||||||
|
|
||||||
if floor_recommender.recommendations:
|
|
||||||
property_recommendations.append(floor_recommender.recommendations)
|
|
||||||
|
|
||||||
# Wall recommendations
|
|
||||||
# We would make this u-value query directly to the database
|
|
||||||
total_floor_area_group_decile = classify_decile_newvalues(
|
|
||||||
decile_boundaries=walls_decile_data["decile_boundaries"],
|
|
||||||
decile_labels=walls_decile_data["decile_labels"],
|
|
||||||
new_values=[float(p.data["total-floor-area"])],
|
|
||||||
)[0]
|
|
||||||
|
|
||||||
# This is placeholder, until the full dataset is loaded into the database and we just make a read to the
|
|
||||||
# database
|
|
||||||
walls_u_value_estimate = [
|
|
||||||
x for x in uvalue_estimates_walls
|
|
||||||
if (x['local-authority'] == p.data["local-authority"]) &
|
|
||||||
(x['property-type'] == p.data["property-type"]) &
|
|
||||||
(x['built-form'] == p.data["built-form"]) &
|
|
||||||
(x['walls-energy-eff'] == p.data["walls-energy-eff"] if p.data["walls-energy-eff"] != 'N/A' else True) &
|
|
||||||
(x['walls-env-eff'] == p.data["walls-env-eff"] if p.data["walls-env-eff"] != 'N/A' else True)
|
|
||||||
]
|
|
||||||
|
|
||||||
wall_recomender = WallRecommendations(
|
|
||||||
property_instance=p,
|
|
||||||
uvalue_estimates=walls_u_value_estimate,
|
|
||||||
total_floor_area_group_decile=total_floor_area_group_decile,
|
|
||||||
materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"]
|
|
||||||
)
|
|
||||||
wall_recomender.recommend()
|
|
||||||
|
|
||||||
if wall_recomender.recommendations:
|
|
||||||
property_recommendations.append(wall_recomender.recommendations)
|
|
||||||
|
|
||||||
# Use the optimiser to pick the default recommendations and decide if we need certain
|
|
||||||
# recommendations to get to the goal
|
|
||||||
property_recommendations = insert_temp_recommendation_id(property_recommendations)
|
|
||||||
|
|
||||||
if not property_recommendations:
|
|
||||||
continue
|
|
||||||
|
|
||||||
input_measures = prepare_input_measures(property_recommendations, body.goal)
|
|
||||||
|
|
||||||
if body.budget:
|
|
||||||
optimiser = GainOptimiser(input_measures, max_cost=body.budget)
|
|
||||||
else:
|
|
||||||
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
|
|
||||||
current_sap_points = int(p.data["current-energy-efficiency"])
|
|
||||||
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
|
|
||||||
|
|
||||||
# If the gain is negative, the optimiser will return an empty solution
|
|
||||||
optimiser = CostOptimiser(
|
|
||||||
input_measures, min_gain=target_sap_points - current_sap_points
|
|
||||||
)
|
)
|
||||||
|
|
||||||
optimiser.setup()
|
# if a new record was not created, we don't produduce recommendations
|
||||||
optimiser.solve()
|
if not is_new:
|
||||||
solution = optimiser.solution
|
continue
|
||||||
|
|
||||||
selected_recommendations = {r["id"] for r in solution}
|
# TODO: Need to add heat demand target
|
||||||
# We'll use the set of selected recommendations to filter the recommendations to upload
|
create_property_targets(
|
||||||
|
session,
|
||||||
|
property_id=property_id,
|
||||||
|
portfolio_id=body.portfolio_id,
|
||||||
|
epc_target=body.goal_value,
|
||||||
|
heat_demand_target=None
|
||||||
|
)
|
||||||
|
|
||||||
property_recommendations = [
|
input_properties.append(
|
||||||
[
|
Property(
|
||||||
{**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
|
postcode=config['postcode'],
|
||||||
for rec in recommendations_by_type
|
address1=config['address'],
|
||||||
|
epc_client=epc_client,
|
||||||
|
id=property_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not input_properties:
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
logger.info("Getting EPC data")
|
||||||
|
for p in input_properties:
|
||||||
|
p.search_address_epc()
|
||||||
|
p.set_year_built()
|
||||||
|
|
||||||
|
logger.info("Getting coordinates")
|
||||||
|
# This is placeholder, until the full dataset is loaded into the database
|
||||||
|
for p in input_properties:
|
||||||
|
coordinate_data = [x for x in open_uprn_data if x['UPRN'] == int(p.data['uprn'])][0]
|
||||||
|
p.set_coordinates(coordinate_data)
|
||||||
|
|
||||||
|
logger.info("Check if property is in conservation area")
|
||||||
|
for p in input_properties:
|
||||||
|
in_conservation_area = [x for x in in_conservation_area_data if x['uprn'] == int(p.data['uprn'])][0].get(
|
||||||
|
"is_in_conservation_area"
|
||||||
|
)
|
||||||
|
p.set_is_in_conservation_area(in_conservation_area)
|
||||||
|
|
||||||
|
# The materials data could be cached or local so we don't need to make
|
||||||
|
# consistent requrests to the backend for
|
||||||
|
# the same data
|
||||||
|
# TODO: It might not be the best choice to store the materials data in a database table since thi
|
||||||
|
# table probably won't be very large and won't be updated that often. It might be better to
|
||||||
|
# store this data in s3 load it into memory when the app starts up. We will test this
|
||||||
|
|
||||||
|
materials = get_materials(session)
|
||||||
|
materials_by_type = filter_materials(materials)
|
||||||
|
|
||||||
|
logger.info("Getting components and properties recommendations")
|
||||||
|
|
||||||
|
# TODO: Move this to a class. We probably was a Recommender class which takes the injects the optimisers
|
||||||
|
# in as a dependency and then the optimisers can take the input measures in as part of the setup() method
|
||||||
|
recommendations = {}
|
||||||
|
for p in input_properties:
|
||||||
|
property_recommendations = []
|
||||||
|
|
||||||
|
# For each property, classiy floor area decide
|
||||||
|
total_floor_area_group_decile = classify_decile_newvalues(
|
||||||
|
decile_boundaries=floors_decile_data["decile_boundaries"],
|
||||||
|
decile_labels=floors_decile_data["decile_labels"],
|
||||||
|
new_values=[float(p.data["total-floor-area"])],
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
# Property recommendations
|
||||||
|
p.get_components(cleaned)
|
||||||
|
|
||||||
|
# This is placeholder, until the full dataset is loaded into the database and we just make a read to the
|
||||||
|
# database
|
||||||
|
floors_u_value_estimate = [
|
||||||
|
x for x in uvalue_estimates_floors
|
||||||
|
if (x['local-authority'] == p.data["local-authority"]) &
|
||||||
|
(x['property-type'] == p.data["property-type"]) &
|
||||||
|
(x['built-form'] == p.data["built-form"]) &
|
||||||
|
(x['floor-energy-eff'] == p.data["floor-energy-eff"] if p.data[
|
||||||
|
"floor-energy-eff"] != 'N/A' else True) &
|
||||||
|
(x['floor-env-eff'] == p.data["floor-env-eff"] if p.data["floor-env-eff"] != 'N/A' else True)
|
||||||
]
|
]
|
||||||
for recommendations_by_type in property_recommendations
|
|
||||||
]
|
|
||||||
|
|
||||||
# We'll also unlist the recommendations so they're a bit easier to handle from here onwards
|
# Floor recommendations
|
||||||
property_recommendations = [
|
floor_recommender = FloorRecommendations(
|
||||||
rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type
|
property_instance=p,
|
||||||
]
|
uvalue_estimates=floors_u_value_estimate,
|
||||||
|
total_floor_area_group_decile=total_floor_area_group_decile,
|
||||||
|
materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"],
|
||||||
|
)
|
||||||
|
floor_recommender.recommend()
|
||||||
|
|
||||||
recommendations[p.id] = property_recommendations
|
if floor_recommender.recommendations:
|
||||||
|
property_recommendations.append(floor_recommender.recommendations)
|
||||||
|
|
||||||
# Once we're done, we'll store:
|
# Wall recommendations
|
||||||
# 1) the property data
|
# We would make this u-value query directly to the database
|
||||||
# 2) the property details (epc)
|
total_floor_area_group_decile = classify_decile_newvalues(
|
||||||
# 3) the recommendations
|
decile_boundaries=walls_decile_data["decile_boundaries"],
|
||||||
|
decile_labels=walls_decile_data["decile_labels"],
|
||||||
|
new_values=[float(p.data["total-floor-area"])],
|
||||||
|
)[0]
|
||||||
|
|
||||||
logger.info("Uploading recommendations to the database")
|
# This is placeholder, until the full dataset is loaded into the database and we just make a read to the
|
||||||
# Upload property data
|
# database
|
||||||
for p in input_properties:
|
walls_u_value_estimate = [
|
||||||
property_details_epc = p.get_property_details_epc(portfolio_id=body.portfolio_id, rating_lookup=rating_lookup)
|
x for x in uvalue_estimates_walls
|
||||||
create_property_details_epc(session, property_details_epc)
|
if (x['local-authority'] == p.data["local-authority"]) &
|
||||||
|
(x['property-type'] == p.data["property-type"]) &
|
||||||
|
(x['built-form'] == p.data["built-form"]) &
|
||||||
|
(x['walls-energy-eff'] == p.data["walls-energy-eff"] if p.data[
|
||||||
|
"walls-energy-eff"] != 'N/A' else True) &
|
||||||
|
(x['walls-env-eff'] == p.data["walls-env-eff"] if p.data["walls-env-eff"] != 'N/A' else True)
|
||||||
|
]
|
||||||
|
|
||||||
property_data = p.get_full_property_data()
|
wall_recomender = WallRecommendations(
|
||||||
update_property_data(session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data)
|
property_instance=p,
|
||||||
|
uvalue_estimates=walls_u_value_estimate,
|
||||||
|
total_floor_area_group_decile=total_floor_area_group_decile,
|
||||||
|
materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"]
|
||||||
|
)
|
||||||
|
wall_recomender.recommend()
|
||||||
|
|
||||||
# Upload recommendations
|
if wall_recomender.recommendations:
|
||||||
recommendations_to_upload = recommendations.get(p.id, [])
|
property_recommendations.append(wall_recomender.recommendations)
|
||||||
|
|
||||||
if not recommendations_to_upload:
|
# Use the optimiser to pick the default recommendations and decide if we need certain
|
||||||
continue
|
# recommendations to get to the goal
|
||||||
|
property_recommendations = insert_temp_recommendation_id(property_recommendations)
|
||||||
|
|
||||||
# Create a plan
|
if not property_recommendations:
|
||||||
new_plan_id = create_plan(
|
continue
|
||||||
session,
|
|
||||||
{
|
|
||||||
"portfolio_id": body.portfolio_id,
|
|
||||||
"property_id": p.id,
|
|
||||||
"is_default": True
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Upload recommendations
|
input_measures = prepare_input_measures(property_recommendations, body.goal)
|
||||||
uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id)
|
|
||||||
|
|
||||||
# Finally, match the recommendation to the plan
|
if body.budget:
|
||||||
create_plan_recommendations(
|
optimiser = GainOptimiser(input_measures, max_cost=body.budget)
|
||||||
session,
|
else:
|
||||||
plan_id=new_plan_id,
|
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
|
||||||
recommendation_ids=uploaded_recommendation_ids
|
current_sap_points = int(p.data["current-energy-efficiency"])
|
||||||
)
|
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
|
||||||
|
|
||||||
logger.info("Creating portfolio aggregations")
|
# If the gain is negative, the optimiser will return an empty solution
|
||||||
# We implement this in the simplest way possible which will be just to query the database for all
|
optimiser = CostOptimiser(
|
||||||
# recommendations associated to the portfolio and then aggregate them. This is not the most efficient
|
input_measures, min_gain=target_sap_points - current_sap_points
|
||||||
# way to do this, but it's the simplest and will be a process that we can re-use since when we change a
|
)
|
||||||
# recommendation from being default to not default, we'll need to re-run this process to re-calculate the
|
|
||||||
# the portfolion level impact
|
|
||||||
aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id)
|
|
||||||
|
|
||||||
# Commit all changes at once
|
optimiser.setup()
|
||||||
session.commit()
|
optimiser.solve()
|
||||||
|
solution = optimiser.solution
|
||||||
|
|
||||||
session.close()
|
selected_recommendations = {r["id"] for r in solution}
|
||||||
|
# We'll use the set of selected recommendations to filter the recommendations to upload
|
||||||
|
|
||||||
|
property_recommendations = [
|
||||||
|
[
|
||||||
|
{**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
|
||||||
|
for rec in recommendations_by_type
|
||||||
|
]
|
||||||
|
for recommendations_by_type in property_recommendations
|
||||||
|
]
|
||||||
|
|
||||||
|
# We'll also unlist the recommendations so they're a bit easier to handle from here onwards
|
||||||
|
property_recommendations = [
|
||||||
|
rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type
|
||||||
|
]
|
||||||
|
|
||||||
|
recommendations[p.id] = property_recommendations
|
||||||
|
|
||||||
|
# Once we're done, we'll store:
|
||||||
|
# 1) the property data
|
||||||
|
# 2) the property details (epc)
|
||||||
|
# 3) the recommendations
|
||||||
|
|
||||||
|
logger.info("Uploading recommendations to the database")
|
||||||
|
# Upload property data
|
||||||
|
for p in input_properties:
|
||||||
|
property_details_epc = p.get_property_details_epc(portfolio_id=body.portfolio_id,
|
||||||
|
rating_lookup=rating_lookup)
|
||||||
|
create_property_details_epc(session, property_details_epc)
|
||||||
|
|
||||||
|
property_data = p.get_full_property_data()
|
||||||
|
update_property_data(session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data)
|
||||||
|
|
||||||
|
# Upload recommendations
|
||||||
|
recommendations_to_upload = recommendations.get(p.id, [])
|
||||||
|
|
||||||
|
if not recommendations_to_upload:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Create a plan
|
||||||
|
new_plan_id = create_plan(
|
||||||
|
session,
|
||||||
|
{
|
||||||
|
"portfolio_id": body.portfolio_id,
|
||||||
|
"property_id": p.id,
|
||||||
|
"is_default": True
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Upload recommendations
|
||||||
|
uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id)
|
||||||
|
|
||||||
|
# Finally, match the recommendation to the plan
|
||||||
|
create_plan_recommendations(
|
||||||
|
session,
|
||||||
|
plan_id=new_plan_id,
|
||||||
|
recommendation_ids=uploaded_recommendation_ids
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Creating portfolio aggregations")
|
||||||
|
# We implement this in the simplest way possible which will be just to query the database for all
|
||||||
|
# recommendations associated to the portfolio and then aggregate them. This is not the most efficient
|
||||||
|
# way to do this, but it's the simplest and will be a process that we can re-use since when we change a
|
||||||
|
# recommendation from being default to not default, we'll need to re-run this process to re-calculate the
|
||||||
|
# the portfolion level impact
|
||||||
|
aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id)
|
||||||
|
|
||||||
|
# Commit all changes at once
|
||||||
|
session.commit()
|
||||||
|
except Exception as e: # General exception handling
|
||||||
|
logger.error(f"An error occurred: {e}")
|
||||||
|
session.rollback()
|
||||||
|
return Response(status_code=500, content="An unexpected error occurred.")
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
return Response(status_code=200)
|
return Response(status_code=200)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue