diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 07d54629..acaf4160 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -134,241 +134,250 @@ async def trigger_plan(body: PlanTriggerRequest): Session = sessionmaker(bind=db_engine) session = Session() - logger.info("Getting the inputs") - # Read in the trigger file from s3 - bucket_name = get_settings().PLAN_TRIGGER_BUCKET - epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) + try: - plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path) + logger.info("Getting the inputs") + # Read in the trigger file from s3 + bucket_name = get_settings().PLAN_TRIGGER_BUCKET + epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) - input_properties = [] - for config in plan_input: - # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly - # TODO: implment validation + plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path) - # Create a record in db - property_id, is_new = create_property( - session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] - ) + input_properties = [] + for config in plan_input: + # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly + # TODO: implment validation - # if a new record was not created, we don't produduce recommendations - if not is_new: - continue - - # TODO: Need to add heat demand target - create_property_targets( - session, - property_id=property_id, - portfolio_id=body.portfolio_id, - epc_target=body.goal_value, - heat_demand_target=None - ) - - input_properties.append( - Property( - postcode=config['postcode'], - address1=config['address'], - epc_client=epc_client, - id=property_id - ) - ) - - if not input_properties: - return Response(status_code=204) - - logger.info("Getting EPC data") - for p in input_properties: - p.search_address_epc() - p.set_year_built() - - logger.info("Getting coordinates") - # This is placeholder, until the full dataset is loaded into the database - for p in input_properties: - coordinate_data = [x for x in open_uprn_data if x['UPRN'] == int(p.data['uprn'])][0] - p.set_coordinates(coordinate_data) - - logger.info("Check if property is in conservation area") - for p in input_properties: - in_conservation_area = [x for x in in_conservation_area_data if x['uprn'] == int(p.data['uprn'])][0].get( - "is_in_conservation_area" - ) - p.set_is_in_conservation_area(in_conservation_area) - - # The materials data could be cached or local so we don't need to make - # consistent requrests to the backend for - # the same data - # TODO: It might not be the best choice to store the materials data in a database table since thi - # table probably won't be very large and won't be updated that often. It might be better to - # store this data in s3 load it into memory when the app starts up. We will test this - - materials = get_materials(session) - materials_by_type = filter_materials(materials) - - logger.info("Getting components and properties recommendations") - - # TODO: Move this to a class. We probably was a Recommender class which takes the injects the optimisers - # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - recommendations = {} - for p in input_properties: - property_recommendations = [] - - # For each property, classiy floor area decide - total_floor_area_group_decile = classify_decile_newvalues( - decile_boundaries=floors_decile_data["decile_boundaries"], - decile_labels=floors_decile_data["decile_labels"], - new_values=[float(p.data["total-floor-area"])], - )[0] - - # Property recommendations - p.get_components(cleaned) - - # This is placeholder, until the full dataset is loaded into the database and we just make a read to the - # database - floors_u_value_estimate = [ - x for x in uvalue_estimates_floors - if (x['local-authority'] == p.data["local-authority"]) & - (x['property-type'] == p.data["property-type"]) & - (x['built-form'] == p.data["built-form"]) & - (x['floor-energy-eff'] == p.data["floor-energy-eff"] if p.data["floor-energy-eff"] != 'N/A' else True) & - (x['floor-env-eff'] == p.data["floor-env-eff"] if p.data["floor-env-eff"] != 'N/A' else True) - ] - - # Floor recommendations - floor_recommender = FloorRecommendations( - property_instance=p, - uvalue_estimates=floors_u_value_estimate, - total_floor_area_group_decile=total_floor_area_group_decile, - materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"], - ) - floor_recommender.recommend() - - if floor_recommender.recommendations: - property_recommendations.append(floor_recommender.recommendations) - - # Wall recommendations - # We would make this u-value query directly to the database - total_floor_area_group_decile = classify_decile_newvalues( - decile_boundaries=walls_decile_data["decile_boundaries"], - decile_labels=walls_decile_data["decile_labels"], - new_values=[float(p.data["total-floor-area"])], - )[0] - - # This is placeholder, until the full dataset is loaded into the database and we just make a read to the - # database - walls_u_value_estimate = [ - x for x in uvalue_estimates_walls - if (x['local-authority'] == p.data["local-authority"]) & - (x['property-type'] == p.data["property-type"]) & - (x['built-form'] == p.data["built-form"]) & - (x['walls-energy-eff'] == p.data["walls-energy-eff"] if p.data["walls-energy-eff"] != 'N/A' else True) & - (x['walls-env-eff'] == p.data["walls-env-eff"] if p.data["walls-env-eff"] != 'N/A' else True) - ] - - wall_recomender = WallRecommendations( - property_instance=p, - uvalue_estimates=walls_u_value_estimate, - total_floor_area_group_decile=total_floor_area_group_decile, - materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"] - ) - wall_recomender.recommend() - - if wall_recomender.recommendations: - property_recommendations.append(wall_recomender.recommendations) - - # Use the optimiser to pick the default recommendations and decide if we need certain - # recommendations to get to the goal - property_recommendations = insert_temp_recommendation_id(property_recommendations) - - if not property_recommendations: - continue - - input_measures = prepare_input_measures(property_recommendations, body.goal) - - if body.budget: - optimiser = GainOptimiser(input_measures, max_cost=body.budget) - else: - # The minimum gain is the minimum number of SAP points required to get to the target SAP band - current_sap_points = int(p.data["current-energy-efficiency"]) - target_sap_points = epc_to_sap_lower_bound(body.goal_value) - - # If the gain is negative, the optimiser will return an empty solution - optimiser = CostOptimiser( - input_measures, min_gain=target_sap_points - current_sap_points + # Create a record in db + property_id, is_new = create_property( + session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] ) - optimiser.setup() - optimiser.solve() - solution = optimiser.solution + # if a new record was not created, we don't produduce recommendations + if not is_new: + continue - selected_recommendations = {r["id"] for r in solution} - # We'll use the set of selected recommendations to filter the recommendations to upload + # TODO: Need to add heat demand target + create_property_targets( + session, + property_id=property_id, + portfolio_id=body.portfolio_id, + epc_target=body.goal_value, + heat_demand_target=None + ) - property_recommendations = [ - [ - {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False} - for rec in recommendations_by_type + input_properties.append( + Property( + postcode=config['postcode'], + address1=config['address'], + epc_client=epc_client, + id=property_id + ) + ) + + if not input_properties: + return Response(status_code=204) + + logger.info("Getting EPC data") + for p in input_properties: + p.search_address_epc() + p.set_year_built() + + logger.info("Getting coordinates") + # This is placeholder, until the full dataset is loaded into the database + for p in input_properties: + coordinate_data = [x for x in open_uprn_data if x['UPRN'] == int(p.data['uprn'])][0] + p.set_coordinates(coordinate_data) + + logger.info("Check if property is in conservation area") + for p in input_properties: + in_conservation_area = [x for x in in_conservation_area_data if x['uprn'] == int(p.data['uprn'])][0].get( + "is_in_conservation_area" + ) + p.set_is_in_conservation_area(in_conservation_area) + + # The materials data could be cached or local so we don't need to make + # consistent requrests to the backend for + # the same data + # TODO: It might not be the best choice to store the materials data in a database table since thi + # table probably won't be very large and won't be updated that often. It might be better to + # store this data in s3 load it into memory when the app starts up. We will test this + + materials = get_materials(session) + materials_by_type = filter_materials(materials) + + logger.info("Getting components and properties recommendations") + + # TODO: Move this to a class. We probably was a Recommender class which takes the injects the optimisers + # in as a dependency and then the optimisers can take the input measures in as part of the setup() method + recommendations = {} + for p in input_properties: + property_recommendations = [] + + # For each property, classiy floor area decide + total_floor_area_group_decile = classify_decile_newvalues( + decile_boundaries=floors_decile_data["decile_boundaries"], + decile_labels=floors_decile_data["decile_labels"], + new_values=[float(p.data["total-floor-area"])], + )[0] + + # Property recommendations + p.get_components(cleaned) + + # This is placeholder, until the full dataset is loaded into the database and we just make a read to the + # database + floors_u_value_estimate = [ + x for x in uvalue_estimates_floors + if (x['local-authority'] == p.data["local-authority"]) & + (x['property-type'] == p.data["property-type"]) & + (x['built-form'] == p.data["built-form"]) & + (x['floor-energy-eff'] == p.data["floor-energy-eff"] if p.data[ + "floor-energy-eff"] != 'N/A' else True) & + (x['floor-env-eff'] == p.data["floor-env-eff"] if p.data["floor-env-eff"] != 'N/A' else True) ] - for recommendations_by_type in property_recommendations - ] - # We'll also unlist the recommendations so they're a bit easier to handle from here onwards - property_recommendations = [ - rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type - ] + # Floor recommendations + floor_recommender = FloorRecommendations( + property_instance=p, + uvalue_estimates=floors_u_value_estimate, + total_floor_area_group_decile=total_floor_area_group_decile, + materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"], + ) + floor_recommender.recommend() - recommendations[p.id] = property_recommendations + if floor_recommender.recommendations: + property_recommendations.append(floor_recommender.recommendations) - # Once we're done, we'll store: - # 1) the property data - # 2) the property details (epc) - # 3) the recommendations + # Wall recommendations + # We would make this u-value query directly to the database + total_floor_area_group_decile = classify_decile_newvalues( + decile_boundaries=walls_decile_data["decile_boundaries"], + decile_labels=walls_decile_data["decile_labels"], + new_values=[float(p.data["total-floor-area"])], + )[0] - logger.info("Uploading recommendations to the database") - # Upload property data - for p in input_properties: - property_details_epc = p.get_property_details_epc(portfolio_id=body.portfolio_id, rating_lookup=rating_lookup) - create_property_details_epc(session, property_details_epc) + # This is placeholder, until the full dataset is loaded into the database and we just make a read to the + # database + walls_u_value_estimate = [ + x for x in uvalue_estimates_walls + if (x['local-authority'] == p.data["local-authority"]) & + (x['property-type'] == p.data["property-type"]) & + (x['built-form'] == p.data["built-form"]) & + (x['walls-energy-eff'] == p.data["walls-energy-eff"] if p.data[ + "walls-energy-eff"] != 'N/A' else True) & + (x['walls-env-eff'] == p.data["walls-env-eff"] if p.data["walls-env-eff"] != 'N/A' else True) + ] - property_data = p.get_full_property_data() - update_property_data(session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data) + wall_recomender = WallRecommendations( + property_instance=p, + uvalue_estimates=walls_u_value_estimate, + total_floor_area_group_decile=total_floor_area_group_decile, + materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"] + ) + wall_recomender.recommend() - # Upload recommendations - recommendations_to_upload = recommendations.get(p.id, []) + if wall_recomender.recommendations: + property_recommendations.append(wall_recomender.recommendations) - if not recommendations_to_upload: - continue + # Use the optimiser to pick the default recommendations and decide if we need certain + # recommendations to get to the goal + property_recommendations = insert_temp_recommendation_id(property_recommendations) - # Create a plan - new_plan_id = create_plan( - session, - { - "portfolio_id": body.portfolio_id, - "property_id": p.id, - "is_default": True - } - ) + if not property_recommendations: + continue - # Upload recommendations - uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) + input_measures = prepare_input_measures(property_recommendations, body.goal) - # Finally, match the recommendation to the plan - create_plan_recommendations( - session, - plan_id=new_plan_id, - recommendation_ids=uploaded_recommendation_ids - ) + if body.budget: + optimiser = GainOptimiser(input_measures, max_cost=body.budget) + else: + # The minimum gain is the minimum number of SAP points required to get to the target SAP band + current_sap_points = int(p.data["current-energy-efficiency"]) + target_sap_points = epc_to_sap_lower_bound(body.goal_value) - logger.info("Creating portfolio aggregations") - # We implement this in the simplest way possible which will be just to query the database for all - # recommendations associated to the portfolio and then aggregate them. This is not the most efficient - # way to do this, but it's the simplest and will be a process that we can re-use since when we change a - # recommendation from being default to not default, we'll need to re-run this process to re-calculate the - # the portfolion level impact - aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id) + # If the gain is negative, the optimiser will return an empty solution + optimiser = CostOptimiser( + input_measures, min_gain=target_sap_points - current_sap_points + ) - # Commit all changes at once - session.commit() + optimiser.setup() + optimiser.solve() + solution = optimiser.solution - session.close() + selected_recommendations = {r["id"] for r in solution} + # We'll use the set of selected recommendations to filter the recommendations to upload + + property_recommendations = [ + [ + {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False} + for rec in recommendations_by_type + ] + for recommendations_by_type in property_recommendations + ] + + # We'll also unlist the recommendations so they're a bit easier to handle from here onwards + property_recommendations = [ + rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type + ] + + recommendations[p.id] = property_recommendations + + # Once we're done, we'll store: + # 1) the property data + # 2) the property details (epc) + # 3) the recommendations + + logger.info("Uploading recommendations to the database") + # Upload property data + for p in input_properties: + property_details_epc = p.get_property_details_epc(portfolio_id=body.portfolio_id, + rating_lookup=rating_lookup) + create_property_details_epc(session, property_details_epc) + + property_data = p.get_full_property_data() + update_property_data(session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data) + + # Upload recommendations + recommendations_to_upload = recommendations.get(p.id, []) + + if not recommendations_to_upload: + continue + + # Create a plan + new_plan_id = create_plan( + session, + { + "portfolio_id": body.portfolio_id, + "property_id": p.id, + "is_default": True + } + ) + + # Upload recommendations + uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) + + # Finally, match the recommendation to the plan + create_plan_recommendations( + session, + plan_id=new_plan_id, + recommendation_ids=uploaded_recommendation_ids + ) + + logger.info("Creating portfolio aggregations") + # We implement this in the simplest way possible which will be just to query the database for all + # recommendations associated to the portfolio and then aggregate them. This is not the most efficient + # way to do this, but it's the simplest and will be a process that we can re-use since when we change a + # recommendation from being default to not default, we'll need to re-run this process to re-calculate the + # the portfolion level impact + aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id) + + # Commit all changes at once + session.commit() + except Exception as e: # General exception handling + logger.error(f"An error occurred: {e}") + session.rollback() + return Response(status_code=500, content="An unexpected error occurred.") + finally: + session.close() return Response(status_code=200)