From 1102d5383ef8be854ea1f578aa93de13caf53c2e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 21 May 2024 10:16:01 +0100 Subject: [PATCH] assembling input construction --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/Property.py | 11 +++- backend/app/plan/router.py | 104 +++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 3 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/Property.py b/backend/Property.py index 2e6cbbb6..3cb8969a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -61,7 +61,14 @@ class Property: n_bedrooms = None def __init__( - self, id, postcode, address, epc_record, already_installed=None, non_invasive_recommendations=None, + self, + id, + postcode, + address, + epc_record, + already_installed=None, + non_invasive_recommendations=None, + measures=None, **kwargs ): @@ -85,6 +92,8 @@ class Property: ast.literal_eval(non_invasive_recommendations['recommendations']) if non_invasive_recommendations else [] ) + # This is a list of measures that have been recommended for the property + self.measures = ast.literal_eval(measures) if measures else None self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index ce5577bb..70827de2 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -627,3 +627,107 @@ async def build_mds(body: PlanTriggerRequest): logger.info("Connecting to db") session = sessionmaker(bind=db_engine)() created_at = datetime.now().isoformat() + + try: + session.begin() + logger.info("Getting the inputs") + plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path) + + cleaning_data = read_dataframe_from_s3_parquet( + bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet", + ) + + input_properties = [] + for property_id, config in tqdm(enumerate(plan_input)): + # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly + uprn = config.get("uprn", None) + if uprn: + uprn = int(float(uprn)) + + epc_searcher = SearchEpc( + address1=config["address"], + postcode=config["postcode"], + uprn=uprn, + auth_token=get_settings().EPC_AUTH_TOKEN, + os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY, + ) + epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None) + epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None) + # For the moment, our OS API access is unavailable, so we skip and interpolate + epc_searcher.find_property(skip_os=True) + # Create a record in db + # TODO: If we productionise the creation of this mds report, we will need to store this in the db + # property_id, is_new = create_property( + # session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn + # ) + # if not is_new: + # continue + # + # create_property_targets( + # session, + # property_id=property_id, + # portfolio_id=body.portfolio_id, + # epc_target=body.goal_value, + # heat_demand_target=None + # ) + + epc_records = { + 'original_epc': epc_searcher.newest_epc.copy(), + 'full_sap_epc': epc_searcher.full_sap_epc.copy(), + 'old_data': epc_searcher.older_epcs.copy(), + } + + # patch = next(( + # x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + # ), {}) + # epc_records = patch_epc(patch, epc_records) + + prepared_epc = EPCRecord( + epc_records=epc_records, + run_mode="newdata", + cleaning_data=cleaning_data + ) + + # property_already_installed = next(( + # x for x in already_installed if + # (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + # ), {}) + # + # property_non_invasive_recommendations = next(( + # x for x in non_invasive_recommendations if + # (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + # ), {}) + + measures = config["measures"] if "measures" in config else None + + input_properties.append( + Property( + id=property_id, + address=epc_searcher.address_clean, + postcode=epc_searcher.postcode_clean, + epc_record=prepared_epc, + # already_installed=property_already_installed, + # non_invasive_recommendations=property_non_invasive_recommendations, + measures=measures, + **Property.extract_kwargs(config) + ) + ) + + except IntegrityError: + logger.error("Database integrity error occurred", exc_info=True) + session.rollback() + return Response(status_code=500, content="Database integrity error.") + except OperationalError: + logger.error("Database operational error occurred", exc_info=True) + session.rollback() + return Response(status_code=500, content="Database operational error.") + except ValueError: + logger.error("Value error - possibly due to malformed data", exc_info=True) + session.rollback() + return Response(status_code=400, content="Bad request: malformed data.") + except Exception as e: # General exception handling + logger.error(f"An error occurred: {e}") + session.rollback() + return Response(status_code=500, content="An unexpected error occurred.") + finally: + session.close()