From 62f812cd71caa5a440a44d54d8a99a348ba4a79d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 12 Sep 2023 18:48:33 +0100 Subject: [PATCH] Fixing issue with sap model parsing event body --- .gitignore | 3 +++ .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/README.md | 4 ++-- backend/app/plan/router.py | 10 ++++++++++ backend/docker/lambda.Dockerfile | 1 + backend/ml_models/sap_change_model/api.py | 2 +- .../handlers/predictions_app.py | 17 +++++++---------- 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 4e43467e..98db3e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -258,3 +258,6 @@ model_data/simulation_system/.idea/ model_data/simulation_system/data* model_data/simulation_system/model_directory/ model_data/simulation_system/predictions/ + +.idea/Model.iml +.idea/misc.iml diff --git a/.idea/Model.iml b/.idea/Model.iml index b03b31b1..05b9012b 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index ca0e1cd9..3b05c6ac 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/README.md b/backend/README.md index 439618c0..005d6fc4 100644 --- a/backend/README.md +++ b/backend/README.md @@ -111,7 +111,7 @@ and aws credentials file at ~/.aws/credentials, locally. To run this, firstly run: ```bash -chmod +x run_local_lambda.sh +chmod +x run_lambda_local.sh ``` Now you can run the script with @@ -142,7 +142,7 @@ and you need to pass a body like this: ``` -```commandline +Logs for the container can quickly be seen via Docker desktop ## Testing diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3884592a..c4d74887 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -346,6 +346,7 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations_scoring_data.append(scoring_dict) + logger.info("Preparing data for scoring in sap change api") recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) # Clean the data @@ -389,6 +390,7 @@ async def trigger_plan(body: PlanTriggerRequest): file_key=file_location ) + logger.info("Making request to sap change api") sap_change_model_api = SAPChangeModelAPI() response = sap_change_model_api.predict( file_location="s3://retrofit-data-dev/" + file_location, @@ -397,16 +399,24 @@ async def trigger_plan(body: PlanTriggerRequest): ) # Retrieve the predictions + logger.info("Retrieving predictions from s3") + test = read_csv_from_s3( + bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT), + filepath=response["storage_filepath"] + ) + predictions = pd.DataFrame(read_csv_from_s3( bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT), filepath=response["storage_filepath"] )) + # We round the predictions predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(0) # Extract property_id and recommendation_id predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True) # Insert the predictions into the recommendations and run the optimiser + logger.info("Storing recommendations") for property_id in recommendations.keys(): property = [p for p in input_properties if p.id == property_id][0] diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index d4588cab..13c30b88 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -43,6 +43,7 @@ COPY ./model_data/EpcClean.py ./model_data/EpcClean.py COPY ./model_data/utils.py ./model_data/utils.py COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/ COPY ./model_data/simulation_system/core/DataProcessor.py ./model_data/simulation_system/core/DataProcessor.py +COPY ./model_data/simulation_system/core/Settings.py ./model_data/simulation_system/core/Settings.py COPY ./datatypes/ ./datatypes/ COPY ./utils/ ./utils/ diff --git a/backend/ml_models/sap_change_model/api.py b/backend/ml_models/sap_change_model/api.py index 03c8423d..4ef4dc23 100644 --- a/backend/ml_models/sap_change_model/api.py +++ b/backend/ml_models/sap_change_model/api.py @@ -30,7 +30,7 @@ class SAPChangeModelAPI: } try: - response = requests.post(url, json=payload) + response = requests.post(url, json=payload, headers={"Content-Type": "application/json"}) # Check if the response status code is 2xx (success) response.raise_for_status() diff --git a/model_data/simulation_system/handlers/predictions_app.py b/model_data/simulation_system/handlers/predictions_app.py index 75d2aff6..4e398566 100644 --- a/model_data/simulation_system/handlers/predictions_app.py +++ b/model_data/simulation_system/handlers/predictions_app.py @@ -47,17 +47,14 @@ def handler(event, context): logger.info("received event: " + str(event)) - # Assuming a file in a bucket landing for now? - # Assuming we have a model to use - - body = json.loads(event["body"]) - - data_path = body["file_location"] - property_id = body["property_id"] - portfolio_id = body["portfolio_id"] - created_at = body["created_at"] - try: + body = json.loads(event["body"]) if not isinstance(event["body"], dict) else event["body"] + + data_path = body["file_location"] + property_id = body["property_id"] + portfolio_id = body["portfolio_id"] + created_at = body["created_at"] + # We could fix the model path but for the moment, we just take the best model path based on the registry outputs = prediction(model_path=None, data_path=data_path) # Store into s3, with key of {portfolio_id}-{property_id}