Fixing issue with sap model parsing event body

.gitignore

@@ -258,3 +258,6 @@ model_data/simulation_system/.idea/
 model_data/simulation_system/data*
 model_data/simulation_system/model_directory/
 model_data/simulation_system/predictions/
+
+.idea/Model.iml
+.idea/misc.iml

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/README.md

@@ -111,7 +111,7 @@ and aws credentials file at ~/.aws/credentials, locally.
 To run this, firstly run:
 
 ```bash
-chmod +x run_local_lambda.sh
+chmod +x run_lambda_local.sh
 ```
 
 Now you can run the script with
@@ -142,7 +142,7 @@ and you need to pass a body like this:
 
 ```
 
-```commandline
+Logs for the container can quickly be seen via Docker desktop
 
 ## Testing
 

backend/app/plan/router.py

@@ -346,6 +346,7 @@ async def trigger_plan(body: PlanTriggerRequest):
 
                     recommendations_scoring_data.append(scoring_dict)
 
+        logger.info("Preparing data for scoring in sap change api")
         recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
 
         # Clean the data
@@ -389,6 +390,7 @@ async def trigger_plan(body: PlanTriggerRequest):
             file_key=file_location
         )
 
+        logger.info("Making request to sap change api")
         sap_change_model_api = SAPChangeModelAPI()
         response = sap_change_model_api.predict(
             file_location="s3://retrofit-data-dev/" + file_location,
@@ -397,16 +399,24 @@ async def trigger_plan(body: PlanTriggerRequest):
         )
 
         # Retrieve the predictions
+        logger.info("Retrieving predictions from s3")
+        test = read_csv_from_s3(
+            bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT),
+            filepath=response["storage_filepath"]
+        )
+
         predictions = pd.DataFrame(read_csv_from_s3(
             bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT),
             filepath=response["storage_filepath"]
         ))
+        
         # We round the predictions
         predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(0)
         # Extract property_id and recommendation_id
         predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True)
 
         # Insert the predictions into the recommendations and run the optimiser
+        logger.info("Storing recommendations")
         for property_id in recommendations.keys():
 
             property = [p for p in input_properties if p.id == property_id][0]

backend/docker/lambda.Dockerfile

@@ -43,6 +43,7 @@ COPY ./model_data/EpcClean.py ./model_data/EpcClean.py
 COPY ./model_data/utils.py ./model_data/utils.py
 COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/
 COPY ./model_data/simulation_system/core/DataProcessor.py ./model_data/simulation_system/core/DataProcessor.py
+COPY ./model_data/simulation_system/core/Settings.py ./model_data/simulation_system/core/Settings.py
 COPY ./datatypes/ ./datatypes/
 COPY ./utils/ ./utils/
 

backend/ml_models/sap_change_model/api.py

@@ -30,7 +30,7 @@ class SAPChangeModelAPI:
         }
 
         try:
-            response = requests.post(url, json=payload)
+            response = requests.post(url, json=payload, headers={"Content-Type": "application/json"})
 
             # Check if the response status code is 2xx (success)
             response.raise_for_status()

model_data/simulation_system/handlers/predictions_app.py

@@ -47,17 +47,14 @@ def handler(event, context):
 
     logger.info("received event: " + str(event))
 
-    # Assuming a file in a bucket landing for now?
-    # Assuming we have a model to use
-
-    body = json.loads(event["body"])
-
-    data_path = body["file_location"]
-    property_id = body["property_id"]
-    portfolio_id = body["portfolio_id"]
-    created_at = body["created_at"]
-
     try:
+        body = json.loads(event["body"]) if not isinstance(event["body"], dict) else event["body"]
+
+        data_path = body["file_location"]
+        property_id = body["property_id"]
+        portfolio_id = body["portfolio_id"]
+        created_at = body["created_at"]
+
         # We could fix the model path but for the moment, we just take the best model path based on the registry
         outputs = prediction(model_path=None, data_path=data_path)
         # Store into s3, with key of {portfolio_id}-{property_id}