completed integration of new sap model process - still needs tests

backend/app/plan/router.py

@@ -207,17 +207,17 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         # Retrieve the predictions
         predictions = pd.DataFrame(
-            read_csv_from_s3(
+            read_parquet_from_s3(
                 bucket_name=get_settings().PREDICTIONS_BUCKET,
-                filepath=response["storage_filepath"].split(get_settings().PREDICTIONS_BUCKET + "/")[1]
+                file_key=response["storage_filepath"].split(get_settings().PREDICTIONS_BUCKET + "/")[1]
             )
         )
 
-        predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(1)
+        predictions["predictions"] = predictions["predictions"].astype(float).round(1)
         predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True)
 
         # Insert the predictions into the recommendations and run the optimiser
-        logger.info("Storing recommendations")
+        logger.info("Optimising recommendations")
         for property_id in recommendations.keys():
 
             property = [p for p in input_properties if p.id == property_id][0]
@@ -225,9 +225,11 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             for recommendations_by_type in recommendations[property_id]:
                 for rec in recommendations_by_type:
-                    rec["sap_points"] = property_predictions[property_predictions["recommendation_id"] == str(
+                    new_sap = property_predictions[property_predictions["recommendation_id"] == str(
                         rec["recommendation_id"]
-                    )]["RDSAP_CHANGE"].values[0]
+                    )]["predictions"].values[0]
+
+                    rec["sap_points"] = new_sap - float(property.data["current-energy-efficiency"])
 
                     if rec["sap_points"] is None:
                         raise ValueError("Sap points missing")
@@ -265,8 +267,6 @@ async def trigger_plan(body: PlanTriggerRequest):
             final_recommendations = [
                 rec for recommendations_by_type in final_recommendations for rec in recommendations_by_type
             ]
-            # We update recommendations[property_id]
-
             recommendations[property_id] = final_recommendations
 
         # 1) the property data

backend/app/plan/utils.py

@@ -118,27 +118,17 @@ def create_recommendation_scoring_data(
             is_at_rafters=property.roof["is_at_rafters"],
         )
 
-    # Tidy up insulation thicknesses, making sure it isn't None
-    if scoring_dict["walls_insulation_thickness"] is None:
-        scoring_dict["walls_insulation_thickness"] = "none"
-
-    if scoring_dict["floor_insulation_thickness"] is None:
-        scoring_dict["floor_insulation_thickness"] = "none"
-
-    if scoring_dict["roof_insulation_thickness"] is None:
-        scoring_dict["roof_insulation_thickness"] = "none"
+    for col in [
+        "walls_insulation_thickness", "floor_insulation_thickness", "roof_insulation_thickness"
+    ]:
+        if scoring_dict[col] is None:
+            scoring_dict[col] = "none"
 
     # We update the description to indicate it's insulated
     if recommendation["type"] == "wall_insulation":
         # The upgrade made here is to the u-value of the walls and the description of the
         # insulation thickness
-        # We may not have the u-value initially, so we calculate it
-        scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value(
-            clean_description=UPGRADES_MAP[property.walls["clean_description"]],
-            age_band=property.age_band,
-            is_granite_or_whinstone=property.walls["is_granite_or_whinstone"],
-            is_sandstone_or_limestone=property.walls["is_sandstone_or_limestone"]
-        )
+        scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
         scoring_dict["walls_insulation_thickness_ENDING"] = "above average"
     else:
         if not scoring_dict["walls_thermal_transmittance_ENDING"]:
@@ -158,14 +148,7 @@ def create_recommendation_scoring_data(
         if len(recommendation["parts"]) > 1:
             raise NotImplementedError("Have more than 1 floor insulation part - handle this case")
 
-        scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value(
-            floor_type=property.floor_type,
-            area=property.floor_area,
-            perimeter=property.perimeter,
-            wall_type=property.wall_type,
-            insulation_thickness=recommendation["parts"][0]["depths"][0],
-            age_band=property.age_band,
-        )
+        scoring_dict["floor_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
         # We don't really see above average for this in the training data
         scoring_dict["floor_insulation_thickness_ENDING"] = "average"
     else:

backend/ml_models/sap_change_model/api.py

@@ -69,7 +69,7 @@ class SAPChangeModelAPI:
         }
 
         try:
-            response = requests.post(url, json=payload, headers={"Content-Type": "application/json"})
+            response = requests.post(url, json=payload, headers={"Content-Type": "application/json"}, timeout=120)
 
             # Check if the response status code is 2xx (success)
             response.raise_for_status()