diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index e2667267..1fb79ce4 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -1,6 +1,9 @@
name: Run unit tests
-on: [ push, pull_request ]
+on:
+ push:
+ branches:
+ - main
jobs:
build:
diff --git a/.idea/Model.iml b/.idea/Model.iml
index b03b31b1..05b9012b 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
-
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
index ca0e1cd9..3b05c6ac 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -1,6 +1,6 @@
-
+
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 8f1413ee..ece9f452 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -17,7 +17,9 @@ from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError, OperationalError
from datetime import datetime
import pandas as pd
-import requests
+
+# model apis
+from backend.ml_models.sap_change_model.api import SAPChangeModelAPI
# database interaction functions
from backend.app.db.functions.property_functions import (
@@ -136,6 +138,13 @@ def insert_temp_recommendation_id(property_recommendations):
return property_recommendations
+def score_measures():
+ """
+ This wrapper function prepares data to be passed to the sap model api
+ :return:
+ """
+
+
@router.post("/trigger")
async def trigger_plan(body: PlanTriggerRequest):
logger.info("Connecting to db")
@@ -289,43 +298,13 @@ async def trigger_plan(body: PlanTriggerRequest):
if wall_recomender.recommendations:
property_recommendations.append(wall_recomender.recommendations)
- # Use the optimiser to pick the default recommendations and decide if we need certain
- # recommendations to get to the goal
+ # We insert temporary ids into the recommendations which is important for the optimiser later
property_recommendations = insert_temp_recommendation_id(property_recommendations)
if not property_recommendations:
continue
- input_measures = prepare_input_measures(property_recommendations, body.goal)
-
- if body.budget:
- optimiser = GainOptimiser(input_measures, max_cost=body.budget)
- else:
- # The minimum gain is the minimum number of SAP points required to get to the target SAP band
- current_sap_points = int(p.data["current-energy-efficiency"])
- target_sap_points = epc_to_sap_lower_bound(body.goal_value)
-
- # If the gain is negative, the optimiser will return an empty solution
- optimiser = CostOptimiser(
- input_measures, min_gain=target_sap_points - current_sap_points
- )
-
- optimiser.setup()
- optimiser.solve()
- solution = optimiser.solution
-
- selected_recommendations = {r["id"] for r in solution}
- # We'll use the set of selected recommendations to filter the recommendations to upload
-
- property_recommendations = [
- [
- {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
- for rec in recommendations_by_type
- ]
- for recommendations_by_type in property_recommendations
- ]
-
- # We'll also unlist the recommendations so they're a bit easier to handle from here onwards
+ # We'll unlist the recommendations so they're a bit easier to handle from here onwards
property_recommendations = [
rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type
]
@@ -373,7 +352,6 @@ async def trigger_plan(body: PlanTriggerRequest):
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
- # TODO: Set the TRANSACTION_TYPE
# Clean the data
cleaning_data = read_parquet_from_s3(
bucket_name="retrofit-data-dev",
@@ -411,33 +389,73 @@ async def trigger_plan(body: PlanTriggerRequest):
save_dataframe_to_s3_parquet(
df=recommendations_scoring_data,
- bucket_name="retrofit-data-dev",
+ bucket_name="retrofit-data-{environment}".format(environment=get_settings().ENVIRONMENT),
file_key=file_location
)
- # Call the sap change model
- response = requests.post(
- url="https://api.dev.hestia.homes/sapmodel/predict",
- json={
- "file_location": "s3://retrofit-data-dev/" + file_location,
- "property_id": 999,
- "portfolio_id": 4,
- "created_at": created_at
- }
+ sap_change_model_api = SAPChangeModelAPI()
+ response = sap_change_model_api.predict(
+ file_location="s3://retrofit-data-dev/" + file_location,
+ created_at=created_at,
+ portfolio_id=body.portfolio_id
)
- # TODO: Handle the response depending on response code
# Retrieve the predictions
- predictions = read_csv_from_s3(
- bucket_name="retrofit-sap-predictions-dev",
- filepath=f"{body.portfolio_id}/999/{created_at}.csv"
- )
- predictions = pd.DataFrame(predictions)
+ predictions = pd.DataFrame(read_csv_from_s3(
+ bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT),
+ filepath=response["storage_filepath"]
+ ))
# We round the predictions
predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(0)
# Extract property_id and recommendation_id
predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True)
+ # Insert the predictions into the recommendations and run the optimiser
+ for property_id in recommendations.keys():
+
+ property = [p for p in input_properties if p.id == property_id][0]
+ property_predictions = predictions[predictions["property_id"] == str(property_id)]
+
+ for rec in recommendations[property_id]:
+ rec["sap_points"] = property_predictions[property_predictions["recommendation_id"] == str(
+ rec["recommendation_id"]
+ )]["RDSAP_CHANGE"].values[0]
+
+ input_measures = prepare_input_measures(recommendations[property_id], body.goal)
+
+ if body.budget:
+ optimiser = GainOptimiser(input_measures, max_cost=body.budget)
+ else:
+ # The minimum gain is the minimum number of SAP points required to get to the target SAP band
+ current_sap_points = int(property.data["current-energy-efficiency"])
+ target_sap_points = epc_to_sap_lower_bound(body.goal_value)
+
+ # If the gain is negative, the optimiser will return an empty solution
+ optimiser = CostOptimiser(
+ input_measures, min_gain=target_sap_points - current_sap_points
+ )
+
+ optimiser.setup()
+ optimiser.solve()
+ solution = optimiser.solution
+
+ selected_recommendations = {r["id"] for r in solution}
+
+ # For selected recommendations, mark them as default
+ for rec in recommendations[property_id]:
+ rec["default"] = rec["recommendation_id"] in selected_recommendations
+
+ for p in input_properties:
+ property_recommendations = [
+ [
+ {**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
+ for rec in recommendations_by_type
+ ]
+ for recommendations_by_type in property_recommendations
+ ]
+
+ input_measures = prepare_input_measures(property_recommendations, body.goal)
+
# 1) the property data
# 2) the property details (epc)
# 3) the recommendations
diff --git a/backend/ml_models/sap_change_model/__init__.py b/backend/ml_models/sap_change_model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/backend/ml_models/sap_change_model/api.py b/backend/ml_models/sap_change_model/api.py
new file mode 100644
index 00000000..03c8423d
--- /dev/null
+++ b/backend/ml_models/sap_change_model/api.py
@@ -0,0 +1,44 @@
+import requests
+from requests.exceptions import RequestException
+from utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class SAPChangeModelAPI:
+ def __init__(self, base_url="https://api.dev.hestia.homes"):
+ self.base_url = base_url
+
+ def predict(self, file_location, property_id="", portfolio_id=4, created_at=None):
+ """Makes a POST request to the SAP Change Model API with the provided parameters.
+
+ Args:
+ file_location (str): The file location to be passed in the request payload.
+ property_id (int, optional): The property ID to be passed in the request payload. Defaults to 999.
+ portfolio_id (int, optional): The portfolio ID to be passed in the request payload. Defaults to 4.
+ created_at (str, optional): The creation timestamp to be passed in the request payload. Defaults to None.
+
+ Returns:
+ dict: The API response as a dictionary if the request was successful, None otherwise.
+ """
+ url = f"{self.base_url}/sapmodel/predict"
+ payload = {
+ "file_location": f"s3://retrofit-data-dev/{file_location}",
+ "property_id": property_id,
+ "portfolio_id": portfolio_id,
+ "created_at": created_at
+ }
+
+ try:
+ response = requests.post(url, json=payload)
+
+ # Check if the response status code is 2xx (success)
+ response.raise_for_status()
+
+ # Return the JSON response as a Python dictionary
+ return response.json()
+ except RequestException as e:
+ logger.error(f"An error occurred: {e}")
+ # In case of an error, you might want to return None or raise the exception
+ # depending on how you want to handle errors in your application
+ return None
diff --git a/model_data/optimiser/optimiser_functions.py b/model_data/optimiser/optimiser_functions.py
index 869880cf..6ff0050a 100644
--- a/model_data/optimiser/optimiser_functions.py
+++ b/model_data/optimiser/optimiser_functions.py
@@ -17,7 +17,7 @@ def prepare_input_measures(property_recommendations, goal):
raise NotImplementedError("Not implemented this gain type - investigate me")
input_measures = []
- for recs in property_recommendations:
+ for rec in property_recommendations:
input_measures.append(
[
{
@@ -26,7 +26,6 @@ def prepare_input_measures(property_recommendations, goal):
"gain": rec[goal_key],
"type": rec["type"]
}
- for rec in recs
]
)
diff --git a/model_data/simulation_system/core/DataProcessor.py b/model_data/simulation_system/core/DataProcessor.py
index a0e0bbc8..6d61d4d5 100644
--- a/model_data/simulation_system/core/DataProcessor.py
+++ b/model_data/simulation_system/core/DataProcessor.py
@@ -1,8 +1,8 @@
from pathlib import Path
import numpy as np
import pandas as pd
-from BaseUtility import Definitions
-from simulation_system.core.Settings import (
+from model_data.BaseUtility import Definitions
+from model_data.simulation_system.core.Settings import (
DATA_PROCESSOR_SETTINGS,
EARLIEST_EPC_DATE,
FULLY_GLAZED_DESCRIPTIONS,