junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

finishing implementation of the model wip

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2023-09-12 17:18:10 +01:00
parent 16f090d987
commit 28bdc119fd
8 changed files with 121 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/unit_tests.yml vendored
View file

@ -1,6 +1,9 @@
name: Run unit tests
on: [ push, pull_request ]
on:
push:
branches:
- main
jobs:
build:

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
.idea/misc.xml generated
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>

118
backend/app/plan/router.py
View file

@ -17,7 +17,9 @@ from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError, OperationalError
from datetime import datetime
import pandas as pd
import requests
# model apis
from backend.ml_models.sap_change_model.api import SAPChangeModelAPI
# database interaction functions
from backend.app.db.functions.property_functions import (
@ -136,6 +138,13 @@ def insert_temp_recommendation_id(property_recommendations):
return property_recommendations
def score_measures():
"""
This wrapper function prepares data to be passed to the sap model api
:return:
"""
@router.post("/trigger")
async def trigger_plan(body: PlanTriggerRequest):
logger.info("Connecting to db")
@ -289,43 +298,13 @@ async def trigger_plan(body: PlanTriggerRequest):
if wall_recomender.recommendations:
property_recommendations.append(wall_recomender.recommendations)
# Use the optimiser to pick the default recommendations and decide if we need certain
# recommendations to get to the goal
# We insert temporary ids into the recommendations which is important for the optimiser later
property_recommendations = insert_temp_recommendation_id(property_recommendations)
if not property_recommendations:
continue
input_measures = prepare_input_measures(property_recommendations, body.goal)
if body.budget:
optimiser = GainOptimiser(input_measures, max_cost=body.budget)
else:
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
current_sap_points = int(p.data["current-energy-efficiency"])
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
# If the gain is negative, the optimiser will return an empty solution
optimiser = CostOptimiser(
input_measures, min_gain=target_sap_points - current_sap_points
)
optimiser.setup()
optimiser.solve()
solution = optimiser.solution
selected_recommendations = {r["id"] for r in solution}
# We'll use the set of selected recommendations to filter the recommendations to upload
property_recommendations = [
[
{**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
for rec in recommendations_by_type
]
for recommendations_by_type in property_recommendations
]
# We'll also unlist the recommendations so they're a bit easier to handle from here onwards
# We'll unlist the recommendations so they're a bit easier to handle from here onwards
property_recommendations = [
rec for recommendations_by_type in property_recommendations for rec in recommendations_by_type
]
@ -373,7 +352,6 @@ async def trigger_plan(body: PlanTriggerRequest):
recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
# TODO: Set the TRANSACTION_TYPE
# Clean the data
cleaning_data = read_parquet_from_s3(
bucket_name="retrofit-data-dev",
@ -411,33 +389,73 @@ async def trigger_plan(body: PlanTriggerRequest):
save_dataframe_to_s3_parquet(
df=recommendations_scoring_data,
bucket_name="retrofit-data-dev",
bucket_name="retrofit-data-{environment}".format(environment=get_settings().ENVIRONMENT),
file_key=file_location
)
# Call the sap change model
response = requests.post(
url="https://api.dev.hestia.homes/sapmodel/predict",
json={
"file_location": "s3://retrofit-data-dev/" + file_location,
"property_id": 999,
"portfolio_id": 4,
"created_at": created_at
}
sap_change_model_api = SAPChangeModelAPI()
response = sap_change_model_api.predict(
file_location="s3://retrofit-data-dev/" + file_location,
created_at=created_at,
portfolio_id=body.portfolio_id
)
# TODO: Handle the response depending on response code
# Retrieve the predictions
predictions = read_csv_from_s3(
bucket_name="retrofit-sap-predictions-dev",
filepath=f"{body.portfolio_id}/999/{created_at}.csv"
)
predictions = pd.DataFrame(predictions)
predictions = pd.DataFrame(read_csv_from_s3(
bucket_name="retrofit-sap-predictions-{environment}".format(environment=get_settings().ENVIRONMENT),
filepath=response["storage_filepath"]
))
# We round the predictions
predictions["RDSAP_CHANGE"] = predictions["RDSAP_CHANGE"].astype(float).round(0)
# Extract property_id and recommendation_id
predictions[['property_id', 'recommendation_id']] = predictions['id'].str.split('+', expand=True)
# Insert the predictions into the recommendations and run the optimiser
for property_id in recommendations.keys():
property = [p for p in input_properties if p.id == property_id][0]
property_predictions = predictions[predictions["property_id"] == str(property_id)]
for rec in recommendations[property_id]:
rec["sap_points"] = property_predictions[property_predictions["recommendation_id"] == str(
rec["recommendation_id"]
)]["RDSAP_CHANGE"].values[0]
input_measures = prepare_input_measures(recommendations[property_id], body.goal)
if body.budget:
optimiser = GainOptimiser(input_measures, max_cost=body.budget)
else:
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
current_sap_points = int(property.data["current-energy-efficiency"])
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
# If the gain is negative, the optimiser will return an empty solution
optimiser = CostOptimiser(
input_measures, min_gain=target_sap_points - current_sap_points
)
optimiser.setup()
optimiser.solve()
solution = optimiser.solution
selected_recommendations = {r["id"] for r in solution}
# For selected recommendations, mark them as default
for rec in recommendations[property_id]:
rec["default"] = rec["recommendation_id"] in selected_recommendations
for p in input_properties:
property_recommendations = [
[
{**rec, "default": True if rec["recommendation_id"] in selected_recommendations else False}
for rec in recommendations_by_type
]
for recommendations_by_type in property_recommendations
]
input_measures = prepare_input_measures(property_recommendations, body.goal)
# 1) the property data
# 2) the property details (epc)
# 3) the recommendations

0
backend/ml_models/sap_change_model/__init__.py Normal file
View file

44
backend/ml_models/sap_change_model/api.py Normal file
View file

@ -0,0 +1,44 @@
import requests
from requests.exceptions import RequestException
from utils.logger import setup_logger
logger = setup_logger()
class SAPChangeModelAPI:
def __init__(self, base_url="https://api.dev.hestia.homes"):
self.base_url = base_url
def predict(self, file_location, property_id="", portfolio_id=4, created_at=None):
"""Makes a POST request to the SAP Change Model API with the provided parameters.
Args:
file_location (str): The file location to be passed in the request payload.
property_id (int, optional): The property ID to be passed in the request payload. Defaults to 999.
portfolio_id (int, optional): The portfolio ID to be passed in the request payload. Defaults to 4.
created_at (str, optional): The creation timestamp to be passed in the request payload. Defaults to None.
Returns:
dict: The API response as a dictionary if the request was successful, None otherwise.
"""
url = f"{self.base_url}/sapmodel/predict"
payload = {
"file_location": f"s3://retrofit-data-dev/{file_location}",
"property_id": property_id,
"portfolio_id": portfolio_id,
"created_at": created_at
}
try:
response = requests.post(url, json=payload)
# Check if the response status code is 2xx (success)
response.raise_for_status()
# Return the JSON response as a Python dictionary
return response.json()
except RequestException as e:
logger.error(f"An error occurred: {e}")
# In case of an error, you might want to return None or raise the exception
# depending on how you want to handle errors in your application
return None

3
model_data/optimiser/optimiser_functions.py
View file

@ -17,7 +17,7 @@ def prepare_input_measures(property_recommendations, goal):
raise NotImplementedError("Not implemented this gain type - investigate me")
input_measures = []
for recs in property_recommendations:
for rec in property_recommendations:
input_measures.append(
[
{
@ -26,7 +26,6 @@ def prepare_input_measures(property_recommendations, goal):
"gain": rec[goal_key],
"type": rec["type"]
}
for rec in recs
]
)

4
model_data/simulation_system/core/DataProcessor.py
View file

@ -1,8 +1,8 @@
from pathlib import Path
import numpy as np
import pandas as pd
from BaseUtility import Definitions
from simulation_system.core.Settings import (
from model_data.BaseUtility import Definitions
from model_data.simulation_system.core.Settings import (
DATA_PROCESSOR_SETTINGS,
EARLIEST_EPC_DATE,
FULLY_GLAZED_DESCRIPTIONS,