diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index ca0e1cd9..3b05c6ac 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3150f870..aeb5235a 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -1,46 +1,40 @@ -from collections import defaultdict -from fastapi import APIRouter, Depends -from backend.app.db.models.portfolio import rating_lookup -from backend.app.dependencies import validate_token -from backend.app.plan.schemas import PlanTriggerRequest -from backend.app.utils import read_csv_from_s3 -from backend.app.config import get_settings -from backend.Property import Property -from epc_api.client import EpcClient -from utils.logger import setup_logger -from utils.s3 import read_from_s3 -from recommendations.FloorRecommendations import FloorRecommendations -from recommendations.WallRecommendations import WallRecommendations -from recommendations.config import UPGRADES_MAP -from utils.uvalue_estimates import classify_decile_newvalues -from backend.app.db.utils import row2dict -from starlette.responses import Response -from sqlalchemy.orm import sessionmaker -from sqlalchemy.exc import IntegrityError, OperationalError from datetime import datetime + import pandas as pd -import msgpack +from epc_api.client import EpcClient +from fastapi import APIRouter, Depends +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.orm import sessionmaker +from starlette.responses import Response -# model apis -from backend.ml_models.sap_change_model.api import SAPChangeModelAPI - -# database interaction functions -from backend.app.db.functions.property_functions import ( - create_property, create_property_targets, update_property_data, create_property_details_epc -) +from backend.app.config import get_settings +from backend.app.db.connection import db_engine from backend.app.db.functions.materials_functions import get_materials +from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations +from backend.app.db.functions.property_functions import ( + create_property, create_property_details_epc, create_property_targets, update_property_data +) from backend.app.db.functions.recommendations_functions import ( create_plan, create_plan_recommendations, upload_recommendations ) -from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations -from backend.app.db.connection import db_engine +from backend.app.db.models.portfolio import rating_lookup +from backend.app.dependencies import validate_token +from backend.app.plan.schemas import PlanTriggerRequest +from backend.app.plan.utils import ( + create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id +) +from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 -from model_data.optimiser.GainOptimiser import GainOptimiser -from model_data.optimiser.CostOptimiser import CostOptimiser -from backend.app.utils import epc_to_sap_lower_bound, read_parquet_from_s3 -from model_data.optimiser.optimiser_functions import prepare_input_measures +from backend.ml_models.sap_change_model.api import SAPChangeModelAPI +from backend.Property import Property from etl.property_change.DataProcessor import DataProcessor from etl.property_change.settings import COLUMNS_TO_MERGE_ON +from recommendations.FloorRecommendations import FloorRecommendations +from recommendations.optimiser.CostOptimiser import CostOptimiser +from recommendations.optimiser.GainOptimiser import GainOptimiser +from recommendations.optimiser.optimiser_functions import prepare_input_measures +from recommendations.WallRecommendations import WallRecommendations +from utils.logger import setup_logger logger = setup_logger() @@ -51,132 +45,6 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) -# TODO: Load this data from db -open_uprn_data = [ - {'UPRN': 6032920, 'X_COORDINATE': 535110.0, 'Y_COORDINATE': 181819.0, 'LATITUDE': 51.5191407, - 'LONGITUDE': -0.0540506}, - {'UPRN': 6038625, 'X_COORDINATE': 535374.0, 'Y_COORDINATE': 182784.0, 'LATITUDE': 51.5277492, - 'LONGITUDE': -0.0498772}, - {'UPRN': 34153991, 'X_COORDINATE': 523238.74, 'Y_COORDINATE': 178003.02, 'LATITUDE': 51.4875579, - 'LONGITUDE': -0.226392}, - {'UPRN': 10008299676, 'X_COORDINATE': 533285.0, 'Y_COORDINATE': 184711.0, 'LATITUDE': 51.5455629, - 'LONGITUDE': -0.0792445}, - {'UPRN': 10008299677, 'X_COORDINATE': 533285.0, 'Y_COORDINATE': 184711.0, 'LATITUDE': 51.5455629, - 'LONGITUDE': -0.0792445}, - {'UPRN': 100021039066, 'X_COORDINATE': 535506.0, 'Y_COORDINATE': 185624.0, 'LATITUDE': 51.5532385, - 'LONGITUDE': -0.0468833}, - {'UPRN': 100021226060, 'X_COORDINATE': 529247.0, 'Y_COORDINATE': 187959.0, 'LATITUDE': 51.5756908, - 'LONGITUDE': -0.1362513}, - {'UPRN': 200003489276, 'X_COORDINATE': 533210.0, 'Y_COORDINATE': 179442.0, 'LATITUDE': 51.4982309, - 'LONGITUDE': -0.0823165} -] - -in_conservation_area_data = [ - {'uprn': 6032920, 'is_in_conservation_area': 'not_in_conservation_area'}, - {'uprn': 6038625, 'is_in_conservation_area': 'not_in_conservation_area'}, - {'uprn': 34153991, 'is_in_conservation_area': 'unknown'}, - {'uprn': 10008299676, 'is_in_conservation_area': 'in_conservation_area'}, - {'uprn': 10008299677, 'is_in_conservation_area': 'in_conservation_area'}, - {'uprn': 100021039066, 'is_in_conservation_area': 'not_in_conservation_area'}, - {'uprn': 100021226060, 'is_in_conservation_area': 'in_conservation_area'}, - {'uprn': 200003489276, 'is_in_conservation_area': 'in_conservation_area'} -] - -# TODO: db -floors_decile_data = { - 'decile_labels': ['Decile 1', 'Decile 2', 'Decile 3', 'Decile 4', 'Decile 5', 'Decile 6', 'Decile 7', 'Decile 8', - 'Decile 9', 'Decile 10'], 'decile_boundaries': [6., 50., 56., 69., 77.6, 87., 98., 112., - 127., 150., 2279.]} - -walls_decile_data = { - 'decile_labels': ['Decile 1', 'Decile 2', 'Decile 3', 'Decile 4', 'Decile 5', 'Decile 6', 'Decile 7', 'Decile 8', - 'Decile 9', 'Decile 10'], 'decile_boundaries': [6., 49., 51., 55., 64., 71., 76., 83., 96., - 120., 2279.]} - - -def filter_materials(materials): - materials_by_type = defaultdict(list) - - for material in materials: - material = row2dict(material) - material_type = material["type"] - materials_by_type[material_type].append(material) - - # Optionally, you can convert the defaultdict to a normal dict if desired - materials_by_type = dict(materials_by_type) - - return materials_by_type - - -def insert_temp_recommendation_id(property_recommendations): - """ - Creates a temporary recommendation id which is needed for - filtering recommendations between default and no, after the optimiser has been - run - :param property_recommendations: nested list of recommendations, grouped by data_types - :return: Updated recommendations_to_upload, where where recommendation has a "recommendation_id" - integer inserted - """ - idx = 0 - - for recs in property_recommendations: - for rec in recs: - rec["recommendation_id"] = idx - idx += 1 - - return property_recommendations - - -def get_cleaned(): - """ - This function will retrieve the cleaned dataset from s3 which has the cleaned - descriptions for the epc dataset - - This data is stored in MessagePack format and therefore needs to be decoded - :return: - """ - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-{environment}".format(environment=get_settings().ENVIRONMENT) - ) - - cleaned = msgpack.unpackb(cleaned, raw=False) - - return cleaned - - -def create_recommendation_scoring_data( - property: Property, - recommendation: dict, - starting_epc_data: pd.DataFrame, - ending_epc_data: pd.DataFrame, - fixed_data: pd.DataFrame, -): - """ - This wrapper function prepares data to be passed to the sap model api - :return: - """ - - scoring_dict = { - "UPRN": property.data["uprn"], - "id": "+".join([str(property.id), str(recommendation["recommendation_id"])]), - "LOCAL_AUTHORITY": property.data["local-authority"], - **starting_epc_data.to_dict("records")[0], - **ending_epc_data.to_dict("records")[0], - **fixed_data.to_dict("records")[0] - } - - # We update the description to indicate it's insulated - if recommendation["type"] == "wall_insulation": - scoring_dict["WALLS_DESCRIPTION_ENDING"] = UPGRADES_MAP[property.walls["clean_description"]] - elif recommendation["type"] == "floor_insulation": - scoring_dict["FLOOR_DESCRIPTION_ENDING"] = UPGRADES_MAP[property.floor["clean_description"]] - else: - raise NotImplementedError("Implement me") - - return scoring_dict - @router.post("/trigger") async def trigger_plan(body: PlanTriggerRequest): @@ -259,20 +127,12 @@ async def trigger_plan(body: PlanTriggerRequest): for p in input_properties: property_recommendations = [] - # For each property, classiy floor area decide - total_floor_area_group_decile = classify_decile_newvalues( - decile_boundaries=floors_decile_data["decile_boundaries"], - decile_labels=floors_decile_data["decile_labels"], - new_values=[float(p.data["total-floor-area"])], - )[0] - # Property recommendations p.get_components(cleaned) # Floor recommendations floor_recommender = FloorRecommendations( property_instance=p, - total_floor_area_group_decile=total_floor_area_group_decile, materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"], ) floor_recommender.recommend() @@ -281,29 +141,9 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations.append(floor_recommender.recommendations) # Wall recommendations - # We would make this u-value query directly to the database - total_floor_area_group_decile = classify_decile_newvalues( - decile_boundaries=walls_decile_data["decile_boundaries"], - decile_labels=walls_decile_data["decile_labels"], - new_values=[float(p.data["total-floor-area"])], - )[0] - - # This is placeholder, until the full dataset is loaded into the database and we just make a read to the - # database - walls_u_value_estimate = [ - x for x in uvalue_estimates_walls - if (x['local-authority'] == p.data["local-authority"]) & - (x['property-type'] == p.data["property-type"]) & - (x['built-form'] == p.data["built-form"]) & - (x['walls-energy-eff'] == p.data["walls-energy-eff"] if p.data[ - "walls-energy-eff"] != 'N/A' else True) & - (x['walls-env-eff'] == p.data["walls-env-eff"] if p.data["walls-env-eff"] != 'N/A' else True) - ] wall_recomender = WallRecommendations( property_instance=p, - uvalue_estimates=walls_u_value_estimate, - total_floor_area_group_decile=total_floor_area_group_decile, materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"] ) wall_recomender.recommend() diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py new file mode 100644 index 00000000..b635f9ee --- /dev/null +++ b/backend/app/plan/utils.py @@ -0,0 +1,89 @@ +from collections import defaultdict +from utils.s3 import read_from_s3 +from recommendations.config import UPGRADES_MAP +from backend.app.db.utils import row2dict +import msgpack + + +def filter_materials(materials): + materials_by_type = defaultdict(list) + + for material in materials: + material = row2dict(material) + material_type = material["type"] + materials_by_type[material_type].append(material) + + # Optionally, you can convert the defaultdict to a normal dict if desired + materials_by_type = dict(materials_by_type) + + return materials_by_type + + +def insert_temp_recommendation_id(property_recommendations): + """ + Creates a temporary recommendation id which is needed for + filtering recommendations between default and no, after the optimiser has been + run + :param property_recommendations: nested list of recommendations, grouped by data_types + :return: Updated recommendations_to_upload, where where recommendation has a "recommendation_id" + integer inserted + """ + idx = 0 + + for recs in property_recommendations: + for rec in recs: + rec["recommendation_id"] = idx + idx += 1 + + return property_recommendations + + +def get_cleaned(): + """ + This function will retrieve the cleaned dataset from s3 which has the cleaned + descriptions for the epc dataset + + This data is stored in MessagePack format and therefore needs to be decoded + :return: + """ + + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-{environment}".format(environment=get_settings().ENVIRONMENT) + ) + + cleaned = msgpack.unpackb(cleaned, raw=False) + + return cleaned + + +def create_recommendation_scoring_data( + property: Property, + recommendation: dict, + starting_epc_data: pd.DataFrame, + ending_epc_data: pd.DataFrame, + fixed_data: pd.DataFrame, +): + """ + This wrapper function prepares data to be passed to the sap model api + :return: + """ + + scoring_dict = { + "UPRN": property.data["uprn"], + "id": "+".join([str(property.id), str(recommendation["recommendation_id"])]), + "LOCAL_AUTHORITY": property.data["local-authority"], + **starting_epc_data.to_dict("records")[0], + **ending_epc_data.to_dict("records")[0], + **fixed_data.to_dict("records")[0] + } + + # We update the description to indicate it's insulated + if recommendation["type"] == "wall_insulation": + scoring_dict["WALLS_DESCRIPTION_ENDING"] = UPGRADES_MAP[property.walls["clean_description"]] + elif recommendation["type"] == "floor_insulation": + scoring_dict["FLOOR_DESCRIPTION_ENDING"] = UPGRADES_MAP[property.floor["clean_description"]] + else: + raise NotImplementedError("Implement me") + + return scoring_dict diff --git a/etl/epc_clean/requirements.txt b/etl/epc_clean/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/etl/property_change/DataProcessor.py b/etl/property_change/DataProcessor.py index a5843500..afa0682d 100644 --- a/etl/property_change/DataProcessor.py +++ b/etl/property_change/DataProcessor.py @@ -1,7 +1,7 @@ from pathlib import Path import numpy as np import pandas as pd -from model_data.BaseUtility import Definitions +from BaseUtility import Definitions from etl.property_change.settings import ( DATA_PROCESSOR_SETTINGS, EARLIEST_EPC_DATE, diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index be32a0fb..ef8c1a68 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -39,11 +39,9 @@ class FloorRecommendations(Definitions): def __init__( self, property_instance: Property, - total_floor_area_group_decile: str, materials: List, ): self.property = property_instance - self.total_floor_area_group_decile = total_floor_area_group_decile # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None