diff --git a/backend/Outputs.py b/backend/Outputs.py index 9cc3e38c..f9538709 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -1,6 +1,10 @@ +import msgpack import pandas as pd +import numpy as np from sqlalchemy.orm import sessionmaker +from datetime import datetime +from utils.s3 import read_from_s3, save_excel_to_s3 from backend.app.utils import sap_to_epc from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel @@ -55,10 +59,19 @@ class Outputs: self.format = format self.portfolio_id = portfolio_id + self.today = datetime.now().strftime("%Y-%m-%d") # Connect to the database self.session = sessionmaker(bind=db_engine)() + # Download cleaned data + self.cleaned_epc_lookup = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + self.cleaned_epc_lookup = msgpack.unpackb(self.cleaned_epc_lookup, raw=False) + def get_properties_from_db(self): # Get properties and their details for a specific portfolio properties_query = self.session.query( @@ -204,14 +217,19 @@ class Outputs: "uprn", "current_epc_rating", "current_sap_points", - # TODO: Need to add current heat demand + "primary_energy_consumption", "property_type", "built_form", "total_floor_area", "walls", "tenure", "mainfuel", - # TODO: For estimated bill, this should probably be without the cost of appliances + # The bills columns are split out - we include them and aggregate, without appliances + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" ] ].copy().rename( columns={ @@ -226,17 +244,46 @@ class Outputs: "total_floor_area": "Floor area m2 (If known)", "walls": "Wall Type (Mandatory field)", "tenure": "Tenure", - "mainfuel": "Existing Fuel Type" - # TODO: For estimated bill, this should probably be without the cost of appliances } ) - # TODO - format - # 1) property type - # 2) walls - # 3) tenure - # 4) mainfuel - # 5) Epc Rating + mds["Estimated bill (£ per year)"] = ( + mds["heating_cost_current"] + + mds["hot_water_cost_current"] + + mds["lighting_cost_current"] + + mds["gas_standing_charge"] + + mds["electricity_standing_charge"] + ) + + mds = mds.drop( + columns=[ + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" + ] + ) + + # Formatting - Pre EPC is an enum + mds["Pre EPC"] = [x.value for x in mds["Pre EPC"].values] + mds["Wall Type (Mandatory field)"] = mds["Wall Type (Mandatory field)"].str.split(",").str[0] + # Remove average thermal transmittance field + mds["Wall Type (Mandatory field)"] = np.where( + mds["Wall Type (Mandatory field)"].str.contains("Average thermal transmittance"), + "", + mds["Wall Type (Mandatory field)"] + ) + + mds = mds.merge( + pd.DataFrame(self.cleaned_epc_lookup["main-fuel"])[["clean_description", "fuel_type"]], + left_on="mainfuel", + right_on="clean_description", + how="left" + ) + mds = mds.rename(columns={"fuel_type": "Existing Fuel Type"}).drop(columns=["clean_description", "mainfuel"]) + + mds["Existing Fuel Type"].value_counts() mds_output_by_scenario = {} for scenario_id in scenario_ids: @@ -264,8 +311,9 @@ class Outputs: # Round Post SAP down to the nearest integer scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x)) scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x)) - - # TODO: Post heat demand + scenario_mds["Heating Demand Kwh/m2/y"] = ( + scenario_mds["Existing Heating Demand Kwh/m2/y"] - scenario_mds["heat_demand"] + ) scenario_mds = scenario_mds.rename( columns={ @@ -275,9 +323,21 @@ class Outputs: } ) + mds_output_by_scenario[scenario_id] = scenario_mds + + # We now save them to s3 as excels + for scenario_id, scenario_mds in mds_output_by_scenario.items(): + save_excel_to_s3( + df=scenario_mds, + file_key=f"engine_outputs/{self.format}/{self.today}_scenario_id={scenario_id}.xlsx", + bucket_name="retrofit-data-dev" + ) + def export(self): """ This function will export the data in the required format """ if self.format == "mds": self.export_mds() + + raise NotImplementedError("Export format not implemented") diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 5f6791f2..feeced10 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -108,6 +108,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property { "property_id": property_id, "type": rec["type"], + "measure_type": rec["measure_type"], "description": rec["description"], "estimated_cost": rec["total"], "default": rec["default"], @@ -121,7 +122,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], "already_installed": rec["already_installed"], - "head_demand": rec["heat_demand"] + "heat_demand": rec["heat_demand"] } for rec in recommendations_to_upload ] diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index a1743436..1089dced 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -15,6 +15,7 @@ class Recommendation(Base): property_id = Column(BigInteger, ForeignKey(PropertyModel.id), nullable=False) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) type = Column(String, nullable=False) + measure_type = Column(String) description = Column(String, nullable=False) estimated_cost = Column(Float) default = Column(Boolean, nullable=False) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6b8c5bea..2dc03e60 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -126,8 +126,8 @@ def extract_portfolio_aggregation_data( pre_retrofit_co2 = p.data["co2-emissions-current"] post_retrofit_co2 = pre_retrofit_co2 - carbon_savings - pre_retrofit_energy_bill = p.current_energy_bill - post_retrofit_energy_bill = p.current_energy_bill - sum( + pre_retrofit_energy_bill = sum(p.current_energy_bill.values()) + post_retrofit_energy_bill = sum(p.current_energy_bill.values()) - sum( [r["energy_cost_savings"] for r in default_recommendations] )