completed mds outputs for the moment

backend/Outputs.py

@@ -1,6 +1,10 @@
+import msgpack
 import pandas as pd
+import numpy as np
 from sqlalchemy.orm import sessionmaker
+from datetime import datetime
 
+from utils.s3 import read_from_s3, save_excel_to_s3
 from backend.app.utils import sap_to_epc
 from backend.app.db.connection import db_engine
 from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel
@@ -55,10 +59,19 @@ class Outputs:
 
         self.format = format
         self.portfolio_id = portfolio_id
+        self.today = datetime.now().strftime("%Y-%m-%d")
 
         # Connect to the database
         self.session = sessionmaker(bind=db_engine)()
 
+        # Download cleaned data
+        self.cleaned_epc_lookup = read_from_s3(
+            s3_file_name="cleaned_epc_data/cleaned.bson",
+            bucket_name="retrofit-data-dev"
+        )
+
+        self.cleaned_epc_lookup = msgpack.unpackb(self.cleaned_epc_lookup, raw=False)
+
     def get_properties_from_db(self):
         # Get properties and their details for a specific portfolio
         properties_query = self.session.query(
@@ -204,14 +217,19 @@ class Outputs:
                 "uprn",
                 "current_epc_rating",
                 "current_sap_points",
-                # TODO: Need to add current heat demand
+                "primary_energy_consumption",
                 "property_type",
                 "built_form",
                 "total_floor_area",
                 "walls",
                 "tenure",
                 "mainfuel",
-                # TODO: For estimated bill, this should probably be without the cost of appliances
+                # The bills columns are split out - we include them and aggregate, without appliances
+                "heating_cost_current",
+                "hot_water_cost_current",
+                "lighting_cost_current",
+                "gas_standing_charge",
+                "electricity_standing_charge"
             ]
         ].copy().rename(
             columns={
@@ -226,17 +244,46 @@ class Outputs:
                 "total_floor_area": "Floor area m2 (If known)",
                 "walls": "Wall Type (Mandatory field)",
                 "tenure": "Tenure",
-                "mainfuel": "Existing Fuel Type"
-                # TODO: For estimated bill, this should probably be without the cost of appliances
             }
         )
 
-        # TODO - format
+        mds["Estimated bill (£ per year)"] = (
-        #       1) property type
+            mds["heating_cost_current"] +
-        #       2) walls
+            mds["hot_water_cost_current"] +
-        #       3) tenure
+            mds["lighting_cost_current"] +
-        #       4) mainfuel
+            mds["gas_standing_charge"] +
-        #       5) Epc Rating
+            mds["electricity_standing_charge"]
+        )
+
+        mds = mds.drop(
+            columns=[
+                "heating_cost_current",
+                "hot_water_cost_current",
+                "lighting_cost_current",
+                "gas_standing_charge",
+                "electricity_standing_charge"
+            ]
+        )
+
+        # Formatting - Pre EPC is an enum
+        mds["Pre EPC"] = [x.value for x in mds["Pre EPC"].values]
+        mds["Wall Type (Mandatory field)"] = mds["Wall Type (Mandatory field)"].str.split(",").str[0]
+        # Remove average thermal transmittance field
+        mds["Wall Type (Mandatory field)"] = np.where(
+            mds["Wall Type (Mandatory field)"].str.contains("Average thermal transmittance"),
+            "",
+            mds["Wall Type (Mandatory field)"]
+        )
+
+        mds = mds.merge(
+            pd.DataFrame(self.cleaned_epc_lookup["main-fuel"])[["clean_description", "fuel_type"]],
+            left_on="mainfuel",
+            right_on="clean_description",
+            how="left"
+        )
+        mds = mds.rename(columns={"fuel_type": "Existing Fuel Type"}).drop(columns=["clean_description", "mainfuel"])
+
+        mds["Existing Fuel Type"].value_counts()
 
         mds_output_by_scenario = {}
         for scenario_id in scenario_ids:
@@ -264,8 +311,9 @@ class Outputs:
             # Round Post SAP down to the nearest integer
             scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x))
             scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x))
-
+            scenario_mds["Heating Demand Kwh/m2/y"] = (
-            # TODO: Post heat demand
+                scenario_mds["Existing Heating Demand Kwh/m2/y"] - scenario_mds["heat_demand"]
+            )
 
             scenario_mds = scenario_mds.rename(
                 columns={
@@ -275,9 +323,21 @@ class Outputs:
                 }
             )
 
+            mds_output_by_scenario[scenario_id] = scenario_mds
+
+        # We now save them to s3 as excels
+        for scenario_id, scenario_mds in mds_output_by_scenario.items():
+            save_excel_to_s3(
+                df=scenario_mds,
+                file_key=f"engine_outputs/{self.format}/{self.today}_scenario_id={scenario_id}.xlsx",
+                bucket_name="retrofit-data-dev"
+            )
+
     def export(self):
         """
         This function will export the data in the required format
         """
         if self.format == "mds":
             self.export_mds()
+
+        raise NotImplementedError("Export format not implemented")

backend/app/db/functions/recommendations_functions.py

@@ -108,6 +108,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
             {
                 "property_id": property_id,
                 "type": rec["type"],
+                "measure_type": rec["measure_type"],
                 "description": rec["description"],
                 "estimated_cost": rec["total"],
                 "default": rec["default"],
@@ -121,7 +122,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
                 "energy_cost_savings": rec["energy_cost_savings"],
                 "labour_days": rec["labour_days"],
                 "already_installed": rec["already_installed"],
-                "head_demand": rec["heat_demand"]
+                "heat_demand": rec["heat_demand"]
             }
             for rec in recommendations_to_upload
         ]

backend/app/db/models/recommendations.py

@@ -15,6 +15,7 @@ class Recommendation(Base):
     property_id = Column(BigInteger, ForeignKey(PropertyModel.id), nullable=False)
     created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())
     type = Column(String, nullable=False)
+    measure_type = Column(String)
     description = Column(String, nullable=False)
     estimated_cost = Column(Float)
     default = Column(Boolean, nullable=False)

backend/app/plan/router.py

@@ -126,8 +126,8 @@ def extract_portfolio_aggregation_data(
         pre_retrofit_co2 = p.data["co2-emissions-current"]
         post_retrofit_co2 = pre_retrofit_co2 - carbon_savings
 
-        pre_retrofit_energy_bill = p.current_energy_bill
+        pre_retrofit_energy_bill = sum(p.current_energy_bill.values())
-        post_retrofit_energy_bill = p.current_energy_bill - sum(
+        post_retrofit_energy_bill = sum(p.current_energy_bill.values()) - sum(
             [r["energy_cost_savings"] for r in default_recommendations]
         )