implemented required measures for mod

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="AssetList" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

asset_list/AssetList.py

@@ -545,7 +545,10 @@ class AssetList:
                 raise ValueError("Missing full address - please specify columns to concatenate")
             self.full_address_colname = self.STANDARD_FULL_ADDRESS
             self.standardised_asset_list[self.full_address_colname] = (
-                self.standardised_asset_list[self.full_address_cols_to_concat].apply(lambda x: ", ".join(x), axis=1)
+                self.standardised_asset_list[self.full_address_cols_to_concat].apply(
+                    lambda x: ", ".join([y for y in x if not pd.isnull(y)]),
+                    axis=1
+                )
             )
         else:
 

asset_list/app.py

@@ -88,6 +88,31 @@ def app():
     # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid)
     # - Or the insulation required is loft/cavity (floors should be solid)
 
+    # PFP
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/East"
+    data_filename = "PFP EAST - Master - DN LN NG NR PE POSTCODES.xlsx"
+    sheet_name = "PFP EAST"
+    postcode_column = 'Postcode'
+    fulladdress_column = None
+    address1_column = "AddressLine1"
+    address1_method = None
+    address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"]
+    missing_postcodes_method = None
+    landlord_year_built = None
+    landlord_os_uprn = None
+    landlord_property_type = "Archetype"
+    landlord_built_form = "Archetype"
+    landlord_wall_construction = None
+    landlord_heating_system = None
+    landlord_existing_pv = None
+    landlord_property_id = "Uprn"
+    outcomes_filename = None
+    outcomes_sheetname = None
+    outcomes_postcode = None
+    outcomes_houseno = None
+    master_filepaths = []
+    master_to_asset_list_filepath = None
+
     # Wates
     data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Wates - "
     data_filename = "ECO 4 Wates.xlsx"

backend/apis/GoogleSolarApi.py

@@ -185,7 +185,7 @@ class GoogleSolarApi:
             ):
                 self.exclude_likely_duplicate_surfaces()
 
-        # TODO: We need to constrain the roof area, based on the floor area to be more conservative
+        # We constrain the roof area, based on the floor area to be more conservative
         self.roof_area = self.insights_data["solarPotential"]["wholeRoofStats"]['areaMeters2']
         if self.roof_area > property_instance.roof_area * self.ROOF_AREA_TOLERANCE:
             self.roof_area = property_instance.roof_area

backend/app/plan/router.py

@@ -704,21 +704,70 @@ async def trigger_plan(body: PlanTriggerRequest):
             if not recommendations.get(p.id):
                 continue
 
-            input_measures = prepare_input_measures(recommendations[p.id], body.goal)
+            # we need to double unlist because we have a list of lists
+            property_measure_types = {rec["type"] for recs in recommendations[p.id] for rec in recs}
+
+            measures_to_optimise = recommendations[p.id]
+            property_required_measures = []
+            if body.required_measures:
+                property_required_measures = [
+                    m for m in measures_to_optimise if m[0]["type"] in body.required_measures
+                ]
+                measures_to_optimise = [
+                    m for m in measures_to_optimise if m[0]["type"] not in body.required_measures
+                ]
+
+            # If we have a wall insulation measure, we MUST include mechanical ventilation
+            # Additionally, if we have required measures, they should also be included. Therefore
+            # we can discount the number of points required to get to the target SAP band (or increase)
+            # in the case of ventilation
+            measures_needing_ventilation = [
+                "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"
+            ]
+            needs_ventilation = any(x in property_measure_types for x in measures_needing_ventilation)
+
+            input_measures = prepare_input_measures(
+                measures_to_optimise, body.goal, needs_ventilation, measures_needing_ventilation
+            )
 
             if not input_measures[0]:
                 # This means that we have no defaults
                 selected_recommendations = {}
             else:
 
+                fixed_gain = 0
+                if property_required_measures:
+                    # We get the SAP points for the required measures
+                    if body.goal != "Increasing EPC":
+                        raise NotImplementedError("Only EPC optimisation is currently supported")
+                    sap_by_type = [
+                        {"type": rec["type"], "sap_points": rec["sap_points"]} for recs in property_required_measures
+                        for rec in recs
+                    ]
+                    # We get a MAX sap points per type
+                    max_per_type = (
+                        pd.DataFrame(sap_by_type).groupby("type")["sap_points"].max().to_dict()
+                    )
+                    fixed_gain = sum(max_per_type.values())
+
+                    property_required_measure_types = {rec["type"] for rec in sap_by_type}
+
+                    # if the property needs ventilation, but the measure we optimise didn't include
+                    # venilation we add the points for ventilation as a fixed gain
+                    if needs_ventilation and any(
+                        r in property_required_measure_types for r in measures_needing_ventilation
+                    ):
+                        fixed_gain += next(
+                            (r[0]["sap_points"] for r in recommendations[p.id] if
+                             r[0]["type"] == "mechanical_ventilation"),
+                            0
+                        )
+
                 current_sap_points = int(p.data["current-energy-efficiency"])
-                ventilation_impact = next(
+
-                    (r[0]["sap_points"] for r in recommendations[p.id] if r[0]["type"] == "mechanical_ventilation"),
-                    0
-                )
                 sap_gain = CostOptimiser.calculate_sap_gain_with_slack(
                     epc_to_sap_lower_bound(body.goal_value) - current_sap_points
-                ) + abs(ventilation_impact)
+                ) - fixed_gain
 
                 if not body.optimise:
                     if body.goal != "Increasing EPC":
@@ -748,6 +797,31 @@ async def trigger_plan(body: PlanTriggerRequest):
 
                 selected_recommendations = {r["id"] for r in solution}
 
+            if property_required_measures:
+                # We select the cheapest of the required measures, into selected
+                for recs in property_required_measures:
+                    # We select the cheapest of the required measures
+                    cost_to_id = {
+                        rec["recommendation_id"]: rec["total"] for rec in recs
+                        if rec["recommendation_id"] not in selected_recommendations
+                    }
+                    # Take the recommendation id with the lowers cost
+
+                    selected_recommendations.add(min(cost_to_id, key=cost_to_id.get))
+                    # Update the solution with the selected recommendaitons
+                    solution = []
+                    for recs in recommendations[p.id]:
+                        for rec in recs:
+                            if rec["recommendation_id"] in selected_recommendations:
+                                solution.append(
+                                    {
+                                        "id": rec["recommendation_id"],
+                                        "cost": rec["total"],
+                                        "gain": rec["sap_points"],
+                                        "type": rec["type"]
+                                    }
+                                )
+
             # If wall insulation is selected, we also include mechanical ventilation as a best practice measure
             if any(x in [r["type"] for r in solution] for x in [
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"

backend/app/plan/schemas.py

@@ -75,6 +75,8 @@ class PlanTriggerRequest(BaseModel):
     valuation_file_path: Optional[str] = None
     exclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1)
     inclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1)
+    # This is a list of measures that we want to be included, if they are options
+    required_measures: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1)
 
     scenario_name: Optional[str] = ""
     multi_plan: Optional[bool] = False

etl/customers/mod/pilot/1. Create Sample.py

@@ -104,13 +104,15 @@ def app():
             }
         )
 
+    # also include the floor area
     asset_list = df[
-        ["uprn", "address1", "postcode", "NUMBER_OF_BEDROOMS", "BLDNG_STOREYS_QTY", ]
+        ["uprn", "address1", "postcode", "NUMBER_OF_BEDROOMS", "BLDNG_STOREYS_QTY", "BLDNG_MSRMNT_VAL"]
     ].rename(
         columns={
             "address1": "address",
             "NUMBER_OF_BEDROOMS": "n_bedrooms",
-            "BLDNG_STOREYS_QTY": "number_of_floors"
+            "BLDNG_STOREYS_QTY": "number_of_floors",
+            "BLDNG_MSRMNT_VAL": "floor_area"
         }
     )
 

etl/customers/mod/pilot/2. Create Excel Model.py

@@ -0,0 +1,398 @@
+from pprint import pprint
+import pandas as pd
+import numpy as np
+from backend.app.utils import sap_to_epc
+from sqlalchemy.orm import sessionmaker
+from backend.app.db.connection import db_engine
+from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations
+from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel
+
+
+def get_data(portfolio_id, scenario_ids):
+    session = sessionmaker(bind=db_engine)()
+    session.begin()
+
+    # Get properties and their details for a specific portfolio
+    properties_query = session.query(
+        PropertyModel,
+        PropertyDetailsEpcModel
+    ).join(
+        PropertyDetailsEpcModel, PropertyModel.id == PropertyDetailsEpcModel.property_id
+    ).filter(
+        PropertyModel.portfolio_id == portfolio_id  # Filter by portfolio ID
+    ).all()
+
+    # Transform properties data to include all fields dynamically
+    properties_data = [
+        {**{col.name: getattr(prop.PropertyModel, col.name) for col in PropertyModel.__table__.columns},
+         **{col.name: getattr(prop.PropertyDetailsEpcModel, col.name) for col in
+            PropertyDetailsEpcModel.__table__.columns}}
+        for prop in properties_query
+    ]
+
+    # Get property IDs from fetched properties
+
+    # Get plans linked to the fetched properties
+    plans_query = session.query(Plan).filter(Plan.scenario_id.in_(scenario_ids)).all()
+
+    # Transform plans data to include all fields dynamically
+    plans_data = [
+        {col.name: getattr(plan, col.name) for col in Plan.__table__.columns}
+        for plan in plans_query
+    ]
+
+    # Extract plan IDs for filtering recommendations through PlanRecommendations
+    plan_ids = [plan['id'] for plan in plans_data]
+
+    # Get recommendations through PlanRecommendations for those plans and that are default
+    recommendations_query = session.query(
+        Recommendation,
+        Plan.scenario_id
+    ).join(
+        PlanRecommendations, Recommendation.id == PlanRecommendations.recommendation_id
+    ).join(
+        Plan, Plan.id == PlanRecommendations.plan_id  # Join with Plan to access scenario_id
+    ).filter(
+        PlanRecommendations.plan_id.in_(plan_ids),
+        Recommendation.default == True  # Filtering for default recommendations
+    ).all()
+
+    # Transform recommendations data to include all fields dynamically and include scenario_id
+    recommendations_data = [
+        {**{col.name: getattr(rec.Recommendation, col.name) if hasattr(rec, 'Recommendation')
+        else getattr(rec, col.name) for
+            col in Recommendation.__table__.columns},
+         "Scenario ID": rec.scenario_id}
+        for rec in recommendations_query
+    ]
+
+    session.close()
+
+    return properties_data, plans_data, recommendations_data
+
+
+def app():
+    """
+    Given a portfolio and a scenario, this function prepares an excel model to present the data
+    """
+
+    # Set the inputs:
+    portfolio_id = 139
+    scenario_ids = [233, 234]
+
+    properties_data, plans_data, recommendations_data = get_data(
+        portfolio_id=portfolio_id, scenario_ids=scenario_ids
+    )
+
+    properties_df = pd.DataFrame(properties_data)
+    plans_df = pd.DataFrame(plans_data)
+    recommendations_df = pd.DataFrame(recommendations_data)
+
+    # Merge on the orignal data
+    mod_property_data = pd.read_csv(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/MOD/Pilot Programme/MOD property data.csv"
+    )
+
+    property_asset_data = properties_df.merge(
+        mod_property_data.drop(columns=["address", "postcode", "tenure"]), how="left", on="uprn"
+    )
+
+    property_asset_data["is_pitched"] = property_asset_data["roof"].str.contains("pitched", case=False)
+    property_asset_data["pre_2002"] = property_asset_data["BUILD_YEAR"] < 2002
+    property_asset_data["wall_type"] = property_asset_data["walls"].str.split(" ").str[0].str.strip()
+    property_asset_data["is_insulated"] = (
+        property_asset_data["walls"].str.split(",").str[1].str.strip().isin(
+            ["filled cavity", "with external insulation", "filled cavity and external insulation"]
+        ) | property_asset_data["walls"].str.split(",").str[2].str.strip().isin(["insulated"])
+    )
+    property_asset_data["is_insulated"] = np.where(
+        property_asset_data["is_insulated"], "Insulated", "Uninsulated"
+    )
+    property_asset_data["is_pitched"] = np.where(
+        property_asset_data["is_pitched"], "Pitched roof", "Not Pitched Roof"
+    )
+    property_asset_data["pre_2002"] = np.where(
+        property_asset_data["pre_2002"], "Pre 2002", "Post 2002"
+    )
+
+    archetype_variables = ["property_type", "wall_type", "is_insulated", "is_pitched", "pre_2002"]
+
+    assigned_archetypes = (
+        property_asset_data.groupby(
+            archetype_variables
+        ).size().reset_index().rename(columns={0: "n_properties"}).sort_values("n_properties", ascending=False)
+    )
+
+    # Make the archetype ID a concatenation of the variables
+    assigned_archetypes["archetype_id"] = assigned_archetypes[archetype_variables].apply(
+        lambda x: "_".join(x.astype(str)), axis=1
+    )
+
+    # Most prominent archetypes
+    prominent_archetypes = assigned_archetypes.head(3)
+    other_archetypes = assigned_archetypes.tail(-3)
+    # 2 or fewer properties in the other archetypes
+
+    property_asset_data = property_asset_data.merge(
+        assigned_archetypes[archetype_variables + ["archetype_id"]],
+        how="left",
+        on=archetype_variables
+    )
+
+    # Create age bands:
+    # 1960-1969
+    # 1970-1979
+    # 1980-1989
+    # 1990-1999
+    # 2000+
+    property_asset_data["age_band"] = pd.cut(
+        property_asset_data["BUILD_YEAR"],
+        bins=[1959, 1969, 1979, 1989, 1999, 2022],
+        labels=["1960-1969", "1970-1979", "1980-1989", "1990-1999", "2000+"]
+    )
+
+    # Create floor area bands
+    # 0-73
+    # 74-97
+    # 98-199
+    # 200+
+    property_asset_data["floor_area_band"] = pd.cut(
+        property_asset_data["total_floor_area"],
+        bins=[0, 73, 97, 199, 10000],
+        labels=["0-73", "74-97", "98-199", "200+"]
+    )
+
+    property_asset_data["archetype_group"] = property_asset_data["archetype_id"].copy()
+    property_asset_data["archetype_group"] = np.where(
+        property_asset_data["archetype_id"].isin(other_archetypes["archetype_id"].values),
+        "other",
+        property_asset_data["archetype_group"]
+    )
+
+    # For colour
+    wall_types = (
+        property_asset_data[["wall_type"]].value_counts().to_frame().reset_index().rename(
+            columns={"wall_type": "Wall Type"}
+        )
+    )
+    # Group into age bands
+    ages = (
+        property_asset_data[["age_band"]].value_counts()
+        .to_frame()
+        .reset_index().sort_values("age_band", ascending=True)
+        .rename(columns={"age_band": "Age Band"})
+    )
+    floor_area_bands = (
+        property_asset_data[["floor_area_band"]].value_counts()
+        .to_frame()
+        .reset_index().sort_values("floor_area_band", ascending=True)
+        .rename(columns={"floor_area_band": "Floor Area Band"})
+    )
+    archetype_counts = (
+        property_asset_data[["archetype_group"]].
+        value_counts().
+        to_frame().
+        reset_index()
+        .rename(columns={"archetype_group": "Archetype"})
+    )
+
+    # epc breakdown
+    epc_breakdown = (
+        property_asset_data["current_epc_rating"]
+        .apply(lambda x: x.value)
+        .value_counts()
+        .to_frame()
+        .reset_index()
+    )
+
+    # Figures for the deck
+    # Carbon per property
+    totals = property_asset_data[
+        [
+            "Total_household_members",
+            "co2_emissions", "current_energy_demand", "current_energy_demand_heating_hotwater",
+            "heating_cost_current", "hot_water_cost_current", "lighting_cost_current",
+            "appliances_cost_current", "gas_standing_charge", "electricity_standing_charge"
+        ]
+    ].copy()
+    totals["total_cost"] = (
+        totals["heating_cost_current"] +
+        totals["hot_water_cost_current"] +
+        totals["lighting_cost_current"] +
+        totals["appliances_cost_current"] +
+        totals["gas_standing_charge"] +
+        totals["electricity_standing_charge"]
+    )
+    print(
+        totals[
+            [
+                "Total_household_members",
+                "co2_emissions",
+                "current_energy_demand",
+                "total_cost",
+            ]
+        ].mean()
+    )
+
+    # Store these to an excel
+    # with pd.ExcelWriter(
+    #     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/MOD/Pilot Programme/MOD archetype breakdowns.xlsx"
+    # ) as writer:
+    #     wall_types.to_excel(writer, sheet_name="Wall Types", index=False)
+    #     ages.to_excel(writer, sheet_name="Ages", index=False)
+    #     floor_area_bands.to_excel(writer, sheet_name="Floor Area Bands", index=False)
+    #     archetype_counts.to_excel(writer, sheet_name="Archetype Counts", index=False)
+    #     epc_breakdown.to_excel(writer, sheet_name="EPC Rating", index=False)
+
+    contingency = 0.26
+
+    # We prepare the outputs, by scenario
+    scenario_data = {}
+    for scenario in scenario_ids:
+
+        scenario_recommendations_df = recommendations_df[
+            recommendations_df["Scenario ID"] == scenario
+            ].copy()
+
+        scenario_recommendations_df["contingency"] = contingency * scenario_recommendations_df["estimated_cost"]
+        scenario_recommendations_df["total_cost"] = (
+            scenario_recommendations_df["estimated_cost"] + scenario_recommendations_df["contingency"]
+        )
+
+        recommended_measures_df = scenario_recommendations_df[
+            ["property_id", "measure_type", "estimated_cost", "default"]
+        ]
+
+        recommended_measures_df = recommended_measures_df[recommended_measures_df["default"]]
+        recommended_measures_df = recommended_measures_df.drop(columns=["default"])
+
+        # Metrics by property ID
+        aggregated_metrics = scenario_recommendations_df[
+            [
+                "property_id", "type", "default", "sap_points",
+                "energy_cost_savings", "kwh_savings", "co2_equivalent_savings", "estimated_cost", "contingency",
+                "total_cost"
+            ]
+        ]
+        aggregated_metrics = aggregated_metrics[aggregated_metrics["default"]]
+        aggregated_metrics = aggregated_metrics.groupby("property_id")[
+            ["sap_points", "co2_equivalent_savings", "energy_cost_savings", "kwh_savings", "estimated_cost",
+             "total_cost", "contingency"]
+        ].sum().reset_index()
+
+        recommendations_measures_pivot = recommended_measures_df.pivot(
+            index='property_id',
+            columns='measure_type',
+            values='estimated_cost'
+        )
+        recommendations_measures_pivot = recommendations_measures_pivot.reset_index()
+        recommendations_measures_pivot = recommendations_measures_pivot.fillna(0)
+
+        # We flag with boolean if the measure is recommended
+        for c in recommendations_measures_pivot.columns:
+            if c == "property_id":
+                continue
+            recommendations_measures_pivot["Recommendation: " + c] = recommendations_measures_pivot[c] > 0
+
+        # We now create a final output
+        df = properties_df[
+            [
+                "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof", "heating", "windows",
+                "current_epc_rating", "current_sap_points", "total_floor_area", "number_of_rooms",
+            ]
+        ].merge(
+            recommendations_measures_pivot, how="left", on="property_id"
+        ).merge(
+            aggregated_metrics, how="left", on="property_id"
+        )
+
+        df = df.drop(columns=["property_id"])
+        for c in ["sap_points", "co2_equivalent_savings", "energy_cost_savings", "kwh_savings"]:
+            df[c] = df[c].fillna(0)
+
+        df = df.rename(
+            columns={
+                "uprn": "UPRN",
+                "address": "Address",
+                "postcode": "Postcode",
+                "walls": "Walls",
+                "roof": "Roof",
+                "heating": "Heating",
+                "windows": "Windows",
+                "current_epc_rating": "Current EPC Rating",
+                "current_sap_points": "Current SAP Points",
+                "total_floor_area": "Total Floor Area",
+                "number_of_rooms": "Number of Habitable Rooms",
+                "floor_height": "Floor Height",
+            }
+        )
+
+        # Calculate post SAP
+        df["Predicted Post Works SAP"] = df["Current SAP Points"] + df["sap_points"]
+        df["Predicted Post Works SAP"] = df["Predicted Post Works SAP"].round()
+        df["Predicted Post Works EPC"] = df["Predicted Post Works SAP"].apply(lambda x: sap_to_epc(x))
+
+        # For properties that don't make it to EPC B, check why. E.g. for a property that has an oil boiler, it
+        # the bills go up recommending HHRSH, so it doesn't make it to EPC B
+        # For mid-terrace units, use the ordnance survey API to check if there is space for a heat pump?
+        # DO it manually???
+
+        # Doesn't make it
+        # misses = df[df["Predicted Post Works EPC"] == "C"]
+        # # 5 of them are flats and so are difficult to get to EPC B without renewables. Possibly not worth it from an
+        # # ROI perspective
+        #
+        # misses[["UPRN", "Address", "Postcode", "property_type"]]
+
+        #              UPRN                               Address  Postcode property_type
+        # 2    100120988937              13 Sidbury Circular Road   SP9 7HX          Flat  No further action
+        # 3    100120988998              74 Sidbury Circular Road   SP9 7JA          Flat  No further action
+        # 4    100120989416                       47 Zouch Avenue   SP9 7LR          Flat  No further action
+        # 6    100060585002  42, Muscott Close, Shipton Bellinger   SP9 7TX         House  Can probably take a heat pump
+        # 37    10000801072        34 Luffenham Place, Chicksands  SG17 5XH         House  Already surveyed as having
+        # an ASHP - should be looked at
+        # 121  100120988259                      8, Karachi Close   SP9 7LW          Flat
+        # 122  100121101217                     599, Pepper Place  BA12 0DW          Flat
+        # 140  100021455241         33 Blenheim Crescent, Ruislip   HA4 7HA         House - Solar isnt recommended
+        # due to bug
+        # 149  100120915656            10 Bower Green, Shrivenham   SN6 8TU         House - Solar isn't recommended
+        # due to bug
+
+        scenario_data[scenario] = df
+
+    measure_counts = {}
+    for scenario in scenario_ids:
+        recommendation_cols = [c for c in scenario_data[scenario].columns if "Recommendation:" in c]
+        measure_counts[scenario] = scenario_data[scenario][recommendation_cols].sum().to_dict()
+
+    pprint(measure_counts[scenario_ids[0]])
+    pprint(measure_counts[scenario_ids[1]])
+
+    df = scenario_data[scenario_ids[1]]
+    z = df[
+        (df["Walls"] == "Cavity wall, as built, no insulation") & (~df["Recommendation: cavity_wall_insulation"])
+        ]
+
+    # Scenario adjustments:
+    # Exclude: boiler_upgrade
+    # Make ASHP COP 3.5
+
+    # Metrics we need by scenario:
+    # Cost
+    # contingency
+    # Carbon
+    # kwh
+    # bill savings
+    scenario_metrics = {}
+    for scenario in scenario_ids:
+        df = scenario_data[scenario].copy()
+        df["cost_per_sap_point"] = df["total_cost"] / df["sap_points"]
+        df["cost_per_carbon"] = df["total_cost"] / df["co2_equivalent_savings"]
+        avg_savings = df[
+            ["sap_points", "co2_equivalent_savings", "energy_cost_savings", "kwh_savings", "estimated_cost",
+             "cost_per_sap_point", "cost_per_carbon", "total_cost", "contingency"]
+        ].mean().to_dict()
+
+    # TODO: Add a slide on valuation improvement, on a sample of properties?
+
+    # TODO: Read in costing data and breakdown

etl/customers/united living/get_data.py

@@ -0,0 +1,73 @@
+import os
+import pandas as pd
+import numpy as np
+from asset_list.utils import get_data
+from backend.SearchEpc import SearchEpc
+from etl.spatial.OpenUprnClient import OpenUprnClient
+
+from dotenv import load_dotenv
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+    filepath = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/United Living/Potential GMCA props 05.03.xlsx"
+
+    df = pd.read_excel(filepath)
+    df["row_id"] = df.index
+
+    df["house_number"] = df.apply(
+        lambda x: SearchEpc.get_house_number(x["Address"], x["Postcode"]),
+        axis=1
+    )
+
+    properties_data, _, _ = get_data(
+        df=df,
+        manual_uprn_map={},
+        epc_auth_token=EPC_AUTH_TOKEN,
+        uprn_column=None,
+        fulladdress_column="Address",
+        address1_column="house_number",
+        postcode_column="Postcode",
+        property_type_column=None,
+        built_form_column=None,
+        epc_api_only=True,
+        row_id_name="row_id",
+    )
+
+    no_data = df[df["row_id"].isin(_)]
+    no_data[["Address", "Postcode"]]
+
+    # 53  108 Alexandra Street  OL6 9QP 100011536830
+    # 56    301 Whiteacre Road  OL6 9QF 100011557437
+    # 65    97 Princess Street  OL6 9QJ 100011551813
+
+    data = df.merge(
+        pd.DataFrame(properties_data)[["uprn", "row_id"]],
+        how="left", left_on="row_id", right_on="row_id"
+    )
+
+    # Fill missing UPRNS
+    data["uprn"] = np.where(data["Address"] == "108 Alexandra Street", 100011536830, data["uprn"])
+    data["uprn"] = np.where(data["Address"] == "301 Whiteacre Road", 100011557437, data["uprn"])
+    data["uprn"] = np.where(data["Address"] == "97 Princess Street", 100011551813, data["uprn"])
+
+    # We now get whether the property is listed, heritage or in a conservation area
+    spatial_data = OpenUprnClient.get_spatial_data(uprns=data["uprn"].tolist(), bucket_name="retrofit-data-dev")
+    spatial_data = spatial_data.rename(columns={"UPRN": "uprn"})
+
+    data["uprn"] = data["uprn"].astype(int)
+
+    merged = data.merge(
+        spatial_data, how="left", on="uprn"
+    )
+    # fill NAs
+    for c in ['conservation_status', 'is_listed_building', 'is_heritage_building']:
+        merged[c] = merged[c].fillna(False)
+
+    merged.to_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/United Living/Potential GMCA props 05.03 - data "
+        "pulled.xlsx",
+        index=False
+    )

recommendations/Costs.py

@@ -37,22 +37,25 @@ MCS_SOLAR_PV_COST_DATA = {
     "average_cost_per_kwh-Northern Ireland": 1347,
 }
 
+# Installers are now working with 435 watt panels
+PANEL_SIZE = 0.435
+
 INSTALLER_SOLAR_COSTS = [
-    {'n_panels': 4, 'array_kwp': 1.6, 'cost': 3040.00, 'installer': 'CEG'},
+    {'n_panels': 4, 'array_kwp': 4 * PANEL_SIZE, 'cost': 4089.25, 'installer': 'CEG'},
-    {'n_panels': 5, 'array_kwp': 2.1, 'cost': 3201.00, 'installer': 'CEG'},
+    {'n_panels': 5, 'array_kwp': 5 * PANEL_SIZE, 'cost': 4242.48, 'installer': 'CEG'},
-    {'n_panels': 6, 'array_kwp': 2.5, 'cost': 3363.00, 'installer': 'CEG'},
+    {'n_panels': 6, 'array_kwp': 6 * PANEL_SIZE, 'cost': 4395.71, 'installer': 'CEG'},
-    {'n_panels': 7, 'array_kwp': 2.9, 'cost': 3524.00, 'installer': 'CEG'},
+    {'n_panels': 7, 'array_kwp': 7 * PANEL_SIZE, 'cost': 4548.94, 'installer': 'CEG'},
-    {'n_panels': 8, 'array_kwp': 3.3, 'cost': 3686.00, 'installer': 'CEG'},
+    {'n_panels': 8, 'array_kwp': 8 * PANEL_SIZE, 'cost': 4702.17, 'installer': 'CEG'},
-    {'n_panels': 9, 'array_kwp': 3.7, 'cost': 3847.00, 'installer': 'CEG'},
+    {'n_panels': 9, 'array_kwp': 9 * PANEL_SIZE, 'cost': 4855.41, 'installer': 'CEG'},
-    {'n_panels': 10, 'array_kwp': 4.1, 'cost': 4009.00, 'installer': 'CEG'},
+    {'n_panels': 10, 'array_kwp': 10 * PANEL_SIZE, 'cost': 5010.95, 'installer': 'CEG'},
-    {'n_panels': 11, 'array_kwp': 4.5, 'cost': 4170.00, 'installer': 'CEG'},
+    {'n_panels': 11, 'array_kwp': 11 * PANEL_SIZE, 'cost': 5166.49, 'installer': 'CEG'},
-    {'n_panels': 12, 'array_kwp': 4.9, 'cost': 4332.00, 'installer': 'CEG'},
+    {'n_panels': 12, 'array_kwp': 12 * PANEL_SIZE, 'cost': 5322.04, 'installer': 'CEG'},
-    {'n_panels': 13, 'array_kwp': 5.3, 'cost': 4835.00, 'installer': 'CEG'},
+    {'n_panels': 13, 'array_kwp': 13 * PANEL_SIZE, 'cost': 5657.6, 'installer': 'CEG'},
-    {'n_panels': 14, 'array_kwp': 5.7, 'cost': 5015.00, 'installer': 'CEG'},
+    {'n_panels': 14, 'array_kwp': 14 * PANEL_SIZE, 'cost': 5993.16, 'installer': 'CEG'},
-    {'n_panels': 15, 'array_kwp': 6.2, 'cost': 5176.00, 'installer': 'CEG'},
+    {'n_panels': 15, 'array_kwp': 15 * PANEL_SIZE, 'cost': 6328.71, 'installer': 'CEG'},
-    {'n_panels': 16, 'array_kwp': 6.6, 'cost': 5338.00, 'installer': 'CEG'},
+    {'n_panels': 16, 'array_kwp': 16 * PANEL_SIZE, 'cost': 6483.33, 'installer': 'CEG'},
-    {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'},
+    {'n_panels': 17, 'array_kwp': 17 * PANEL_SIZE, 'cost': 6637.95, 'installer': 'CEG'},
-    {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'}
+    {'n_panels': 18, 'array_kwp': 18 * PANEL_SIZE, 'cost': 6792.57, 'installer': 'CEG'}
 ]
 # This is the maximum number of panels that we have a cost from the installers for
 INSTALLER_MAX_PANELS = 18
@@ -62,11 +65,11 @@ INSTALLER_MAX_PANELS = 18
 INSTALLER_SOLAR_PV_INVERTER_COST = 7500
 INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST = 500  # Just a rough guess to labour costs
 
-INSTALLER_SCAFFOLDING_COSTS = [
+# INSTALLER_SCAFFOLDING_COSTS = [
-    {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'},
+#     {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'},
-    {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'},
+#     {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'},
-    {'stories': 3, 'description': '3 Story Scaffold', 'cost': 1077.00, 'installer': 'CEG'}
+#     {'stories': 3, 'description': '3 Story Scaffold', 'cost': 1077.00, 'installer': 'CEG'}
-]
+# ]
 
 # This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average,
 # to be conservative
@@ -772,18 +775,14 @@ class Costs:
             battery_cost = [c for c in INSTALLER_SOLAR_BATTERY_COSTS if c["capacity_kwh"] == battery_kwh][0]["cost"]
             subtotal += battery_cost
 
-        scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"]
-        subtotal += scaffolding_cost
-
         if needs_inverter:
             subtotal += INSTALLER_SOLAR_PV_INVERTER_COST
             # We also add an additional labour cost
             subtotal += INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST
 
-        # We add an additional cost for scaffolding
+        # Solar doesn't have VAT but we add a high risk contingency
-        # The costs from installers exclude VAT
+        # to account for design variation that we see in practice
-        vat = subtotal * cls.VAT_RATE
+        total_cost = subtotal * (1 + cls.HIGH_RISK_CONTINGENCY)
-        total_cost = subtotal + vat
 
         # Labour hours are based on estimates from online research but an average team seems to consist of 3 people
         # and most jobs take around 2 days. Assuming an 8 hour day for 3 people across 2 days, gives us 48 hours of
@@ -791,7 +790,7 @@ class Costs:
         return {
             "total": total_cost,
             "subtotal": subtotal,
-            "vat": vat,
+            "vat": 0,
             "labour_hours": 48,
             "labour_days": 2,
         }
@@ -1161,7 +1160,6 @@ class Costs:
         pump. This cost will include the boiler upgrade scheme grant
 
         """
-
         # This is the average cost of a project, we'll add some additional contingency
 
         if ashp_size is None:
@@ -1170,7 +1168,7 @@ class Costs:
             cost = [x for x in INSTALLER_ASHP_COSTS if x][0]["cost"]
 
         # We add some contingency since there are additional costs such as resizing radiators, that could be required
-        subtotal = cost * (1 + self.CONTINGENCY)
+        subtotal = cost * (1 + self.HIGH_RISK_CONTINGENCY)
         # The costs from installers exclude VAT
         vat = subtotal * self.VAT_RATE
         total_cost = subtotal + vat
@@ -1180,7 +1178,7 @@ class Costs:
         labour_hours = labour_days * 8
 
         return {
-            "total": subtotal,
+            "total": total_cost,
             "subtotal": subtotal,
             "vat": vat,
             "labour_hours": labour_hours,

recommendations/FloorRecommendations.py

@@ -145,7 +145,9 @@ class FloorRecommendations(Definitions):
             )
             return
 
-        raise NotImplementedError("Implement me!")
+        # In this case, we have no recommendation to make. E.g., if we have a solid floor property
+        # but solid floor insulation has been excluded as a measure, we get here
+        return
 
     @staticmethod
     def _make_floor_description(material):

recommendations/optimiser/optimiser_functions.py

@@ -1,10 +1,12 @@
-def prepare_input_measures(property_recommendations, goal):
+def prepare_input_measures(property_recommendations, goal, needs_ventilation, measures_needing_ventilation):
     """
     Basic function to convert recommendations_to_upload to a format that is
     suitable for the optimiser - large
     :param property_recommendations:   object containing the recommendations, created in the plan trigger api
     :param goal:    goal to be optimised for, should be one of the keys in gain_map. E.g. if the gain is SAP points,
                     the goal should reflect that desired gain
+    :param needs_ventilation:  boolean to indicate if the property needs ventilation
+    :param measures_needing_ventilation:    list of measures that need ventilation
     :return:    Nested list of input measures
     """
 
@@ -16,9 +18,20 @@ def prepare_input_measures(property_recommendations, goal):
     if not goal_key:
         raise NotImplementedError("Not implemented this gain type - investigate me")
 
+    # We ony ever have one ventilation measure with now
+    ventilation_recommendation = next(
+        (measure[0] for measure in property_recommendations if measure[0]["type"] == "mechanical_ventilation"),
+        {}
+    )
+
     input_measures = []
     for recs in property_recommendations:
 
+        if needs_ventilation and recs[0]["type"] == "mechanical_ventilation":
+            # If we house needs ventilation, ventilation will be packaged with the fabric measure so
+            # we don't need to optimise it independently
+            continue
+
         if recs[0]["type"] == "solar_pv":
             # if the recommendation is a solar recommendation with a battery, we exclude it from the optimisation.
             recs = [r for r in recs if ~r["has_battery"]]
@@ -27,16 +40,34 @@ def prepare_input_measures(property_recommendations, goal):
         if not recs_to_append:
             continue
 
-        input_measures.append(
+        to_append = []
-            [
+        for rec in recs:
+            # We bundle the impact of ventilation with the measure
+            total = (
+                rec["total"] + ventilation_recommendation["total"] if rec["type"] in measures_needing_ventilation
+                else rec["total"]
+            )
+            gain = (
+                rec[goal_key] + ventilation_recommendation[goal_key] if rec["type"] in measures_needing_ventilation
+                else rec[goal_key]
+            )
+
+            rec_type = (
+                "+".join(
+                    [rec["type"], ventilation_recommendation["type"]]
+                ) if rec["type"] in measures_needing_ventilation
+                else rec["type"]
+            )
+
+            to_append.append(
                 {
                     "id": rec["recommendation_id"],
-                    "cost": rec["total"],
+                    "cost": total,
-                    "gain": rec[goal_key],
+                    "gain": gain,
-                    "type": rec["type"]
+                    "type": rec_type
                 }
-                for rec in recs if rec["energy_cost_savings"] >= 0
+            )
-            ]
+
-        )
+        input_measures.append(to_append)
 
     return input_measures