Merge pull request #448 from Hestia-Homes/debugging-api

Allow engine to utilising an existing scenario
2026-06-08 11:17:27 +00:00 · 2025-07-15 18:50:09 +01:00 · 2025-07-15 18:50:09 +01:00 · c6217ed1fb
commit c6217ed1fb
parent f414fbf01d 0bc1299e69
11 changed files with 184 additions and 36 deletions
--- a/asset_list/AssetList.py
+++ b/asset_list/AssetList.py
@ -3155,10 +3155,19 @@ class AssetList:
            date_col = "Date letters sent"
        elif "Date Letter sent" in self.outcomes.columns:
            date_col = "Date Letter sent"
+        elif "WEEK COMMENCING" in self.outcomes.columns:
+            date_col = "WEEK COMMENCING"
        else:
            raise NotImplementedError("Invalid date in outcomes - implement me")

-        notes_col = "Notes" if "Notes" in self.outcomes.columns else "Notes / Outcomes"
+        if "Notes" in self.outcomes.columns:
+            notes_col = "Notes"
+        elif "Notes / Outcomes" in self.outcomes.columns:
+            notes_col = "Notes / Outcomes"
+        elif "NOTES" in self.outcomes.columns:
+            notes_col = "NOTES"
+        else:
+            raise NotImplementedError("Invalid notes in outcomes - implement me")

        lookup = lookup.merge(
            self.outcomes[["row_id", "Outcome", notes_col, date_col]], how="left", on="row_id"
@ -3342,6 +3351,10 @@ class AssetList:
                installer_notes_col = 'Installers Notes'
            elif 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM' in master_data.columns:
                installer_notes_col = 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM'
+            elif ('INSTALLERS NOTES / REASONS FOR CANCELLATIONS / WHERE INSTALL DATE WAS RECEIVED FROM' in
+                  master_data.columns):
+                installer_notes_col = ('INSTALLERS NOTES / REASONS FOR CANCELLATIONS / WHERE INSTALL DATE WAS RECEIVED '
+                                       'FROM')
            else:
                raise ValueError("No installer notes column found in master data")

--- a/asset_list/app.py
+++ b/asset_list/app.py
@ -59,6 +59,75 @@ def app():
    Property UPRN
    """

+    # TODO: Delete me
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild"
+    data_filename = "Bromford Asset List.xlsx"
+    sheet_name = "Asset List"
+    postcode_column = 'PostCode'
+    fulladdress_column = "FullAddress"
+    address1_column = None
+    address1_method = "house_number_extraction"
+    address_cols_to_concat = []
+    missing_postcodes_method = None
+    landlord_year_built = None
+    landlord_os_uprn = None
+    landlord_property_type = None
+    landlord_built_form = None
+    landlord_wall_construction = None
+    landlord_heating_system = None
+    landlord_existing_pv = None
+    landlord_property_id = "Asset"
+    outcomes_filename = []
+    outcomes_sheetname = []
+    outcomes_postcode = []
+    outcomes_houseno = []
+    outcomes_address = []
+    outcomes_id = [None]
+    master_filepaths = [os.path.join("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/",
+                                     "Needs ID/SOLAR PV ONLY-Table 1.csv")]
+    master_to_asset_list_filepath = None
+    asset_list_header = 0
+    landlord_block_reference = None
+    master_id_colnames = [None]
+    landlord_roof_construction = None
+    phase = False
+    landlord_sap = None
+    ecosurv_landlords = None
+
+    # For Housing
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
+    data_filename = "FOR HOUSING Asset List (Combined).xlsx"
+    sheet_name = "Asset List"
+    postcode_column = 'Postcode'
+    fulladdress_column = "Address"
+    address1_column = None
+    address1_method = "house_number_extraction"
+    address_cols_to_concat = []
+    missing_postcodes_method = None
+    landlord_year_built = None
+    landlord_os_uprn = None
+    landlord_property_type = "Type"
+    landlord_built_form = "Type"
+    landlord_wall_construction = None
+    landlord_heating_system = "Heating - full"
+    landlord_existing_pv = None
+    landlord_property_id = "UPRN"
+    outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
+    outcomes_sheetname = ["Sheet1"]
+    outcomes_postcode = ["POSTCODE"]
+    outcomes_houseno = ["NO"]
+    outcomes_address = ["ADDRESS"]
+    outcomes_id = [None]
+    master_filepaths = [os.path.join(data_folder, "submissions.csv")]
+    master_to_asset_list_filepath = None
+    asset_list_header = 0
+    landlord_block_reference = None
+    master_id_colnames = [None]
+    landlord_roof_construction = None
+    phase = False
+    landlord_sap = "SAP"
+    ecosurv_landlords = "for housing"
+
    # CDS
    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
    data_filename = "Founder Estates - Asset List.xlsx"
--- a/asset_list/hubspot/config.py
+++ b/asset_list/hubspot/config.py
@ -15,7 +15,7 @@ class HubspotProcessStatus(IntEnum):
    # This is the first stage, where a survey is ready to go
    READY_TO_BE_SCHEDULED = 1, "READY TO BE SCHEDULED"
    # The property didn't get access and needs sign off
-    SURVEYED_NO_ACCESS_NEEDS_SIGN_OFF = 2, "SURVEYED - NO ACCESS - NEED SIGN OFF"
+    SURVEYED_NO_ACCESS_NEEDS_SIGN_OFF = 2, "NO ACCESS - NEED SIGN OFF"
    # The survey has been completed. We don't have any update as to whether the property has been installed
    SURVEYED_COMPLETED_SIGNED_OFF = 3, "SURVEYED - AUTOMATED SIGNED OFF"
    # The property turned out to be ineligibile
@ -34,6 +34,7 @@ class Installer(Enum):
    SCIS = "SCIS"
    JJ_CRUMP = "J & J CRUMP"
    SGEC = "SGEC"
+    WARMFRONT = "WARM FRONT"

    @classmethod
    def is_valid_value(cls, value):
--- a/asset_list/hubspot/prepare_for_hubspot.py
+++ b/asset_list/hubspot/prepare_for_hubspot.py
@ -45,13 +45,13 @@ def app():

    # inputs:
    reconcile_programme = True  # If True, the hubspot upload will include all properties with a project code
-    customer_domain = "https://ealing.gov.uk"
-    installer_name = "SCIS"
+    customer_domain = "https://calico.org.uk"
+    installer_name = "WARM FRONT"
    asset_list_filepath = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Hubspot/20250707 Ealing Flats - Prepared "
-        "programme.xlsx"
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Calico/Hubspot/07.04 CALICO - Final List - "
+        "Standardised.xlsx"
    )
-    asset_list_sheet_name = "Standardised Asset List"
+    asset_list_sheet_name = "Final Route March"
    asset_list_header = 0

    contact_details_filepath = None
--- a/asset_list/mappings/built_form.py
+++ b/asset_list/mappings/built_form.py
@ -372,5 +372,18 @@ BUILT_FORM_MAPPINGS = {
    'MAISONETTE': 'unknown',
    'HOUSE': 'unknown',
    'FLAT': 'unknown',
-    'BLOCK': 'unknown'
+    'BLOCK': 'unknown',
+
+    'Semi Detached Bungalow': 'semi-detached',
+    'End Terraced Bungalow': 'end-terrace',
+    'Mid Terraced Town House': 'mid-terrace',
+    'Semi-Detached House': 'detached',
+    'Low Rise Flat': 'low rise',
+    'Mid Terraced Bungalow': 'mid-terrace',
+    'End Terraced Town House': 'end-terrace',
+    'Cottage Flat': 'ground floor',
+    'Maisonette Over Shop': 'mid-floor',
+    'Medium Rise Flat': 'mid-floor',
+    'Maisonette Medium Rise': 'unknown'
+
 }
--- a/asset_list/mappings/heating_systems.py
+++ b/asset_list/mappings/heating_systems.py
@ -364,5 +364,19 @@ HEATING_MAPPINGS = {
    'Boiler, Electricity': 'electric boiler',
    'Boiler, LPG': 'gas boiler, radiators',
    'Boiler, Mains gas': 'gas boiler, radiators',
-    'Storage heating, Electricity': 'electric storage heaters'
+    'Storage heating, Electricity': 'electric storage heaters',
+
+    'No Heating Types None': 'no heating',
+    'Boiler Smokeless': 'boiler - other fuel',
+    'Boiler House coal': 'boiler - other fuel',
+    'Warm air Mains gas': 'warm air heating',
+    'Storage heaters None': 'electric storage heaters',
+    'Boiler Anthracite': 'boiler - other fuel',
+    'Mains gas': 'gas boiler, radiators',
+    'Community heating Mains Gas': 'communal gas boiler',
+    'Warm air Electricity': 'warm air heating',
+    'None': 'no heating',
+    'Boiler None': 'unknown',
+    'Storage heaters Electricity': 'electric storage heaters'
+
 }
--- a/asset_list/mappings/property_type.py
+++ b/asset_list/mappings/property_type.py
@ -270,6 +270,19 @@ PROPERTY_MAPPING = {
    'HFOP FLAT': 'flat',
    'HFOP BEDSIT': 'bedsit',
    'LINKED FLAT': 'flat',
-    'LINKED BUNGALOW': 'bungalow'
+    'LINKED BUNGALOW': 'bungalow',
+
+    'Semi Detached Bungalow': 'bungalow',
+    'End Terraced Bungalow': 'bungalow',
+    'Mid Terraced Town House': 'house',
+    'Cottage Flat': 'flat',
+    'Semi-Detached House': 'house',
+    'Low Rise Flat': 'flat',
+    'Mid Terraced Bungalow': 'bungalow',
+    'Maisonette Over Shop': 'maisonette',
+    'Flat Over Shop': 'flat',
+    'Medium Rise Flat': 'flat',
+    'End Terraced Town House': 'house',
+    'Maisonette Medium Rise': 'maisonette'

 }
--- a/backend/Property.py
+++ b/backend/Property.py
@ -1360,4 +1360,4 @@ class Property:
            'mechanical, supply and extract'
        ]

-        return self.data["mechanical-ventilation"] in ventilation_descriptions
+        return self.data.get("mechanical-ventilation") in ventilation_descriptions
--- a/backend/app/plan/schemas.py
+++ b/backend/app/plan/schemas.py
@ -14,7 +14,8 @@ SPECIFIC_MEASURES = [
    "suspended_floor_insulation", "solid_floor_insulation",
    "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump",
    "secondary_heating", "solar_pv", "double_glazing", "secondary_glazing",
-    "ventilation", "low_energy_lighting", "fireplace", "hot_water"
+    "ventilation", "low_energy_lighting", "fireplace", "hot_water_tank_insulation",
+    "cylinder_thermostat"
 ]

 NON_INVASIVE_SPECIFIC_MEASURES = [
@ -87,6 +88,7 @@ class PlanTriggerRequest(BaseModel):
    required_measures: Optional[List[InclusionOrExclusionItem]] = Field(default=[], min_length=0)

    scenario_name: Optional[str] = ""
+    scenario_id: Optional[str | int] = None  # Used to utilise and existing scenario for a engine run
    multi_plan: Optional[bool] = False
    optimise: Optional[bool] = True
    default_u_values: Optional[bool] = True
--- a/backend/engine/engine.py
+++ b/backend/engine/engine.py
@ -186,13 +186,28 @@ def extract_portfolio_aggregation_data(
                                                   f"{format_money(valuation_improvement_upper_bound_per_unit)})")
    )

+    if agg_data["cost"].sum() == 0:
+        valuation_percentage_increase = 0
+        valuation_increase_lower = 0
+        valuation_increase_upper = 0
+    else:
+        valuation_percentage_increase = round(total_valuation_increase / agg_data["cost"].sum(), 2)
+        valuation_increase_lower = agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum()
+        valuation_increase_upper = agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum()
+
    valuation_return_on_investment = str(
-        str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) +
+        str(valuation_percentage_increase) +
        f" ("
-        f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - "
-        f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})"
+        f"{valuation_increase_lower:,.2f} - "
+        f"{valuation_increase_upper:,.2f})"
    )

+    cost_per_co2_saved = agg_data["cost"].sum() / total_carbon_saved if total_carbon_saved > 0 else 0
+    cost_per_co2_saved = format_money(cost_per_co2_saved)
+
+    cost_per_sap_point = agg_data["cost"].sum() / total_sap_points if total_sap_points > 0 else 0
+    cost_per_sap_point = format_money(cost_per_sap_point)
+
    aggregation_data = {
        "epc_breakdown_pre_retrofit": json.dumps(
            reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict())
@ -212,8 +227,8 @@ def extract_portfolio_aggregation_data(
            round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh",
        "valuation_improvement_per_unit": valuation_improvment_per_unit,
        "cost_per_unit": format_money(agg_data["cost"].mean()),
-        "cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved),
-        "cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points),
+        "cost_per_co2_saved": cost_per_co2_saved,
+        "cost_per_sap_point": cost_per_sap_point,
        "valuation_return_on_investment": valuation_return_on_investment,
        # TODO: Could we add 10yr carbon credits value?
    }
@ -917,23 +932,28 @@ async def model_engine(body: PlanTriggerRequest):

        logger.info("Uploading recommendations to the database")
        # If we have any work to do, we create a new scenario
-        engine_scenario = create_scenario(
-            session=session,
-            scenario={
-                "name": body.scenario_name,
-                "created_at": created_at,
-                "budget": body.budget,
-                "portfolio_id": body.portfolio_id,
-                "housing_type": body.housing_type,
-                "goal": body.goal,
-                "trigger_file_path": body.trigger_file_path,
-                "already_installed_file_path": body.already_installed_file_path,
-                "patches_file_path": body.patches_file_path,
-                "non_invasive_recommendations_file_path": body.non_invasive_recommendations_file_path,
-                "exclusions": body.exclusions,
-                "multi_plan": body.multi_plan
-            }
-        )
+        if body.scenario_id:
+            # We don't need to create a new scenario, we just use the existing one
+            scenario_id = body.scenario_id
+        else:
+            engine_scenario = create_scenario(
+                session=session,
+                scenario={
+                    "name": body.scenario_name,
+                    "created_at": created_at,
+                    "budget": body.budget,
+                    "portfolio_id": body.portfolio_id,
+                    "housing_type": body.housing_type,
+                    "goal": body.goal,
+                    "trigger_file_path": body.trigger_file_path,
+                    "already_installed_file_path": body.already_installed_file_path,
+                    "patches_file_path": body.patches_file_path,
+                    "non_invasive_recommendations_file_path": body.non_invasive_recommendations_file_path,
+                    "exclusions": body.exclusions,
+                    "multi_plan": body.multi_plan
+                }
+            )
+            scenario_id = engine_scenario.id

        property_valuation_increases = []
        session.commit()
@ -979,7 +999,7 @@ async def model_engine(body: PlanTriggerRequest):
                    new_plan_id = create_plan(session, {
                        "portfolio_id": body.portfolio_id,
                        "property_id": p.id,
-                        "scenario_id": engine_scenario.id,
+                        "scenario_id": scenario_id,
                        "is_default": True if p.is_new else False,
                        "name": body.scenario_name,
                        "valuation_increase_lower_bound": (
@ -1033,7 +1053,7 @@ async def model_engine(body: PlanTriggerRequest):
        aggregate_portfolio_recommendations(
            session,
            portfolio_id=body.portfolio_id,
-            scenario_id=engine_scenario.id,
+            scenario_id=scenario_id,
            total_valuation_increase=total_valuation_increase,
            labour_days=labour_days,
            aggregated_data=aggregated_data
--- a/etl/find_my_epc/RetrieveFindMyEpc.py
+++ b/etl/find_my_epc/RetrieveFindMyEpc.py
@ -678,6 +678,9 @@ class RetrieveFindMyEpc:
            "Internal wall insulation": ["internal_wall_insulation"],
            "High heat retention storage heaters and dual immersion cylinder and dual rate meter": [
                "high_heat_retention_storage_heater"
+            ],
+            "High heat retention storage heaters and dual rate meter": [
+                "high_heat_retention_storage_heater"
            ]
        }