diff --git a/.idea/Model.iml b/.idea/Model.iml index 09f2e496..c6561970 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index fb10c6b0..50cad4ca 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 21376708..611d0257 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -3155,10 +3155,19 @@ class AssetList: date_col = "Date letters sent" elif "Date Letter sent" in self.outcomes.columns: date_col = "Date Letter sent" + elif "WEEK COMMENCING" in self.outcomes.columns: + date_col = "WEEK COMMENCING" else: raise NotImplementedError("Invalid date in outcomes - implement me") - notes_col = "Notes" if "Notes" in self.outcomes.columns else "Notes / Outcomes" + if "Notes" in self.outcomes.columns: + notes_col = "Notes" + elif "Notes / Outcomes" in self.outcomes.columns: + notes_col = "Notes / Outcomes" + elif "NOTES" in self.outcomes.columns: + notes_col = "NOTES" + else: + raise NotImplementedError("Invalid notes in outcomes - implement me") lookup = lookup.merge( self.outcomes[["row_id", "Outcome", notes_col, date_col]], how="left", on="row_id" @@ -3342,6 +3351,10 @@ class AssetList: installer_notes_col = 'Installers Notes' elif 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM' in master_data.columns: installer_notes_col = 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM' + elif ('INSTALLERS NOTES / REASONS FOR CANCELLATIONS / WHERE INSTALL DATE WAS RECEIVED FROM' in + master_data.columns): + installer_notes_col = ('INSTALLERS NOTES / REASONS FOR CANCELLATIONS / WHERE INSTALL DATE WAS RECEIVED ' + 'FROM') else: raise ValueError("No installer notes column found in master data") diff --git a/asset_list/app.py b/asset_list/app.py index 1f0fe570..efc9cf44 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -59,12 +59,12 @@ def app(): Property UPRN """ - # Dorrington - data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Dorrington" - data_filename = "Copy of Eco Funding.xlsx" - sheet_name = "Sheet1" - postcode_column = 'Postcode' - fulladdress_column = "Property Address" + # TODO: Delete me + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild" + data_filename = "Bromford Asset List.xlsx" + sheet_name = "Asset List" + postcode_column = 'PostCode' + fulladdress_column = "FullAddress" address1_column = None address1_method = "house_number_extraction" address_cols_to_concat = [] @@ -76,23 +76,58 @@ def app(): landlord_wall_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "Row ID" + landlord_property_id = "Asset" outcomes_filename = [] outcomes_sheetname = [] outcomes_postcode = [] outcomes_houseno = [] outcomes_address = [] - outcomes_id = [] - master_filepaths = [] + outcomes_id = [None] + master_filepaths = [os.path.join("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/", + "Needs ID/SOLAR PV ONLY-Table 1.csv")] master_to_asset_list_filepath = None asset_list_header = 0 landlord_block_reference = None - master_id_colnames = [] + master_id_colnames = [None] landlord_roof_construction = None phase = False landlord_sap = None ecosurv_landlords = None + # For Housing + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025" + data_filename = "FOR HOUSING Asset List (Combined).xlsx" + sheet_name = "Asset List" + postcode_column = 'Postcode' + fulladdress_column = "Address" + address1_column = None + address1_method = "house_number_extraction" + address_cols_to_concat = [] + missing_postcodes_method = None + landlord_year_built = None + landlord_os_uprn = None + landlord_property_type = "Type" + landlord_built_form = "Type" + landlord_wall_construction = None + landlord_heating_system = "Heating - full" + landlord_existing_pv = None + landlord_property_id = "UPRN" + outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")] + outcomes_sheetname = ["Sheet1"] + outcomes_postcode = ["POSTCODE"] + outcomes_houseno = ["NO"] + outcomes_address = ["ADDRESS"] + outcomes_id = [None] + master_filepaths = [os.path.join(data_folder, "submissions.csv")] + master_to_asset_list_filepath = None + asset_list_header = 0 + landlord_block_reference = None + master_id_colnames = [None] + landlord_roof_construction = None + phase = False + landlord_sap = "SAP" + ecosurv_landlords = "for housing" + # CDS data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS" data_filename = "Founder Estates - Asset List.xlsx" diff --git a/asset_list/hubspot/config.py b/asset_list/hubspot/config.py index 23ff900a..403bead4 100644 --- a/asset_list/hubspot/config.py +++ b/asset_list/hubspot/config.py @@ -15,7 +15,7 @@ class HubspotProcessStatus(IntEnum): # This is the first stage, where a survey is ready to go READY_TO_BE_SCHEDULED = 1, "READY TO BE SCHEDULED" # The property didn't get access and needs sign off - SURVEYED_NO_ACCESS_NEEDS_SIGN_OFF = 2, "SURVEYED - NO ACCESS - NEED SIGN OFF" + SURVEYED_NO_ACCESS_NEEDS_SIGN_OFF = 2, "NO ACCESS - NEED SIGN OFF" # The survey has been completed. We don't have any update as to whether the property has been installed SURVEYED_COMPLETED_SIGNED_OFF = 3, "SURVEYED - AUTOMATED SIGNED OFF" # The property turned out to be ineligibile @@ -34,6 +34,7 @@ class Installer(Enum): SCIS = "SCIS" JJ_CRUMP = "J & J CRUMP" SGEC = "SGEC" + WARMFRONT = "WARM FRONT" @classmethod def is_valid_value(cls, value): diff --git a/asset_list/hubspot/prepare_for_hubspot.py b/asset_list/hubspot/prepare_for_hubspot.py index b12f4c04..ba2a2d23 100644 --- a/asset_list/hubspot/prepare_for_hubspot.py +++ b/asset_list/hubspot/prepare_for_hubspot.py @@ -45,13 +45,13 @@ def app(): # inputs: reconcile_programme = True # If True, the hubspot upload will include all properties with a project code - customer_domain = "https://ealing.gov.uk" - installer_name = "SCIS" + customer_domain = "https://calico.org.uk" + installer_name = "WARM FRONT" asset_list_filepath = ( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Hubspot/20250707 Ealing Flats - Prepared " - "programme.xlsx" + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Calico/Hubspot/07.04 CALICO - Final List - " + "Standardised.xlsx" ) - asset_list_sheet_name = "Standardised Asset List" + asset_list_sheet_name = "Final Route March" asset_list_header = 0 contact_details_filepath = None diff --git a/asset_list/mappings/built_form.py b/asset_list/mappings/built_form.py index c17e0ed4..4ebe016f 100644 --- a/asset_list/mappings/built_form.py +++ b/asset_list/mappings/built_form.py @@ -372,5 +372,18 @@ BUILT_FORM_MAPPINGS = { 'MAISONETTE': 'unknown', 'HOUSE': 'unknown', 'FLAT': 'unknown', - 'BLOCK': 'unknown' + 'BLOCK': 'unknown', + + 'Semi Detached Bungalow': 'semi-detached', + 'End Terraced Bungalow': 'end-terrace', + 'Mid Terraced Town House': 'mid-terrace', + 'Semi-Detached House': 'detached', + 'Low Rise Flat': 'low rise', + 'Mid Terraced Bungalow': 'mid-terrace', + 'End Terraced Town House': 'end-terrace', + 'Cottage Flat': 'ground floor', + 'Maisonette Over Shop': 'mid-floor', + 'Medium Rise Flat': 'mid-floor', + 'Maisonette Medium Rise': 'unknown' + } diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index 010d49a5..89bc2933 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -364,5 +364,19 @@ HEATING_MAPPINGS = { 'Boiler, Electricity': 'electric boiler', 'Boiler, LPG': 'gas boiler, radiators', 'Boiler, Mains gas': 'gas boiler, radiators', - 'Storage heating, Electricity': 'electric storage heaters' + 'Storage heating, Electricity': 'electric storage heaters', + + 'No Heating Types None': 'no heating', + 'Boiler Smokeless': 'boiler - other fuel', + 'Boiler House coal': 'boiler - other fuel', + 'Warm air Mains gas': 'warm air heating', + 'Storage heaters None': 'electric storage heaters', + 'Boiler Anthracite': 'boiler - other fuel', + 'Mains gas': 'gas boiler, radiators', + 'Community heating Mains Gas': 'communal gas boiler', + 'Warm air Electricity': 'warm air heating', + 'None': 'no heating', + 'Boiler None': 'unknown', + 'Storage heaters Electricity': 'electric storage heaters' + } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index caca0cf0..d45fd109 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -270,6 +270,19 @@ PROPERTY_MAPPING = { 'HFOP FLAT': 'flat', 'HFOP BEDSIT': 'bedsit', 'LINKED FLAT': 'flat', - 'LINKED BUNGALOW': 'bungalow' + 'LINKED BUNGALOW': 'bungalow', + + 'Semi Detached Bungalow': 'bungalow', + 'End Terraced Bungalow': 'bungalow', + 'Mid Terraced Town House': 'house', + 'Cottage Flat': 'flat', + 'Semi-Detached House': 'house', + 'Low Rise Flat': 'flat', + 'Mid Terraced Bungalow': 'bungalow', + 'Maisonette Over Shop': 'maisonette', + 'Flat Over Shop': 'flat', + 'Medium Rise Flat': 'flat', + 'End Terraced Town House': 'house', + 'Maisonette Medium Rise': 'maisonette' } diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 6b8b192d..ef73f133 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -14,7 +14,8 @@ SPECIFIC_MEASURES = [ "suspended_floor_insulation", "solid_floor_insulation", "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump", "secondary_heating", "solar_pv", "double_glazing", "secondary_glazing", - "ventilation", "low_energy_lighting", "fireplace", "hot_water" + "ventilation", "low_energy_lighting", "fireplace", "hot_water_tank_insulation", + "cylinder_thermostat" ] NON_INVASIVE_SPECIFIC_MEASURES = [ @@ -87,6 +88,7 @@ class PlanTriggerRequest(BaseModel): required_measures: Optional[List[InclusionOrExclusionItem]] = Field(default=[], min_length=0) scenario_name: Optional[str] = "" + scenario_id: Optional[str | int] = None # Used to utilise and existing scenario for a engine run multi_plan: Optional[bool] = False optimise: Optional[bool] = True default_u_values: Optional[bool] = True diff --git a/backend/engine/engine.py b/backend/engine/engine.py index d631e349..98862107 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -186,13 +186,28 @@ def extract_portfolio_aggregation_data( f"{format_money(valuation_improvement_upper_bound_per_unit)})") ) + if agg_data["cost"].sum() == 0: + valuation_percentage_increase = 0 + valuation_increase_lower = 0 + valuation_increase_upper = 0 + else: + valuation_percentage_increase = round(total_valuation_increase / agg_data["cost"].sum(), 2) + valuation_increase_lower = agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum() + valuation_increase_upper = agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum() + valuation_return_on_investment = str( - str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) + + str(valuation_percentage_increase) + f" (" - f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - " - f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})" + f"{valuation_increase_lower:,.2f} - " + f"{valuation_increase_upper:,.2f})" ) + cost_per_co2_saved = agg_data["cost"].sum() / total_carbon_saved if total_carbon_saved > 0 else 0 + cost_per_co2_saved = format_money(cost_per_co2_saved) + + cost_per_sap_point = agg_data["cost"].sum() / total_sap_points if total_sap_points > 0 else 0 + cost_per_sap_point = format_money(cost_per_sap_point) + aggregation_data = { "epc_breakdown_pre_retrofit": json.dumps( reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict()) @@ -212,8 +227,8 @@ def extract_portfolio_aggregation_data( round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh", "valuation_improvement_per_unit": valuation_improvment_per_unit, "cost_per_unit": format_money(agg_data["cost"].mean()), - "cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved), - "cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points), + "cost_per_co2_saved": cost_per_co2_saved, + "cost_per_sap_point": cost_per_sap_point, "valuation_return_on_investment": valuation_return_on_investment, # TODO: Could we add 10yr carbon credits value? } @@ -917,23 +932,28 @@ async def model_engine(body: PlanTriggerRequest): logger.info("Uploading recommendations to the database") # If we have any work to do, we create a new scenario - engine_scenario = create_scenario( - session=session, - scenario={ - "name": body.scenario_name, - "created_at": created_at, - "budget": body.budget, - "portfolio_id": body.portfolio_id, - "housing_type": body.housing_type, - "goal": body.goal, - "trigger_file_path": body.trigger_file_path, - "already_installed_file_path": body.already_installed_file_path, - "patches_file_path": body.patches_file_path, - "non_invasive_recommendations_file_path": body.non_invasive_recommendations_file_path, - "exclusions": body.exclusions, - "multi_plan": body.multi_plan - } - ) + if body.scenario_id: + # We don't need to create a new scenario, we just use the existing one + scenario_id = body.scenario_id + else: + engine_scenario = create_scenario( + session=session, + scenario={ + "name": body.scenario_name, + "created_at": created_at, + "budget": body.budget, + "portfolio_id": body.portfolio_id, + "housing_type": body.housing_type, + "goal": body.goal, + "trigger_file_path": body.trigger_file_path, + "already_installed_file_path": body.already_installed_file_path, + "patches_file_path": body.patches_file_path, + "non_invasive_recommendations_file_path": body.non_invasive_recommendations_file_path, + "exclusions": body.exclusions, + "multi_plan": body.multi_plan + } + ) + scenario_id = engine_scenario.id property_valuation_increases = [] session.commit() @@ -979,7 +999,7 @@ async def model_engine(body: PlanTriggerRequest): new_plan_id = create_plan(session, { "portfolio_id": body.portfolio_id, "property_id": p.id, - "scenario_id": engine_scenario.id, + "scenario_id": scenario_id, "is_default": True if p.is_new else False, "name": body.scenario_name, "valuation_increase_lower_bound": ( @@ -1033,7 +1053,7 @@ async def model_engine(body: PlanTriggerRequest): aggregate_portfolio_recommendations( session, portfolio_id=body.portfolio_id, - scenario_id=engine_scenario.id, + scenario_id=scenario_id, total_valuation_increase=total_valuation_increase, labour_days=labour_days, aggregated_data=aggregated_data