mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
300 lines
10 KiB
Python
300 lines
10 KiB
Python
import time
|
|
|
|
import pandas as pd
|
|
|
|
from backend.SearchEpc import SearchEpc
|
|
from dotenv import load_dotenv
|
|
import os
|
|
from tqdm import tqdm
|
|
from utils.s3 import save_csv_to_s3, read_excel_from_s3
|
|
|
|
# Read in the .env file in backend
|
|
load_dotenv(dotenv_path="backend/.env")
|
|
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
|
|
# Stored in my notes
|
|
ORDNANCE_SURVEY_API_KEY = ""
|
|
|
|
PORTFOLIO_ID = 80
|
|
USER_ID = 8
|
|
|
|
|
|
def extract_mds_measures(config):
|
|
measures = []
|
|
if not pd.isnull(config["EWI (Trad Const)"]):
|
|
measures.append({"external_wall_insulation": "EWI (Trad Const)"})
|
|
|
|
if not pd.isnull(config["EWI (Non Trad Const)"]):
|
|
measures.append({"external_wall_insulation": "EWI (Non Trad Const)"})
|
|
|
|
if not pd.isnull(config["CWI"]):
|
|
measures.append({"cavity_wall_insulation": "CWI"})
|
|
|
|
if not pd.isnull(config["LI"]):
|
|
measures.append({"loft_insulation": "LI"})
|
|
|
|
if not pd.isnull(config["Party Wall Insu"]):
|
|
measures.append({"party_wall_insulation": "Party Wall Insu"})
|
|
|
|
if not pd.isnull(config["IWI (POA - Prov Sum Only)"]):
|
|
measures.append({"internal_wall_insulation": "IWI (POA - Prov Sum Only)"})
|
|
|
|
if not pd.isnull(config["U/F Insu (Manual install)"]):
|
|
measures.append({"suspended_floor_insulation": "U/F Insu (Manual install)"})
|
|
|
|
if not pd.isnull(config["U/F insu (Qbot)"]):
|
|
measures.append({"suspended_floor_insulation": "U/F insu (Qbot)"})
|
|
|
|
if not pd.isnull(config["Solid floor insl (Out of scope - Prov sum only)"]):
|
|
measures.append({"solid_floor_insulation": "Solid floor insl (Out of scope - Prov sum only)"})
|
|
|
|
if not pd.isnull(config["ASHP Htg"]):
|
|
measures.append({"air_source_heat_pump": "ASHP Htg"})
|
|
|
|
if not pd.isnull(config["GSHP Htg"]):
|
|
measures.append({"ground_source_heat_pump": "GSHP Htg"})
|
|
|
|
if not pd.isnull(config["Shared ground loops"]):
|
|
measures.append({"shared_ground_loops": "Shared ground loops"})
|
|
|
|
if not pd.isnull(config["Communal heat networks"]):
|
|
measures.append({"communal_heat_networks": "Communal heat networks"})
|
|
|
|
if not pd.isnull(config["District heating networks"]):
|
|
measures.append({"district_heating_networks": "District heating networks"})
|
|
|
|
if not pd.isnull(config["Elec Storage Htrs (Out of scope -Prov sum only)"]):
|
|
measures.append({"high_heat_retention_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)"})
|
|
|
|
if not pd.isnull(config["Low Energy Bulbs"]):
|
|
measures.append({"low_energy_lighting": "Low Energy Bulbs"})
|
|
|
|
if not pd.isnull(config["Cyl Insulation"]):
|
|
measures.append({"cylinder_insulation": "Cyl Insulation"})
|
|
|
|
if not pd.isnull(config["Smart controls"]):
|
|
measures.append({"smart_controls": "Smart controls"})
|
|
|
|
if not pd.isnull(config["Zone controls"]):
|
|
measures.append({"zone_controls": "Zone controls"})
|
|
|
|
if not pd.isnull(config["Upgrade TRV's"]):
|
|
measures.append({"trvs": "Upgrade TRV's"})
|
|
|
|
if not pd.isnull(config["Solar PV"]):
|
|
measures.append({"solar_pv": "Solar PV"})
|
|
|
|
if not pd.isnull(config["Solar Thermal"]):
|
|
measures.append({"solar_thermal": "Solar Thermal"})
|
|
|
|
if not pd.isnull(config["Double Glazing (POA - Prov sum only)"]):
|
|
measures.append({"double_glazing": "Double Glazing (POA - Prov sum only)"})
|
|
|
|
if not pd.isnull(config["Draught Proofing"]):
|
|
measures.append({"draught_proofing": "Draught Proofing"})
|
|
|
|
if not pd.isnull(config["Ventilation upgrade"]):
|
|
measures.append({"mechanical_ventilation": "Ventilation upgrade"})
|
|
|
|
if not pd.isnull(config["Gas Boiler Replacement"]):
|
|
measures.append({"gas_boiler": "Gas Boiler Replacement"})
|
|
|
|
if not pd.isnull(config["Flat roof (Out of scope - prov sum only)"]):
|
|
measures.append({"flat_roof_insulation": "Flat roof (Out of scope - prov sum only)"})
|
|
|
|
if not pd.isnull(config["RIR (POA - Prov sum only)"]):
|
|
measures.append({"room_in_roof_insulation": "RIR (POA - Prov sum only)"})
|
|
|
|
if not pd.isnull(config["EV Charging"]):
|
|
measures.append({"ev_charging": "EV Charging"})
|
|
|
|
if not pd.isnull(config["Battery"]):
|
|
measures.append({"battery": "Battery"})
|
|
|
|
return measures
|
|
|
|
|
|
def parse_property_type(config):
|
|
# This should come from the ordnance survey api eventually
|
|
|
|
# array(['Detached', 'Semi-detached', 'Bungalow', 'Mid Terrace',
|
|
# 'End Terrace', 'Top Flat', 'Mid Flat',
|
|
# 'Low rise flat (1-2 storey)', nan], dtype=object)
|
|
|
|
if config["Address"] == "Flat Central Garage":
|
|
return {"property_type": "Bungalow", "built_form": "Mid-Terrace"}
|
|
|
|
if pd.isnull(config["Property Type"]):
|
|
return {"property_type": None, "built_form": None}
|
|
|
|
lookup = {
|
|
"Detached": {"property_type": "House", "built_form": "Detached"},
|
|
"Semi-detached": {"property_type": "House", "built_form": "Semi-detached"},
|
|
"Bungalow": {"property_type": "Bungalow", "built_form": "Detached"},
|
|
"Mid Terrace": {"property_type": "House", "built_form": "Mid-Terrace"},
|
|
"End Terrace": {"property_type": "House", "built_form": "End-Terrace"},
|
|
"Top Flat": {"property_type": "Flat", "built_form": None},
|
|
"Mid Flat": {"property_type": "Flat", "built_form": None},
|
|
"Low rise flat (1-2 storey)": {"property_type": "Flat", "built_form": None},
|
|
}
|
|
|
|
return lookup[config["Property Type"]]
|
|
|
|
|
|
def app():
|
|
"""
|
|
Create the initial asset list for the E.ON pilot
|
|
:return:
|
|
"""
|
|
|
|
raw_asset_list = read_excel_from_s3(
|
|
bucket_name="retrofit-datalake-dev",
|
|
file_key="customers/E.ON/sample SHDF Information MDS Template Vr3.0.xlsx",
|
|
header_row=11,
|
|
drop_all_na=False
|
|
)
|
|
|
|
# Keep just the columns we need
|
|
raw_asset_list_base = raw_asset_list[
|
|
[
|
|
"Address", "Postcode", "No Bedrooms"
|
|
]
|
|
].copy().rename(
|
|
columns={
|
|
"Address": "address",
|
|
"Postcode": "postcode",
|
|
"No Bedrooms": "n_bedrooms"
|
|
}
|
|
)
|
|
|
|
# For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free
|
|
# trial with Ordnance Survey with my personal account as a temporary solution.
|
|
# Let's just pull the full EPC data for this
|
|
asset_list_with_uprn = []
|
|
for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]):
|
|
searcher = SearchEpc(
|
|
address1=property_meta["address"],
|
|
postcode=property_meta["postcode"],
|
|
auth_token=EPC_AUTH_TOKEN,
|
|
os_api_key=ORDNANCE_SURVEY_API_KEY,
|
|
full_address=", ".join([property_meta["address"], property_meta["postcode"]])
|
|
)
|
|
|
|
searcher.find_property(skip_os=True)
|
|
if searcher.newest_epc["uprn-source"] == SearchEpc.UPRN_SOURCE_SIMULATED:
|
|
uprn = None
|
|
else:
|
|
uprn = searcher.uprn
|
|
# searcher.find_property(skip_os=False)
|
|
|
|
asset_list_with_uprn.append(
|
|
{
|
|
**property_meta,
|
|
"uprn": uprn,
|
|
"matched_address": searcher.address1,
|
|
"matched_postcode": searcher.postcode
|
|
}
|
|
)
|
|
|
|
# Store this as a backup
|
|
# import pandas as pd
|
|
# asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn)
|
|
# asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn_2.csv", index=False)
|
|
# Read in
|
|
# asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records")
|
|
|
|
# Store the asset list and create the portfolio payload
|
|
asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn)
|
|
asset_list_with_uprn_df["uprn"] = asset_list_with_uprn_df["uprn"].astype(str).astype(int)
|
|
|
|
# We now determine which measures we need for each property
|
|
finalised_asset_list = []
|
|
for i, config in raw_asset_list.iterrows():
|
|
|
|
asset_config = asset_list_with_uprn_df[
|
|
(asset_list_with_uprn_df["address"] == config["Address"]) &
|
|
(asset_list_with_uprn_df["postcode"] == config["Postcode"])
|
|
]
|
|
if asset_config.shape[0] != 1:
|
|
raise ValueError("Could not find a unique match for the property")
|
|
|
|
measures = extract_mds_measures(config)
|
|
|
|
# Get the property type
|
|
pt = parse_property_type(config)
|
|
|
|
if config["Address"] in [
|
|
"28 Hermitage Lane",
|
|
"35a High Street",
|
|
"35b High Street",
|
|
"Flat Over 20 Holborough Road",
|
|
"Flat above 7 Malling Road",
|
|
"Cobnut Barn",
|
|
]:
|
|
print(config["Address"])
|
|
uprn = None
|
|
else:
|
|
uprn = asset_config["uprn"].values[0]
|
|
|
|
finalised_asset_list.append(
|
|
{
|
|
"address": config["Address"],
|
|
"postcode": config["Postcode"],
|
|
"uprn": uprn,
|
|
"n_bedrooms": config["No Bedrooms"],
|
|
"measures": measures,
|
|
**pt
|
|
}
|
|
)
|
|
finalised_asset_list = pd.DataFrame(finalised_asset_list)
|
|
|
|
# Store the asset list in s3
|
|
filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
|
|
save_csv_to_s3(
|
|
dataframe=finalised_asset_list,
|
|
bucket_name="retrofit-plan-inputs-dev",
|
|
file_name=filename
|
|
)
|
|
|
|
# EPC C portoflio
|
|
body = {
|
|
"portfolio_id": str(PORTFOLIO_ID),
|
|
"housing_type": "Social",
|
|
"goal": "Increase EPC",
|
|
"goal_value": "C",
|
|
"trigger_file_path": filename,
|
|
"already_installed_file_path": "",
|
|
"patches_file_path": "",
|
|
"non_invasive_recommendations_file_path": "",
|
|
"budget": None,
|
|
}
|
|
print(body)
|
|
|
|
# Optimised version where we specify the measures
|
|
measures = [
|
|
"external_wall_insulation",
|
|
"cavity_wall_insulation",
|
|
"loft_insulation",
|
|
"air_source_heat_pump",
|
|
"high_heat_retention_storage_heaters",
|
|
"solar_pv"
|
|
]
|
|
|
|
body = {
|
|
"portfolio_id": str(PORTFOLIO_ID),
|
|
"housing_type": "Social",
|
|
"goal": "Increase EPC",
|
|
"goal_value": "C",
|
|
"trigger_file_path": filename,
|
|
"already_installed_file_path": "",
|
|
"patches_file_path": "",
|
|
"non_invasive_recommendations_file_path": "",
|
|
"measures": measures,
|
|
"budget": None,
|
|
}
|
|
|
|
|
|
output = []
|
|
for r in self.results:
|
|
output.append(r["DPA"])
|
|
|
|
output = pd.DataFrame(output)
|