junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixing route march data pull

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-01-17 18:53:04 +00:00
parent ee0edec07c
commit ad3ba92475
12 changed files with 254 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Stonewater-wave-3" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyNamespacePackagesService">

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Stonewater-wave-3" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>

13
backend/SearchEpc.py
View file

@ -693,9 +693,20 @@ class SearchEpc:
estimated_epc[variable] = str(int(estimated_epc[variable]))
# This is a string
estimated_epc["low-energy-fixed-light-count"] = str(estimated_epc["low-energy-fixed-light-count"])
estimated_epc["low-energy-fixed-light-count"] = (
str(estimated_epc["low-energy-fixed-light-count"]) if estimated_epc["low-energy-fixed-light-count"] else ""
)
# This is an int
estimated_epc["photo-supply"] = (
int(np.round(estimated_epc["photo-supply"])) if estimated_epc["photo-supply"] else estimated_epc[
"photo-supply"]
)
estimated_epc["postcode"] = self.postcode
if not self.uprn:
# Update self.uprn too
self.uprn = hash(self.address1 + self.postcode)
estimated_epc["uprn"] = self.uprn
estimated_epc["address"] = self.full_address
# Indicate that this epc was estimated

58
backend/app/plan/router.py
View file

@ -407,6 +407,7 @@ async def trigger_plan(body: PlanTriggerRequest):
plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path)
# Check for duplicate UPRNS
input_uprns = [x.get("uprn") for x in plan_input if "uprn" in x and x.get("uprn")]
if input_uprns:
# Check for dupes
if len(input_uprns) != len(set(input_uprns)):
@ -680,37 +681,42 @@ async def trigger_plan(body: PlanTriggerRequest):
input_measures = prepare_input_measures(recommendations[p.id], body.goal)
current_sap_points = int(p.data["current-energy-efficiency"])
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
sap_gain = CostOptimiser.calculate_sap_gain_with_slack(target_sap_points - current_sap_points)
if not body.optimise:
if body.goal != "Increasing EPC":
raise NotImplementedError("Only EPC optimisation is currently supported")
solution = []
for sub_list in input_measures:
# Select the entry with the highest gain, and if tied, choose the one with the lowest cost
best_measure = max(sub_list, key=lambda x: (x['gain'], -x['cost']))
solution.append(best_measure)
if not input_measures[0]:
# This means that we have no defaults
selected_recommendations = {}
else:
if body.budget:
optimiser = GainOptimiser(
input_measures, max_cost=body.budget, max_gain=sap_gain if sap_gain > 0 else 0
)
current_sap_points = int(p.data["current-energy-efficiency"])
target_sap_points = epc_to_sap_lower_bound(body.goal_value)
sap_gain = CostOptimiser.calculate_sap_gain_with_slack(target_sap_points - current_sap_points)
if not body.optimise:
if body.goal != "Increasing EPC":
raise NotImplementedError("Only EPC optimisation is currently supported")
solution = []
for sub_list in input_measures:
# Select the entry with the highest gain, and if tied, choose the one with the lowest cost
best_measure = max(sub_list, key=lambda x: (x['gain'], -x['cost']))
solution.append(best_measure)
else:
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
# If the gain is negative, the optimiser will return an empty solution
optimiser = CostOptimiser(
input_measures,
min_gain=sap_gain
)
optimiser.setup()
optimiser.solve()
solution = optimiser.solution
if body.budget:
optimiser = GainOptimiser(
input_measures, max_cost=body.budget, max_gain=sap_gain if sap_gain > 0 else 0
)
else:
# The minimum gain is the minimum number of SAP points required to get to the target SAP band
# If the gain is negative, the optimiser will return an empty solution
optimiser = CostOptimiser(
input_measures,
min_gain=sap_gain
)
selected_recommendations = {r["id"] for r in solution}
optimiser.setup()
optimiser.solve()
solution = optimiser.solution
selected_recommendations = {r["id"] for r in solution}
# If wall insulation is selected, we also include mechanical ventilation as a best practice measure
if any(x in [r["type"] for r in solution] for x in [

8
backend/ml_models/AnnualBillSavings.py
View file

@ -28,8 +28,8 @@ class AnnualBillSavings:
# Latest price cap figures from Ofgem are for April 2024
# https://www.ofgem.gov.uk/energy-price-cap
ELECTRICITY_PRICE_CAP = 0.2236
GAS_PRICE_CAP = 0.0548
ELECTRICITY_PRICE_CAP = 0.2486
GAS_PRICE_CAP = 0.0634
# This is the most recent export payment figure, at 9.28p/kWh
# Smart export guarantee rates can be found here:
# https://www.sunsave.energy/solar-panels-advice/exporting-to-the-grid/best-seg-rates
@ -39,8 +39,8 @@ class AnnualBillSavings:
PRICE_FACTOR = 0.09549999999999999
# Daily standard charge, based on average across England, Scotland and Wales, and includes VAT
DAILY_STANDARD_CHARGE_GAS = 0.3143
DAILY_STANDARD_CHARGE_ELECTRICITY = 0.601
DAILY_STANDARD_CHARGE_GAS = 0.3165
DAILY_STANDARD_CHARGE_ELECTRICITY = 0.6097
# Based on https://www.nottenergy.com/advice-and-tools/project-energy-cost-comparison
# For July 2024. These quotes are based on the east midlands region, so we

8
etl/customers/cambridge/remote_assessment.py
View file

@ -21,10 +21,10 @@ def app():
"property_type": "House", "built-form": "Semi-Detached"
},
{
"address": "21 High Street", "postcode": "CB23 8AB", "uprn": 100090136026
"address": "21 High Street", "postcode": "CB23 8AB", "uprn": 100090144815
},
{
"address": "22 High Street", "postcode": "CB23 8AB", "uprn": 100090136027
"address": "22 High Street", "postcode": "CB23 8AB", "uprn": 100090144816
},
{
"address": "5 Bunkers Hill", "postcode": "CB3 0LY", "uprn": 10008078615
@ -52,8 +52,8 @@ def app():
valuations_data = [
{'uprn': 100090136018, "valuation": 586_000},
{'uprn': 100090136026, "valuation": 551_000},
{'uprn': 100090136027, "valuation": 844_000},
{'uprn': 100090144815, "valuation": 446_000},
{'uprn': 100090144816, "valuation": 448_000},
{'uprn': 10008078615, "valuation": 763_000},
{'uprn': 10008078616, "valuation": 616_000},
{'uprn': 10008078617, "valuation": 593_000},

166
etl/customers/l_and_g/ic_asset_list.py Normal file
View file

@ -0,0 +1,166 @@
"""
This script prepares the asset list for modelling the properties from the L&Q dataset, for their January IC
"""
import pandas as pd
import numpy as np
from etl.route_march_data_pull.app import get_data
from utils.s3 import save_csv_to_s3
PORTFOLIO_ID = 124
USER_ID = 8
def app():
asset_data = pd.read_excel(
"/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Basildon information for Domna/Basildon MDS v1.4 "
"(1).xlsx",
sheet_name="Basildon",
header=5
)
asset_data = asset_data.head(-3)
asset_data["address1"] = np.where(
pd.isnull(asset_data["Address 1"]),
asset_data["Address 2"],
asset_data["Address 1"]
)
asset_data["full_address"] = np.where(
pd.isnull(asset_data["Address 1"]),
asset_data["Address 2"] + ", " + asset_data["Address 3"],
asset_data["Address 1"] + ", " + asset_data["Address 2"] + ", " + asset_data["Address 3"],
)
asset_list = asset_data[["address1", "PostCode", "full_address", "Bedrooms"]]
asset_list = asset_list.reset_index(drop=True)
asset_list["row_id"] = asset_list.index
# L&G's focus:
# Measures: loft and cavity insulation, replacement thermally efficient windows, PV cells, AS heat pumps.
epc_data, errors, no_epc = get_data(
asset_list=asset_list,
fulladdress_column="full_address",
address1_column="address1",
postcode_column="PostCode",
manual_uprn_map={}
)
missed = asset_list[
asset_list["row_id"].isin(no_epc)
]
# We merge on the property types, where we have them
missed = missed.merge(
asset_data[["address1", "PostCode", "Property Type"]],
how="left",
on=["address1", "PostCode"]
)
# Remap Block: Residential to Flat
missed["Property Type"] = np.where(
missed["Property Type"] == "Block: Residential",
"Flat",
missed["Property Type"]
)
# We create the asset list - we have some properties that genuninely never had an EPC
epc_df = pd.DataFrame(epc_data)
fetched_asset_list = epc_df[["address1", "postcode", "uprn", "row_id"]]
fetched_asset_list = fetched_asset_list.merge(
asset_list[["row_id", "Bedrooms"]],
how="left",
on=["row_id"]
)
missed = missed.rename(columns={"PostCode": "postcode"}).drop(columns=["row_id"])
# missed.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/missed_epcs.csv")
missed_uprns = pd.read_csv(
"/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/missed_epcs_uprn.csv",
)
missed = missed.merge(
missed_uprns[["address1", "postcode", "UPRN"]].rename(
columns={"UPRN": "uprn"},
),
how="left",
on=["address1", "postcode"]
)
fetched_asset_list = fetched_asset_list.drop(columns=["row_id"])
# We concatename them
final_asset_list = pd.concat(
[fetched_asset_list, missed[["address1", "postcode", "Property Type", "Bedrooms", "uprn"]]]
)
final_asset_list = final_asset_list.rename(
columns={
"address1": "address",
"Property Type": "property_type",
"Bedrooms": "n_bedrooms"
}
)
# Finally, we merge on the numeber of bedrooms
# Extract the non-invasive recommendations:
non_invasive_recommendations = []
for x in epc_data:
non_invasive_recommendations.append(
{
"uprn": x["uprn"],
"recommendations": x["find_my_epc_data"]["recommendations"]
}
)
filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
save_csv_to_s3(
dataframe=pd.DataFrame(final_asset_list),
bucket_name="retrofit-plan-inputs-dev",
file_name=filename
)
# Store the non-invasive recommendations in s3
non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.csv"
save_csv_to_s3(
dataframe=pd.DataFrame(non_invasive_recommendations),
bucket_name="retrofit-plan-inputs-dev",
file_name=non_invasive_recommendations_filename
)
# Store the valuations data in s3
# valuations_filename = f"{USER_ID}/{PORTFOLIO_ID}/valuations.csv"
# save_csv_to_s3(
# dataframe=pd.DataFrame(valuations_data),
# bucket_name="retrofit-plan-inputs-dev",
# file_name=valuations_filename
# )
body = {
"portfolio_id": str(PORTFOLIO_ID),
"housing_type": "Private",
"goal": "Increasing EPC",
"goal_value": "A",
"trigger_file_path": filename,
"already_installed_file_path": "",
"patches_file_path": "",
"non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
"valuation_file_path": "",
"scenario_name": "Retrofit Packages",
"multi_plan": True,
"budget": None,
"inclusions": [
"cavity_wall_insulation",
"loft_insulation",
"windows",
"solar_pv",
"air_source_heat_pump"
]
}
print(body)

8
etl/find_my_epc/RetrieveFindMyEpc.py
View file

@ -300,6 +300,8 @@ class RetrieveFindMyEpc:
"Fan assisted storage heaters": [],
"Fan-assisted storage heaters": [],
"Step 1:": [],
"Step 2:": [],
'Step 3:': [],
"Biomass stove with boiler": [],
"Replace boiler with biomass boiler": [],
"Heating controls (room thermostat and thermostatic radiator valves)": [
@ -308,7 +310,11 @@ class RetrieveFindMyEpc:
"Heating controls (programmer, and thermostatic radiator valves)": [
"roomstat_programmer_trvs", "time_temperature_zone_control"
],
"Replacement warm air unit": []
"Heating controls (programmer and TRVs)": [
"roomstat_programmer_trvs", "time_temperature_zone_control"
],
"Replacement warm air unit": [],
"Secondary glazing": ["secondary_glazing"]
}
survey = True

18
etl/route_march_data_pull/app.py
View file

@ -1,6 +1,5 @@
import os
import time
from idlelib.iomenu import errors
import pandas as pd
import numpy as np
@ -25,7 +24,6 @@ def get_data(asset_list, fulladdress_column, address1_column, postcode_column, m
epc_data = []
errors = []
no_epc = []
# home = asset_list[asset_list["row_id"] == errors[5]].squeeze()
for _, home in tqdm(asset_list.iterrows(), total=len(asset_list)):
try:
postcode = home[postcode_column]
@ -154,13 +152,13 @@ def app():
Property UPRN
"""
DATA_FOLDER = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Watford"
DATA_FILENAME = "JS Mailing List 10122024.xlsx"
SHEET_NAME = "Export"
DATA_FOLDER = "/Users/khalimconn-kowlessar/Documents/hestia/Route Marches"
DATA_FILENAME = "Full Below SAP C Stock - RN Copy.xlsx"
SHEET_NAME = "Electric Properties"
POSTCODE_COLUMN = "Postcode"
FULLADDRESS_COLUMN = "Property Address"
ADDRESS1_COLUMN = "Address Line 1"
ADDRESS1_METHOD = None
FULLADDRESS_COLUMN = "Address"
ADDRESS1_COLUMN = None
ADDRESS1_METHOD = "first_two_words"
ADDRESS_COLS_TO_CONCAT = []
# Maps addresses to uprn in problematic cases
@ -372,7 +370,9 @@ def app():
how="left",
on="row_id"
)
asset_list = asset_list.drop(columns=["row_id"])
asset_list = asset_list.drop(columns=["row_id", "index"])
asset_list[asset_list["Assessor’s name"] == "Robin Bailey"]["Assessor's Email"].value_counts()
# Store as an excel
filename = os.path.join(DATA_FOLDER, ".".join(DATA_FILENAME.split(".")[:-1])) + " EPC Data Pull - Main.xlsx"

13
recommendations/Recommendations.py
View file

@ -28,6 +28,9 @@ class Recommendations:
High level recommendations class, which sits above the measure specific recommendation classes
"""
# Constant for carbon intensity calculation, as of 16th Jan 2025
CARBON_INTENSITY = 0.232
def __init__(
self,
property_instance: Property,
@ -691,6 +694,10 @@ class Recommendations:
"""
This method inserts the kwh savings and the bill savings that the customer will make from the recommendations
based on the predictions from the ML model
It also ensures we base our solar savings and solar carbon savings from the calculations based on
the solar API and size of the array, instead of ML model
:param property_instance: Instance of the Property class, for the home associated to property_id
:param kwh_simulation_predictions: dictionary of predictions from the model apis
:param property_recommendations: dictionary of recommendations for the property
@ -824,6 +831,12 @@ class Recommendations:
if rec["type"] == "solar_pv":
rec["kwh_savings"] = rec_impact["solar_kwh_savings"].values[0]
# Calculate carbon savings from this
emissions_kg = rec["kwh_savings"] * cls.CARBON_INTENSITY # Calculate emissions in kg
emissions_tonnes = emissions_kg / 1000
rec["co2_equivalent_savings"] = emissions_tonnes
rec["energy_cost_savings"] = (
rec_impact["solar_kwh_savings"].values[0] * AnnualBillSavings.ELECTRICITY_PRICE_CAP
)

4
recommendations/RoofRecommendations.py
View file

@ -138,6 +138,10 @@ class RoofRecommendations:
u_value = self.property.roof["thermal_transmittance"]
# If we have a flat roof but we don't have flat roof as a measure, we exit
if self.property.roof["is_flat"] and "flat_roof_insulation" not in measures:
return
# We check if the roof is already insulated and if so, we exit
# Building regulations part L recommend installing at least 270mm of insulation, however generally we

2
recommendations/optimiser/optimiser_functions.py
View file

@ -31,7 +31,7 @@ def prepare_input_measures(property_recommendations, goal):
"gain": rec[goal_key],
"type": rec["type"]
}
for rec in recs
for rec in recs if rec["energy_cost_savings"] >= 0
]
)