From 14c292a3ec2bc9fd19d39505ca510f2f7e773db5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 15 Dec 2025 09:30:01 +0800 Subject: [PATCH 1/2] handling coal community heating fuel --- backend/Property.py | 5 ++--- backend/engine/engine.py | 1 - .../d_restart_failed_subtasks.py | 9 +++++++++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py diff --git a/backend/Property.py b/backend/Property.py index 8f9637a9..f8013fb5 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -22,7 +22,6 @@ from recommendations.recommendation_utils import ( ) from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.app.utils import sap_to_epc -from backend.Funding import Funding import backend.app.assumptions as assumptions ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev") @@ -340,7 +339,6 @@ class Property: recommendation_record=recommendation_record, recommendations=property_representative_recommendations, primary_recommendation_id=self.id, - non_invasive_recommendations=self.non_invasive_recommendations, ) return scoring_dict @@ -1243,7 +1241,8 @@ class Property: "heat network": "Natural Gas (Community Scheme)", "lpg": 'LPG', "biodiesel": "Smokeless Fuel", - "b30d": "B30K Biofuel" + "b30d": "B30K Biofuel", + "coal": "Coal", } self.heating_energy_source = list({ diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 43b18d1b..0c6ed1de 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -15,7 +15,6 @@ from sqlmodel import Session from etl.epc.Record import EPCRecord from sqlalchemy.exc import IntegrityError, OperationalError -from sqlalchemy.orm import sessionmaker from starlette.responses import Response from backend.app.BatterySapScorer import BatterySAPScorer diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py b/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py new file mode 100644 index 00000000..79a8bdc2 --- /dev/null +++ b/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py @@ -0,0 +1,9 @@ +""" +For the Peabody project, there were a number of subtasks that failed due to issues, with the most +prominent being errors with the property address and ID data. + +This script will fetch those failed subtasks, get the associated properties and delete the properties +from the database so that the subtasks can be restarted cleanly. + +Additionally, we wil find the problematic records and remove them +""" From 1ae3aa5d6b9af3954bc4d8f9bff762295ed4c468 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 15 Dec 2025 11:06:00 +0800 Subject: [PATCH 2/2] upgrade storage type of rds to gp3 --- .../d_restart_failed_subtasks.py | 3 +++ etl/epc/Dataset.py | 8 ++++++-- infrastructure/terraform/main.tf | 4 ++++ recommendations/Recommendations.py | 7 +++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py b/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py index 79a8bdc2..5ef901b2 100644 --- a/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py +++ b/etl/customers/peabody/Nov 2025 Consulting Project/d_restart_failed_subtasks.py @@ -6,4 +6,7 @@ This script will fetch those failed subtasks, get the associated properties and from the database so that the subtasks can be restarted cleanly. Additionally, we wil find the problematic records and remove them + +Given we ran an EPC C scenario, we should check how many properties, below EPC C we have, that have no plan +or recommendations in case something went wrong """ diff --git a/etl/epc/Dataset.py b/etl/epc/Dataset.py index 01391874..7c27de51 100644 --- a/etl/epc/Dataset.py +++ b/etl/epc/Dataset.py @@ -697,8 +697,12 @@ class TrainingDataset(BaseDataset): for x in missing_descriptions: desc_cleaner = cleaner(x) cleaned = desc_cleaner.process() - # IF NODATA, REMAP TO NONE VALUES - if all((pd.DataFrame(cleaned, index=[0]).T)[0] == False): + # IF NODATA, REMAP TO NONE VALUES, apart from walls which we want to keep as is + # If we convert the walls data to None, we end up converting booleans to None which + # causes issues downstream + if all( + (pd.DataFrame(cleaned, index=[0]).T)[0] == False + ) and component != "walls": cleaned = {key: None for key in cleaned.keys()} cleaned_data.append( { diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index f345c985..a26a1af4 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -85,6 +85,10 @@ resource "aws_db_instance" "default" { ca_cert_identifier = "rds-ca-rsa2048-g1" # Temporary to enfore immediate change apply_immediately = true + # Set up storage type to gp3 for better performance + storage_type = "gp3" + iops = 3000 + storage_throughput = 125 } # Set up the bucket that recieve the csv uploads of epc to be retrofit diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 5d97a554..f984acc3 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -698,6 +698,13 @@ class Recommendations: "heating_cop": 0.85, "hotwater_cop": 0.85 } + if main_fuel_description in ['coal (community)']: + return { + "heating_fuel_type": "Coal", + "hotwater_fuel_type": "Coal", + "heating_cop": 0.85, + "hotwater_cop": 0.85 + } logger.warning( "Unhandled community fuel." f"Fuel: {main_fuel_description}"