From 3fc1e69e50f5e0926a6f41ab0993108406010751 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 2 Jan 2026 17:39:48 +0800 Subject: [PATCH 1/3] debugging some failed runs --- .../db/functions/recommendations_functions.py | 158 +++++++++++------- .../epc_attributes/HotWaterAttributes.py | 1 + .../epc_attributes/WallAttributes.py | 2 + recommendations/WallRecommendations.py | 2 + 4 files changed, 105 insertions(+), 58 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 1ffb35d6..726e919c 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -1,3 +1,4 @@ +from sqlalchemy import text from sqlalchemy import insert, delete, select from sqlalchemy.orm import Session from sqlalchemy.exc import SQLAlchemyError @@ -365,84 +366,122 @@ def delete_property_batch(session: Session, property_ids: list[int]): if not property_ids: return - # -------------------------------------------------- - # Shared subqueries (computed once) - # -------------------------------------------------- - plan_ids = ( - select(Plan.id) - .where(Plan.property_id.in_(property_ids)) - ) + params = {"property_ids": property_ids} - recommendation_ids = ( - select(Recommendation.id) - .where(Recommendation.property_id.in_(property_ids)) - ) - - funding_package_ids = ( - select(FundingPackage.id) - .where(FundingPackage.plan_id.in_(plan_ids)) + # -------------------------------------------------- + # recommendation_materials (via recommendation) + # -------------------------------------------------- + session.execute( + text(""" + DELETE FROM recommendation_materials rm + USING recommendation r + WHERE rm.recommendation_id = r.id + AND r.property_id = ANY(:property_ids) + """), + params, ) # -------------------------------------------------- - # Leaf tables FIRST + # plan_recommendations (via plan) # -------------------------------------------------- session.execute( - delete(RecommendationMaterials) - .where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) - ) - - session.execute( - delete(PlanRecommendations) - .where(PlanRecommendations.plan_id.in_(plan_ids)) - ) - - session.execute( - delete(FundingPackageMeasures) - .where(FundingPackageMeasures.funding_package_id.in_(funding_package_ids)) - ) - - session.execute( - delete(InspectionModel) - .where(InspectionModel.property_id.in_(property_ids)) + text(""" + DELETE FROM plan_recommendations pr + USING plan p + WHERE pr.plan_id = p.id + AND p.property_id = ANY(:property_ids) + """), + params, ) # -------------------------------------------------- - # Mid-level tables + # funding_package_measures # -------------------------------------------------- session.execute( - delete(FundingPackage) - .where(FundingPackage.id.in_(funding_package_ids)) - ) - - session.execute( - delete(Recommendation) - .where(Recommendation.id.in_(recommendation_ids)) - ) - - session.execute( - delete(Plan) - .where(Plan.id.in_(plan_ids)) + text(""" + DELETE FROM funding_package_measures fpm + USING funding_package fp, plan p + WHERE fpm.funding_package_id = fp.id + AND fp.plan_id = p.id + AND p.property_id = ANY(:property_ids) + """), + params, ) # -------------------------------------------------- - # Property-scoped tables + # inspections (direct) # -------------------------------------------------- session.execute( - delete(PropertyDetailsEpcModel) - .where(PropertyDetailsEpcModel.property_id.in_(property_ids)) - ) - - session.execute( - delete(PropertyTargetsModel) - .where(PropertyTargetsModel.property_id.in_(property_ids)) + text(""" + DELETE FROM inspections + WHERE property_id = ANY(:property_ids) + """), + params, ) # -------------------------------------------------- - # Properties LAST + # funding_package # -------------------------------------------------- session.execute( - delete(PropertyModel) - .where(PropertyModel.id.in_(property_ids)) + text(""" + DELETE FROM funding_package fp + USING plan p + WHERE fp.plan_id = p.id + AND p.property_id = ANY(:property_ids) + """), + params, + ) + + # -------------------------------------------------- + # recommendation (direct — CRITICAL FIX) + # -------------------------------------------------- + session.execute( + text(""" + DELETE FROM recommendation + WHERE property_id = ANY(:property_ids) + """), + params, + ) + + # -------------------------------------------------- + # plan (direct) + # -------------------------------------------------- + session.execute( + text(""" + DELETE FROM plan + WHERE property_id = ANY(:property_ids) + """), + params, + ) + + # -------------------------------------------------- + # property-scoped tables + # -------------------------------------------------- + session.execute( + text(""" + DELETE FROM property_details_epc + WHERE property_id = ANY(:property_ids) + """), + params, + ) + + session.execute( + text(""" + DELETE FROM property_targets + WHERE property_id = ANY(:property_ids) + """), + params, + ) + + # -------------------------------------------------- + # properties LAST + # -------------------------------------------------- + session.execute( + text(""" + DELETE FROM property + WHERE id = ANY(:property_ids) + """), + params, ) @@ -481,11 +520,14 @@ def clear_portfolio_in_batches( return total = (len(property_ids) + property_batch_size - 1) // property_batch_size - + import time for i, batch in enumerate(chunked(property_ids, property_batch_size), start=1): print(f"Deleting batch {i}/{total} ({len(batch)} properties)") + start_time = time.time() with db_session() as session: delete_property_batch(session, batch) + finish_time = time.time() + print(f"Batch {i} deleted in {finish_time - start_time:.2f} seconds") # scenario deletion happens AFTER all properties are gone delete_portfolio_scenarios_if_empty(portfolio_id) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index 9966edea..53cd2f97 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -100,6 +100,7 @@ class HotWaterAttributes(Definitions): WELSH_TEXT = { "ogçör brif system": "from main system", "o r brif system": "from main system", + "o’r brif system": "from main system", "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery", "bwyler/cylchredydd nwy": "gas boiler/circulator", "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", diff --git a/etl/epc_clean/epc_attributes/WallAttributes.py b/etl/epc_clean/epc_attributes/WallAttributes.py index a390e0a5..0e211983 100644 --- a/etl/epc_clean/epc_attributes/WallAttributes.py +++ b/etl/epc_clean/epc_attributes/WallAttributes.py @@ -39,6 +39,8 @@ class WallAttributes(Definitions): "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, " "insulated (assumed)", "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Cavity wall, as built, insulated", + "Waliau ceudod, fel y’u hadeiladwyd, wedi’u hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, insulated (" + "assumed)", "Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Granite or whinstone, " "as built, no insulation (" "assumed)", diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 328f1ab8..49483d2f 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -72,6 +72,7 @@ class WallRecommendations(Definitions): 'Timber frame, as built, partial insulation': 'Timber frame, with external insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with external insulation", "Sandstone, as built, no insulation": "Sandstone, with external insulation", + "Sandstone, as built, partial insulation": "Sandstone, with external insulation", } # These are the ending descriptions we consider for walls with internal insulation @@ -88,6 +89,7 @@ class WallRecommendations(Definitions): 'Timber frame, as built, partial insulation': 'Timber frame, with internal insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with internal insulation", "Sandstone, as built, no insulation": "Sandstone, with internal insulation", + "Sandstone, as built, partial insulation": "Sandstone, with internal insulation", } def __init__( From ddd4aa479b4b79cc15721c9230a623bb92ecc60b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 2 Jan 2026 20:23:50 +0800 Subject: [PATCH 2/3] fixed bug with wall attributes cleaning --- backend/Property.py | 3 +++ etl/epc_clean/epc_attributes/WallAttributes.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 10af56cc..70a70307 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -614,6 +614,9 @@ class Property: # Handling edge case for walls fill_with = False if description == "walls-description" else None fill_dict = dict(zip(template.keys(), [fill_with] * len(template))) + if description == "walls-description": + fill_dict["thermal_transmittance_unit"] = None + fill_dict["insulation_thickness"] = "none" fill_dict.update( { diff --git a/etl/epc_clean/epc_attributes/WallAttributes.py b/etl/epc_clean/epc_attributes/WallAttributes.py index 0e211983..075dee96 100644 --- a/etl/epc_clean/epc_attributes/WallAttributes.py +++ b/etl/epc_clean/epc_attributes/WallAttributes.py @@ -148,6 +148,9 @@ class WallAttributes(Definitions): for key in self.DEFAULT_KEYS: result[key] = False + result["thermal_transmittance_unit"] = None + result["insulation_thickness"] = "none" + return result description = self.description.lower() From 8c7f5f8fb112c16222b8408b981dd04e0f2b6390 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 2 Jan 2026 20:44:34 +0800 Subject: [PATCH 3/3] fixed minor bugs --- backend/engine/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/engine.py b/backend/engine/engine.py index eb933cc0..f9820204 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -730,7 +730,7 @@ async def model_engine(body: PlanTriggerRequest): epc_searcher.ordnance_survey_client.property_type = addr.property_type # For the moment, our OS API access is unavailable, so we skip and interpolate - epc_searcher.find_property(skip_os=True, api_data=epc_api_data, overwrite_sap05=True) + epc_searcher.find_property(skip_os=True, api_data=None, overwrite_sap05=True) epc_searcher.set_uprn_source(file_format=body.file_format) lookup_key = (