debugging some failed runs

backend/app/db/functions/recommendations_functions.py

@@ -1,3 +1,4 @@
+from sqlalchemy import text
 from sqlalchemy import insert, delete, select
 from sqlalchemy.orm import Session
 from sqlalchemy.exc import SQLAlchemyError
@@ -365,84 +366,122 @@ def delete_property_batch(session: Session, property_ids: list[int]):
     if not property_ids:
         return
 
-    # --------------------------------------------------
+    params = {"property_ids": property_ids}
-    # Shared subqueries (computed once)
-    # --------------------------------------------------
-    plan_ids = (
-        select(Plan.id)
-        .where(Plan.property_id.in_(property_ids))
-    )
 
-    recommendation_ids = (
+    # --------------------------------------------------
-        select(Recommendation.id)
+    # recommendation_materials (via recommendation)
-        .where(Recommendation.property_id.in_(property_ids))
+    # --------------------------------------------------
-    )
+    session.execute(
-
+        text("""
-    funding_package_ids = (
+            DELETE FROM recommendation_materials rm
-        select(FundingPackage.id)
+            USING recommendation r
-        .where(FundingPackage.plan_id.in_(plan_ids))
+            WHERE rm.recommendation_id = r.id
+              AND r.property_id = ANY(:property_ids)
+        """),
+        params,
     )
 
     # --------------------------------------------------
-    # Leaf tables FIRST
+    # plan_recommendations (via plan)
     # --------------------------------------------------
     session.execute(
-        delete(RecommendationMaterials)
+        text("""
-        .where(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
+            DELETE FROM plan_recommendations pr
-    )
+            USING plan p
-
+            WHERE pr.plan_id = p.id
-    session.execute(
+              AND p.property_id = ANY(:property_ids)
-        delete(PlanRecommendations)
+        """),
-        .where(PlanRecommendations.plan_id.in_(plan_ids))
+        params,
-    )
-
-    session.execute(
-        delete(FundingPackageMeasures)
-        .where(FundingPackageMeasures.funding_package_id.in_(funding_package_ids))
-    )
-
-    session.execute(
-        delete(InspectionModel)
-        .where(InspectionModel.property_id.in_(property_ids))
     )
 
     # --------------------------------------------------
-    # Mid-level tables
+    # funding_package_measures
     # --------------------------------------------------
     session.execute(
-        delete(FundingPackage)
+        text("""
-        .where(FundingPackage.id.in_(funding_package_ids))
+            DELETE FROM funding_package_measures fpm
-    )
+            USING funding_package fp, plan p
-
+            WHERE fpm.funding_package_id = fp.id
-    session.execute(
+              AND fp.plan_id = p.id
-        delete(Recommendation)
+              AND p.property_id = ANY(:property_ids)
-        .where(Recommendation.id.in_(recommendation_ids))
+        """),
-    )
+        params,
-
-    session.execute(
-        delete(Plan)
-        .where(Plan.id.in_(plan_ids))
     )
 
     # --------------------------------------------------
-    # Property-scoped tables
+    # inspections (direct)
     # --------------------------------------------------
     session.execute(
-        delete(PropertyDetailsEpcModel)
+        text("""
-        .where(PropertyDetailsEpcModel.property_id.in_(property_ids))
+            DELETE FROM inspections
-    )
+            WHERE property_id = ANY(:property_ids)
-
+        """),
-    session.execute(
+        params,
-        delete(PropertyTargetsModel)
-        .where(PropertyTargetsModel.property_id.in_(property_ids))
     )
 
     # --------------------------------------------------
-    # Properties LAST
+    # funding_package
     # --------------------------------------------------
     session.execute(
-        delete(PropertyModel)
+        text("""
-        .where(PropertyModel.id.in_(property_ids))
+            DELETE FROM funding_package fp
+            USING plan p
+            WHERE fp.plan_id = p.id
+              AND p.property_id = ANY(:property_ids)
+        """),
+        params,
+    )
+
+    # --------------------------------------------------
+    # recommendation (direct — CRITICAL FIX)
+    # --------------------------------------------------
+    session.execute(
+        text("""
+            DELETE FROM recommendation
+            WHERE property_id = ANY(:property_ids)
+        """),
+        params,
+    )
+
+    # --------------------------------------------------
+    # plan (direct)
+    # --------------------------------------------------
+    session.execute(
+        text("""
+            DELETE FROM plan
+            WHERE property_id = ANY(:property_ids)
+        """),
+        params,
+    )
+
+    # --------------------------------------------------
+    # property-scoped tables
+    # --------------------------------------------------
+    session.execute(
+        text("""
+            DELETE FROM property_details_epc
+            WHERE property_id = ANY(:property_ids)
+        """),
+        params,
+    )
+
+    session.execute(
+        text("""
+            DELETE FROM property_targets
+            WHERE property_id = ANY(:property_ids)
+        """),
+        params,
+    )
+
+    # --------------------------------------------------
+    # properties LAST
+    # --------------------------------------------------
+    session.execute(
+        text("""
+            DELETE FROM property
+            WHERE id = ANY(:property_ids)
+        """),
+        params,
     )
 
 
@@ -481,11 +520,14 @@ def clear_portfolio_in_batches(
         return
 
     total = (len(property_ids) + property_batch_size - 1) // property_batch_size
-
+    import time
     for i, batch in enumerate(chunked(property_ids, property_batch_size), start=1):
         print(f"Deleting batch {i}/{total} ({len(batch)} properties)")
+        start_time = time.time()
         with db_session() as session:
             delete_property_batch(session, batch)
+        finish_time = time.time()
+        print(f"Batch {i} deleted in {finish_time - start_time:.2f} seconds")
 
     # scenario deletion happens AFTER all properties are gone
     delete_portfolio_scenarios_if_empty(portfolio_id)

etl/epc_clean/epc_attributes/HotWaterAttributes.py

@@ -100,6 +100,7 @@ class HotWaterAttributes(Definitions):
     WELSH_TEXT = {
         "ogçör brif system": "from main system",
         "o r brif system": "from main system",
+        "o’r brif system": "from main system",
         "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery",
         "bwyler/cylchredydd nwy": "gas boiler/circulator",
         "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat",

etl/epc_clean/epc_attributes/WallAttributes.py

@@ -39,6 +39,8 @@ class WallAttributes(Definitions):
         "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, "
                                                                                       "insulated (assumed)",
         "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Cavity wall, as built, insulated",
+        "Waliau ceudod, fel y’u hadeiladwyd, wedi’u hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, insulated ("
+                                                                                  "assumed)",
         "Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Granite or whinstone, "
                                                                                            "as built, no insulation ("
                                                                                            "assumed)",

recommendations/WallRecommendations.py

@@ -72,6 +72,7 @@ class WallRecommendations(Definitions):
         'Timber frame, as built, partial insulation': 'Timber frame, with external insulation',
         "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with external insulation",
         "Sandstone, as built, no insulation": "Sandstone, with external insulation",
+        "Sandstone, as built, partial insulation": "Sandstone, with external insulation",
     }
 
     # These are the ending descriptions we consider for walls with internal insulation
@@ -88,6 +89,7 @@ class WallRecommendations(Definitions):
         'Timber frame, as built, partial insulation': 'Timber frame, with internal insulation',
         "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with internal insulation",
         "Sandstone, as built, no insulation": "Sandstone, with internal insulation",
+        "Sandstone, as built, partial insulation": "Sandstone, with internal insulation",
     }
 
     def __init__(