Merge pull request #578 from Hestia-Homes/main

fixed ashp non invasive recommendation bug

backend/app/db/functions/recommendations_functions.py

@@ -1,3 +1,4 @@
+from tqdm import tqdm
 from sqlalchemy import insert, delete
 from sqlalchemy.orm import Session
 from sqlalchemy.exc import SQLAlchemyError
@@ -169,62 +170,188 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
         return False
 
 
-def clear_portfolio(session: Session, portfolio_id: int):
+# def clear_portfolio(session: Session, portfolio_id: int):
-    # Fetch all property IDs associated with the given portfolio
+#     # Fetch all property IDs associated with the given portfolio
-    property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all()
+#     property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all()
-    property_ids = [p.id for p in property_ids]
+#     property_ids = [p.id for p in property_ids]
+#
+#     # Fetch all recommendation IDs associated with the properties
+#     recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all()
+#     recommendation_ids = [r.id for r in recommendation_ids]
+#
+#     # Fetch all plan IDs associated with the portfolio
+#     plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all()
+#     plan_ids = [p.id for p in plan_ids]
+#
+#     # Delete all entries from RecommendationMaterials for these recommendations
+#     session.execute(
+#         delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
+#     )
+#
+#     # Delete all entries from PlanRecommendations that reference plans in the portfolio
+#     session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_(
+#         session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar()
+#     )))
+#
+#     # Delete FundingPackageMeasures → FundingPackage → Plan
+#     session.execute(
+#         delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_(
+#             session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids))
+#         ))
+#     )
+#     session.execute(
+#         delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids))
+#     )
+#
+#     # Delete all Plans associated with the portfolio
+#     session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id))
+#
+#     # Delete all Scenarios associated with the portfolio
+#     session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id))
+#
+#     # Delete all Recommendations associated with the properties
+#     session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids)))
+#
+#     session.execute(
+#         delete(InspectionModel)
+#         .where(InspectionModel.property_id.in_(
+#             session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id)
+#         ))
+#         .execution_options(synchronize_session=False)
+#     )
+#
+#     # Now, delete the PropertyModels and related details
+#     # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel
+#     session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id))
+#     # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids)))
+#     session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id))
+#     session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id))
+#
+#     # Commit the changes
+#     session.commit()
 
-    # Fetch all recommendation IDs associated with the properties
-    recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all()
-    recommendation_ids = [r.id for r in recommendation_ids]
 
-    # Fetch all plan IDs associated with the portfolio
+def chunked(iterable, size=100):
-    plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all()
+    for i in range(0, len(iterable), size):
-    plan_ids = [p.id for p in plan_ids]
+        yield iterable[i:i + size]
 
-    # Delete all entries from RecommendationMaterials for these recommendations
-    session.execute(
-        delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
-    )
 
-    # Delete all entries from PlanRecommendations that reference plans in the portfolio
+def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
-    session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_(
+    # --------------------------
-        session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar()
+    # Collect IDs up-front
-    )))
+    # --------------------------
+    property_ids = [
+        p.id for p in session.query(PropertyModel.id)
+        .filter(PropertyModel.portfolio_id == portfolio_id)
+    ]
 
-    # Delete FundingPackageMeasures → FundingPackage → Plan
+    recommendation_ids = [
-    session.execute(
+        r.id for r in session.query(Recommendation.id)
-        delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_(
+        .filter(Recommendation.property_id.in_(property_ids))
-            session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids))
+    ]
-        ))
-    )
-    session.execute(
-        delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids))
-    )
 
-    # Delete all Plans associated with the portfolio
+    plan_ids = [
-    session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id))
+        p.id for p in session.query(Plan.id)
+        .filter(Plan.portfolio_id == portfolio_id)
+    ]
 
-    # Delete all Scenarios associated with the portfolio
+    funding_package_ids = [
+        fp.id for fp in session.query(FundingPackage.id)
+        .filter(FundingPackage.plan_id.in_(plan_ids))
+    ]
+
+    # --------------------------
+    # Batch deletes with tqdm
+    # --------------------------
+
+    # RecommendationMaterials
+    for chunk in tqdm(chunked(recommendation_ids, batch_size),
+                      total=(len(recommendation_ids) // batch_size) + 1,
+                      desc="Deleting RecommendationMaterials"):
+        session.execute(
+            delete(RecommendationMaterials)
+            .where(RecommendationMaterials.recommendation_id.in_(chunk))
+        )
+
+    # PlanRecommendations
+    for chunk in tqdm(chunked(plan_ids, batch_size),
+                      total=(len(plan_ids) // batch_size) + 1,
+                      desc="Deleting PlanRecommendations"):
+        session.execute(
+            delete(PlanRecommendations)
+            .where(PlanRecommendations.plan_id.in_(chunk))
+        )
+
+    # FundingPackageMeasures
+    for chunk in tqdm(chunked(funding_package_ids, batch_size),
+                      total=(len(funding_package_ids) // batch_size) + 1,
+                      desc="Deleting FundingPackageMeasures"):
+        session.execute(
+            delete(FundingPackageMeasures)
+            .where(FundingPackageMeasures.funding_package_id.in_(chunk))
+        )
+
+    # FundingPackage
+    for chunk in tqdm(chunked(plan_ids, batch_size),
+                      total=(len(plan_ids) // batch_size) + 1,
+                      desc="Deleting FundingPackages"):
+        session.execute(
+            delete(FundingPackage)
+            .where(FundingPackage.plan_id.in_(chunk))
+        )
+
+    # Plans
+    for chunk in tqdm(chunked(plan_ids, batch_size),
+                      total=(len(plan_ids) // batch_size) + 1,
+                      desc="Deleting Plans"):
+        session.execute(
+            delete(Plan)
+            .where(Plan.id.in_(chunk))
+        )
+
+    # Scenarios (no chunks needed)
+    tqdm.write("Deleting Scenarios…")
     session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id))
 
-    # Delete all Recommendations associated with the properties
+    # Recommendations
-    session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids)))
+    for chunk in tqdm(chunked(property_ids, batch_size),
+                      total=(len(property_ids) // batch_size) + 1,
+                      desc="Deleting Recommendations"):
+        session.execute(
+            delete(Recommendation)
+            .where(Recommendation.property_id.in_(chunk))
+        )
 
+    # Inspections
+    for chunk in tqdm(chunked(property_ids, batch_size),
+                      total=(len(property_ids) // batch_size) + 1,
+                      desc="Deleting Inspections"):
+        session.execute(
+            delete(InspectionModel)
+            .where(InspectionModel.property_id.in_(chunk))
+        )
+
+    # Property-related detail tables
+    tqdm.write("Deleting PropertyTargetsModel…")
     session.execute(
-        delete(InspectionModel)
+        delete(PropertyTargetsModel)
-        .where(InspectionModel.property_id.in_(
+        .where(PropertyTargetsModel.portfolio_id == portfolio_id)
-            session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id)
-        ))
-        .execution_options(synchronize_session=False)
     )
 
-    # Now, delete the PropertyModels and related details
+    tqdm.write("Deleting PropertyDetailsEpcModel…")
-    # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel
+    session.execute(
-    session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id))
+        delete(PropertyDetailsEpcModel)
-    # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids)))
+        .where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)
-    session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id))
+    )
-    session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id))
+
+    # Properties
+    for chunk in tqdm(chunked(property_ids, batch_size),
+                      total=(len(property_ids) // batch_size) + 1,
+                      desc="Deleting Properties"):
+        session.execute(
+            delete(PropertyModel)
+            .where(PropertyModel.id.in_(chunk))
+        )
 
-    # Commit the changes
     session.commit()
+    tqdm.write("Portfolio cleared.")

backend/engine/engine.py

@@ -612,7 +612,7 @@ async def model_engine(body: PlanTriggerRequest):
                 address1 = config.get("domna_address_1", None)
 
             address1 = str(int(address1)) if isinstance(address1, float) else str(address1)
-            full_address = config["domna_full_address"] if body.file_format == "domna_asset_list" else None
+            full_address = config.get("domna_full_address") if body.file_format == "domna_asset_list" else None
             heating_system = parse_heating_system(config)
 
             associated_uprns = []

recommendations/HeatingRecommender.py

@@ -321,7 +321,7 @@ class HeatingRecommender:
         # if we have a non-invasive ashp recommendation, we get the configuration directly from the property instance
         non_invasive_ashp_recommendation = next(
             (r for r in self.property.non_invasive_recommendations if r["type"] == "air_source_heat_pump"),
-            {"suitable": True}
+            {"survey": False}
         )
         # We allow for the non-invasive recommendation to be that ASHP is not suitable
 
@@ -366,7 +366,7 @@ class HeatingRecommender:
 
         if (
             self.property.is_ashp_valid(measures=measures) and
-            non_invasive_ashp_recommendation["suitable"] and
+            len(non_invasive_ashp_recommendation) and
             not self.has_ashp and not self.has_gshp
         ):
             self.recommend_air_source_heat_pump(