From 5a9e018b1989cbd36db41a3b7044414b958d2a32 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 30 Nov 2025 18:57:13 +0000 Subject: [PATCH] fixed ashp non invasic rec bug --- .../db/functions/recommendations_functions.py | 215 ++++++++++++++---- backend/engine/engine.py | 2 +- recommendations/HeatingRecommender.py | 4 +- 3 files changed, 174 insertions(+), 47 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 8c6e710a..2d444544 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -1,3 +1,4 @@ +from tqdm import tqdm from sqlalchemy import insert, delete from sqlalchemy.orm import Session from sqlalchemy.exc import SQLAlchemyError @@ -169,62 +170,188 @@ def upload_recommendations(session: Session, recommendations_to_upload, property return False -def clear_portfolio(session: Session, portfolio_id: int): - # Fetch all property IDs associated with the given portfolio - property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all() - property_ids = [p.id for p in property_ids] +# def clear_portfolio(session: Session, portfolio_id: int): +# # Fetch all property IDs associated with the given portfolio +# property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all() +# property_ids = [p.id for p in property_ids] +# +# # Fetch all recommendation IDs associated with the properties +# recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all() +# recommendation_ids = [r.id for r in recommendation_ids] +# +# # Fetch all plan IDs associated with the portfolio +# plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all() +# plan_ids = [p.id for p in plan_ids] +# +# # Delete all entries from RecommendationMaterials for these recommendations +# session.execute( +# delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) +# ) +# +# # Delete all entries from PlanRecommendations that reference plans in the portfolio +# session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_( +# session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar() +# ))) +# +# # Delete FundingPackageMeasures → FundingPackage → Plan +# session.execute( +# delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_( +# session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids)) +# )) +# ) +# session.execute( +# delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids)) +# ) +# +# # Delete all Plans associated with the portfolio +# session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id)) +# +# # Delete all Scenarios associated with the portfolio +# session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id)) +# +# # Delete all Recommendations associated with the properties +# session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids))) +# +# session.execute( +# delete(InspectionModel) +# .where(InspectionModel.property_id.in_( +# session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id) +# )) +# .execution_options(synchronize_session=False) +# ) +# +# # Now, delete the PropertyModels and related details +# # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel +# session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id)) +# # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids))) +# session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)) +# session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id)) +# +# # Commit the changes +# session.commit() - # Fetch all recommendation IDs associated with the properties - recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all() - recommendation_ids = [r.id for r in recommendation_ids] - # Fetch all plan IDs associated with the portfolio - plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all() - plan_ids = [p.id for p in plan_ids] +def chunked(iterable, size=500): + for i in range(0, len(iterable), size): + yield iterable[i:i + size] - # Delete all entries from RecommendationMaterials for these recommendations - session.execute( - delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) - ) - # Delete all entries from PlanRecommendations that reference plans in the portfolio - session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_( - session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar() - ))) +def clear_portfolio(session: Session, portfolio_id: int, batch_size=500): + # -------------------------- + # Collect IDs up-front + # -------------------------- + property_ids = [ + p.id for p in session.query(PropertyModel.id) + .filter(PropertyModel.portfolio_id == portfolio_id) + ] - # Delete FundingPackageMeasures → FundingPackage → Plan - session.execute( - delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_( - session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids)) - )) - ) - session.execute( - delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids)) - ) + recommendation_ids = [ + r.id for r in session.query(Recommendation.id) + .filter(Recommendation.property_id.in_(property_ids)) + ] - # Delete all Plans associated with the portfolio - session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id)) + plan_ids = [ + p.id for p in session.query(Plan.id) + .filter(Plan.portfolio_id == portfolio_id) + ] - # Delete all Scenarios associated with the portfolio + funding_package_ids = [ + fp.id for fp in session.query(FundingPackage.id) + .filter(FundingPackage.plan_id.in_(plan_ids)) + ] + + # -------------------------- + # Batch deletes with tqdm + # -------------------------- + + # RecommendationMaterials + for chunk in tqdm(chunked(recommendation_ids, batch_size), + total=(len(recommendation_ids) // batch_size) + 1, + desc="Deleting RecommendationMaterials"): + session.execute( + delete(RecommendationMaterials) + .where(RecommendationMaterials.recommendation_id.in_(chunk)) + ) + + # PlanRecommendations + for chunk in tqdm(chunked(plan_ids, batch_size), + total=(len(plan_ids) // batch_size) + 1, + desc="Deleting PlanRecommendations"): + session.execute( + delete(PlanRecommendations) + .where(PlanRecommendations.plan_id.in_(chunk)) + ) + + # FundingPackageMeasures + for chunk in tqdm(chunked(funding_package_ids, batch_size), + total=(len(funding_package_ids) // batch_size) + 1, + desc="Deleting FundingPackageMeasures"): + session.execute( + delete(FundingPackageMeasures) + .where(FundingPackageMeasures.funding_package_id.in_(chunk)) + ) + + # FundingPackage + for chunk in tqdm(chunked(plan_ids, batch_size), + total=(len(plan_ids) // batch_size) + 1, + desc="Deleting FundingPackages"): + session.execute( + delete(FundingPackage) + .where(FundingPackage.plan_id.in_(chunk)) + ) + + # Plans + for chunk in tqdm(chunked(plan_ids, batch_size), + total=(len(plan_ids) // batch_size) + 1, + desc="Deleting Plans"): + session.execute( + delete(Plan) + .where(Plan.id.in_(chunk)) + ) + + # Scenarios (no chunks needed) + tqdm.write("Deleting Scenarios…") session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id)) - # Delete all Recommendations associated with the properties - session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids))) + # Recommendations + for chunk in tqdm(chunked(property_ids, batch_size), + total=(len(property_ids) // batch_size) + 1, + desc="Deleting Recommendations"): + session.execute( + delete(Recommendation) + .where(Recommendation.property_id.in_(chunk)) + ) + # Inspections + for chunk in tqdm(chunked(property_ids, batch_size), + total=(len(property_ids) // batch_size) + 1, + desc="Deleting Inspections"): + session.execute( + delete(InspectionModel) + .where(InspectionModel.property_id.in_(chunk)) + ) + + # Property-related detail tables + tqdm.write("Deleting PropertyTargetsModel…") session.execute( - delete(InspectionModel) - .where(InspectionModel.property_id.in_( - session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id) - )) - .execution_options(synchronize_session=False) + delete(PropertyTargetsModel) + .where(PropertyTargetsModel.portfolio_id == portfolio_id) ) - # Now, delete the PropertyModels and related details - # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel - session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id)) - # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids))) - session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)) - session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id)) + tqdm.write("Deleting PropertyDetailsEpcModel…") + session.execute( + delete(PropertyDetailsEpcModel) + .where(PropertyDetailsEpcModel.portfolio_id == portfolio_id) + ) + + # Properties + for chunk in tqdm(chunked(property_ids, batch_size), + total=(len(property_ids) // batch_size) + 1, + desc="Deleting Properties"): + session.execute( + delete(PropertyModel) + .where(PropertyModel.id.in_(chunk)) + ) - # Commit the changes session.commit() + tqdm.write("Portfolio cleared.") diff --git a/backend/engine/engine.py b/backend/engine/engine.py index c0261e57..bf533117 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -612,7 +612,7 @@ async def model_engine(body: PlanTriggerRequest): address1 = config.get("domna_address_1", None) address1 = str(int(address1)) if isinstance(address1, float) else str(address1) - full_address = config["domna_full_address"] if body.file_format == "domna_asset_list" else None + full_address = config.get("domna_full_address") if body.file_format == "domna_asset_list" else None heating_system = parse_heating_system(config) associated_uprns = [] diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index fdd4376d..c5aa8b38 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -321,7 +321,7 @@ class HeatingRecommender: # if we have a non-invasive ashp recommendation, we get the configuration directly from the property instance non_invasive_ashp_recommendation = next( (r for r in self.property.non_invasive_recommendations if r["type"] == "air_source_heat_pump"), - {"suitable": True} + {"survey": False} ) # We allow for the non-invasive recommendation to be that ASHP is not suitable @@ -366,7 +366,7 @@ class HeatingRecommender: if ( self.property.is_ashp_valid(measures=measures) and - non_invasive_ashp_recommendation["suitable"] and + len(non_invasive_ashp_recommendation) and not self.has_ashp and not self.has_gshp ): self.recommend_air_source_heat_pump(