diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 0a3b7164..51ff59c5 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -176,18 +176,18 @@ def chunked(iterable, size=100): def fast_delete_recommendations(session, chunk): - values = ",".join(f"({pid})" for pid in chunk) + placeholders = ",".join([f"({i})" for i in range(len(chunk))]) sql = text(f""" - WITH ids(property_id) AS ( - VALUES {values} - ) - DELETE FROM recommendation r - USING ids - WHERE r.property_id = ids.property_id; - """) + WITH ids(property_id) AS ( + VALUES {placeholders} + ) + DELETE FROM recommendation r + USING ids + WHERE r.property_id = ids.property_id; + """) - session.execute(sql) + session.execute(sql, execution_options={"synchronize_session": False}) # Note; we may be able to go even faster like this: # def delete_with_temp_table(session, chunk): @@ -204,6 +204,9 @@ def fast_delete_recommendations(session, chunk): def clear_portfolio(session: Session, portfolio_id: int, batch_size=100): + def print_progress(prefix, i, total): + print(f"{prefix} ({i}/{total})") + # -------------------------- # Collect IDs up-front # -------------------------- @@ -227,95 +230,130 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100): .filter(FundingPackage.plan_id.in_(plan_ids)) ] - # -------------------------- - # Batch deletes with tqdm - # -------------------------- + # ========== BATCH HELPERS ========== + def chunked(lst, n): + for i in range(0, len(lst), n): + yield lst[i:i + n] - # RecommendationMaterials - for chunk in tqdm(chunked(recommendation_ids, batch_size), - total=(len(recommendation_ids) // batch_size) + 1, - desc="Deleting RecommendationMaterials"): + # -------------------------- + # Deleting RecommendationMaterials + # -------------------------- + rm_chunks = list(chunked(recommendation_ids, batch_size)) + total = len(rm_chunks) + for i, chunk in enumerate(rm_chunks, start=1): + print_progress("Deleting RecommendationMaterials", i, total) session.execute( delete(RecommendationMaterials) .where(RecommendationMaterials.recommendation_id.in_(chunk)) ) + # -------------------------- # PlanRecommendations - for chunk in tqdm(chunked(plan_ids, batch_size), - total=(len(plan_ids) // batch_size) + 1, - desc="Deleting PlanRecommendations"): + # -------------------------- + pr_chunks = list(chunked(plan_ids, batch_size)) + total = len(pr_chunks) + for i, chunk in enumerate(pr_chunks, start=1): + print_progress("Deleting PlanRecommendations", i, total) session.execute( delete(PlanRecommendations) .where(PlanRecommendations.plan_id.in_(chunk)) ) + # -------------------------- # FundingPackageMeasures - for chunk in tqdm(chunked(funding_package_ids, batch_size), - total=(len(funding_package_ids) // batch_size) + 1, - desc="Deleting FundingPackageMeasures"): + # -------------------------- + fpm_chunks = list(chunked(funding_package_ids, batch_size)) + total = len(fpm_chunks) + for i, chunk in enumerate(fpm_chunks, start=1): + print_progress("Deleting FundingPackageMeasures", i, total) session.execute( delete(FundingPackageMeasures) .where(FundingPackageMeasures.funding_package_id.in_(chunk)) ) - # FundingPackage - for chunk in tqdm(chunked(plan_ids, batch_size), - total=(len(plan_ids) // batch_size) + 1, - desc="Deleting FundingPackages"): + # -------------------------- + # FundingPackages + # -------------------------- + fp_chunks = list(chunked(plan_ids, batch_size)) + total = len(fp_chunks) + for i, chunk in enumerate(fp_chunks, start=1): + print_progress("Deleting FundingPackages", i, total) session.execute( delete(FundingPackage) .where(FundingPackage.plan_id.in_(chunk)) ) + # -------------------------- # Plans - for chunk in tqdm(chunked(plan_ids, batch_size), - total=(len(plan_ids) // batch_size) + 1, - desc="Deleting Plans"): + # -------------------------- + plan_chunks = list(chunked(plan_ids, batch_size)) + total = len(plan_chunks) + for i, chunk in enumerate(plan_chunks, start=1): + print_progress("Deleting Plans", i, total) session.execute( delete(Plan) .where(Plan.id.in_(chunk)) ) - # Scenarios (no chunks needed) - tqdm.write("Deleting Scenarios…") - session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id)) + # -------------------------- + # Scenarios + # -------------------------- + print("Deleting Scenarios…") + session.execute( + delete(Scenario) + .where(Scenario.portfolio_id == portfolio_id) + ) - # Recommendations - fast delete - for chunk in tqdm(chunked(property_ids, batch_size), - total=(len(property_ids) // batch_size) + 1, - desc="Deleting Recommendations"): + # -------------------------- + # Recommendations (fast delete) + # -------------------------- + rec_chunks = list(chunked(property_ids, batch_size)) + total = len(rec_chunks) + for i, chunk in enumerate(rec_chunks, start=1): + print_progress("Deleting Recommendations", i, total) fast_delete_recommendations(session, chunk) + # -------------------------- # Inspections - for chunk in tqdm(chunked(property_ids, batch_size), - total=(len(property_ids) // batch_size) + 1, - desc="Deleting Inspections"): + # -------------------------- + insp_chunks = list(chunked(property_ids, batch_size)) + total = len(insp_chunks) + for i, chunk in enumerate(insp_chunks, start=1): + print_progress("Deleting Inspections", i, total) session.execute( delete(InspectionModel) .where(InspectionModel.property_id.in_(chunk)) ) - # Property-related detail tables - tqdm.write("Deleting PropertyTargetsModel…") + # -------------------------- + # PropertyTargetsModel + # -------------------------- + print("Deleting PropertyTargetsModel…") session.execute( delete(PropertyTargetsModel) .where(PropertyTargetsModel.portfolio_id == portfolio_id) ) - tqdm.write("Deleting PropertyDetailsEpcModel…") + # -------------------------- + # PropertyDetailsEpcModel + # -------------------------- + print("Deleting PropertyDetailsEpcModel…") session.execute( delete(PropertyDetailsEpcModel) .where(PropertyDetailsEpcModel.portfolio_id == portfolio_id) ) + # -------------------------- # Properties - for chunk in tqdm(chunked(property_ids, batch_size), - total=(len(property_ids) // batch_size) + 1, - desc="Deleting Properties"): + # -------------------------- + prop_chunks = list(chunked(property_ids, batch_size)) + total = len(prop_chunks) + for i, chunk in enumerate(prop_chunks, start=1): + print_progress("Deleting Properties", i, total) session.execute( delete(PropertyModel) .where(PropertyModel.id.in_(chunk)) ) session.commit() - tqdm.write("Portfolio cleared.") + print("Portfolio cleared.")