junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

removing tqdm

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-11-30 22:01:03 +00:00
parent b84dc3a6a8
commit 6c8b65f3fb
1 changed files with 84 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
backend/app/db/functions/recommendations_functions.py
View file

@ -176,18 +176,18 @@ def chunked(iterable, size=100):
def fast_delete_recommendations(session, chunk): def fast_delete_recommendations(session, chunk):
values = ",".join(f"({pid})" for pid in chunk) placeholders = ",".join([f"({i})" for i in range(len(chunk))])
sql = text(f""" sql = text(f"""
WITH ids(property_id) AS ( WITH ids(property_id) AS (
VALUES {values} VALUES {placeholders}
) )
DELETE FROM recommendation r DELETE FROM recommendation r
USING ids USING ids
WHERE r.property_id = ids.property_id; WHERE r.property_id = ids.property_id;
""") """)
session.execute(sql) session.execute(sql, execution_options={"synchronize_session": False})
# Note; we may be able to go even faster like this: # Note; we may be able to go even faster like this:
# def delete_with_temp_table(session, chunk): # def delete_with_temp_table(session, chunk):
@ -204,6 +204,9 @@ def fast_delete_recommendations(session, chunk):
def clear_portfolio(session: Session, portfolio_id: int, batch_size=100): def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
def print_progress(prefix, i, total):
print(f"{prefix} ({i}/{total})")
# -------------------------- # --------------------------
# Collect IDs up-front # Collect IDs up-front
# -------------------------- # --------------------------
@ -227,95 +230,130 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
.filter(FundingPackage.plan_id.in_(plan_ids)) .filter(FundingPackage.plan_id.in_(plan_ids))
] ]
# -------------------------- # ========== BATCH HELPERS ==========
# Batch deletes with tqdm def chunked(lst, n):
# -------------------------- for i in range(0, len(lst), n):
yield lst[i:i + n]
# RecommendationMaterials # --------------------------
for chunk in tqdm(chunked(recommendation_ids, batch_size), # Deleting RecommendationMaterials
total=(len(recommendation_ids) // batch_size) + 1, # --------------------------
desc="Deleting RecommendationMaterials"): rm_chunks = list(chunked(recommendation_ids, batch_size))
total = len(rm_chunks)
for i, chunk in enumerate(rm_chunks, start=1):
print_progress("Deleting RecommendationMaterials", i, total)
session.execute( session.execute(
delete(RecommendationMaterials) delete(RecommendationMaterials)
.where(RecommendationMaterials.recommendation_id.in_(chunk)) .where(RecommendationMaterials.recommendation_id.in_(chunk))
) )
# --------------------------
# PlanRecommendations # PlanRecommendations
for chunk in tqdm(chunked(plan_ids, batch_size), # --------------------------
total=(len(plan_ids) // batch_size) + 1, pr_chunks = list(chunked(plan_ids, batch_size))
desc="Deleting PlanRecommendations"): total = len(pr_chunks)
for i, chunk in enumerate(pr_chunks, start=1):
print_progress("Deleting PlanRecommendations", i, total)
session.execute( session.execute(
delete(PlanRecommendations) delete(PlanRecommendations)
.where(PlanRecommendations.plan_id.in_(chunk)) .where(PlanRecommendations.plan_id.in_(chunk))
) )
# --------------------------
# FundingPackageMeasures # FundingPackageMeasures
for chunk in tqdm(chunked(funding_package_ids, batch_size), # --------------------------
total=(len(funding_package_ids) // batch_size) + 1, fpm_chunks = list(chunked(funding_package_ids, batch_size))
desc="Deleting FundingPackageMeasures"): total = len(fpm_chunks)
for i, chunk in enumerate(fpm_chunks, start=1):
print_progress("Deleting FundingPackageMeasures", i, total)
session.execute( session.execute(
delete(FundingPackageMeasures) delete(FundingPackageMeasures)
.where(FundingPackageMeasures.funding_package_id.in_(chunk)) .where(FundingPackageMeasures.funding_package_id.in_(chunk))
) )
# FundingPackage # --------------------------
for chunk in tqdm(chunked(plan_ids, batch_size), # FundingPackages
total=(len(plan_ids) // batch_size) + 1, # --------------------------
desc="Deleting FundingPackages"): fp_chunks = list(chunked(plan_ids, batch_size))
total = len(fp_chunks)
for i, chunk in enumerate(fp_chunks, start=1):
print_progress("Deleting FundingPackages", i, total)
session.execute( session.execute(
delete(FundingPackage) delete(FundingPackage)
.where(FundingPackage.plan_id.in_(chunk)) .where(FundingPackage.plan_id.in_(chunk))
) )
# --------------------------
# Plans # Plans
for chunk in tqdm(chunked(plan_ids, batch_size), # --------------------------
total=(len(plan_ids) // batch_size) + 1, plan_chunks = list(chunked(plan_ids, batch_size))
desc="Deleting Plans"): total = len(plan_chunks)
for i, chunk in enumerate(plan_chunks, start=1):
print_progress("Deleting Plans", i, total)
session.execute( session.execute(
delete(Plan) delete(Plan)
.where(Plan.id.in_(chunk)) .where(Plan.id.in_(chunk))
) )
# Scenarios (no chunks needed) # --------------------------
tqdm.write("Deleting Scenarios…") # Scenarios
session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id)) # --------------------------
print("Deleting Scenarios…")
session.execute(
delete(Scenario)
.where(Scenario.portfolio_id == portfolio_id)
)
# Recommendations - fast delete # --------------------------
for chunk in tqdm(chunked(property_ids, batch_size), # Recommendations (fast delete)
total=(len(property_ids) // batch_size) + 1, # --------------------------
desc="Deleting Recommendations"): rec_chunks = list(chunked(property_ids, batch_size))
total = len(rec_chunks)
for i, chunk in enumerate(rec_chunks, start=1):
print_progress("Deleting Recommendations", i, total)
fast_delete_recommendations(session, chunk) fast_delete_recommendations(session, chunk)
# --------------------------
# Inspections # Inspections
for chunk in tqdm(chunked(property_ids, batch_size), # --------------------------
total=(len(property_ids) // batch_size) + 1, insp_chunks = list(chunked(property_ids, batch_size))
desc="Deleting Inspections"): total = len(insp_chunks)
for i, chunk in enumerate(insp_chunks, start=1):
print_progress("Deleting Inspections", i, total)
session.execute( session.execute(
delete(InspectionModel) delete(InspectionModel)
.where(InspectionModel.property_id.in_(chunk)) .where(InspectionModel.property_id.in_(chunk))
) )
# Property-related detail tables # --------------------------
tqdm.write("Deleting PropertyTargetsModel…") # PropertyTargetsModel
# --------------------------
print("Deleting PropertyTargetsModel…")
session.execute( session.execute(
delete(PropertyTargetsModel) delete(PropertyTargetsModel)
.where(PropertyTargetsModel.portfolio_id == portfolio_id) .where(PropertyTargetsModel.portfolio_id == portfolio_id)
) )
tqdm.write("Deleting PropertyDetailsEpcModel…") # --------------------------
# PropertyDetailsEpcModel
# --------------------------
print("Deleting PropertyDetailsEpcModel…")
session.execute( session.execute(
delete(PropertyDetailsEpcModel) delete(PropertyDetailsEpcModel)
.where(PropertyDetailsEpcModel.portfolio_id == portfolio_id) .where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)
) )
# --------------------------
# Properties # Properties
for chunk in tqdm(chunked(property_ids, batch_size), # --------------------------
total=(len(property_ids) // batch_size) + 1, prop_chunks = list(chunked(property_ids, batch_size))
desc="Deleting Properties"): total = len(prop_chunks)
for i, chunk in enumerate(prop_chunks, start=1):
print_progress("Deleting Properties", i, total)
session.execute( session.execute(
delete(PropertyModel) delete(PropertyModel)
.where(PropertyModel.id.in_(chunk)) .where(PropertyModel.id.in_(chunk))
) )
session.commit() session.commit()
tqdm.write("Portfolio cleared.") print("Portfolio cleared.")