junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

removing tqdm

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-11-30 22:01:03 +00:00
parent b84dc3a6a8
commit 6c8b65f3fb
1 changed files with 84 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
backend/app/db/functions/recommendations_functions.py
View file

@ -176,18 +176,18 @@ def chunked(iterable, size=100):
def fast_delete_recommendations(session, chunk):
values = ",".join(f"({pid})" for pid in chunk)
placeholders = ",".join([f"({i})" for i in range(len(chunk))])
sql = text(f"""
WITH ids(property_id) AS (
VALUES {values}
)
DELETE FROM recommendation r
USING ids
WHERE r.property_id = ids.property_id;
""")
WITH ids(property_id) AS (
VALUES {placeholders}
)
DELETE FROM recommendation r
USING ids
WHERE r.property_id = ids.property_id;
""")
session.execute(sql)
session.execute(sql, execution_options={"synchronize_session": False})
# Note; we may be able to go even faster like this:
# def delete_with_temp_table(session, chunk):
@ -204,6 +204,9 @@ def fast_delete_recommendations(session, chunk):
def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
def print_progress(prefix, i, total):
print(f"{prefix} ({i}/{total})")
# --------------------------
# Collect IDs up-front
# --------------------------
@ -227,95 +230,130 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
.filter(FundingPackage.plan_id.in_(plan_ids))
]
# --------------------------
# Batch deletes with tqdm
# --------------------------
# ========== BATCH HELPERS ==========
def chunked(lst, n):
for i in range(0, len(lst), n):
yield lst[i:i + n]
# RecommendationMaterials
for chunk in tqdm(chunked(recommendation_ids, batch_size),
total=(len(recommendation_ids) // batch_size) + 1,
desc="Deleting RecommendationMaterials"):
# --------------------------
# Deleting RecommendationMaterials
# --------------------------
rm_chunks = list(chunked(recommendation_ids, batch_size))
total = len(rm_chunks)
for i, chunk in enumerate(rm_chunks, start=1):
print_progress("Deleting RecommendationMaterials", i, total)
session.execute(
delete(RecommendationMaterials)
.where(RecommendationMaterials.recommendation_id.in_(chunk))
)
# --------------------------
# PlanRecommendations
for chunk in tqdm(chunked(plan_ids, batch_size),
total=(len(plan_ids) // batch_size) + 1,
desc="Deleting PlanRecommendations"):
# --------------------------
pr_chunks = list(chunked(plan_ids, batch_size))
total = len(pr_chunks)
for i, chunk in enumerate(pr_chunks, start=1):
print_progress("Deleting PlanRecommendations", i, total)
session.execute(
delete(PlanRecommendations)
.where(PlanRecommendations.plan_id.in_(chunk))
)
# --------------------------
# FundingPackageMeasures
for chunk in tqdm(chunked(funding_package_ids, batch_size),
total=(len(funding_package_ids) // batch_size) + 1,
desc="Deleting FundingPackageMeasures"):
# --------------------------
fpm_chunks = list(chunked(funding_package_ids, batch_size))
total = len(fpm_chunks)
for i, chunk in enumerate(fpm_chunks, start=1):
print_progress("Deleting FundingPackageMeasures", i, total)
session.execute(
delete(FundingPackageMeasures)
.where(FundingPackageMeasures.funding_package_id.in_(chunk))
)
# FundingPackage
for chunk in tqdm(chunked(plan_ids, batch_size),
total=(len(plan_ids) // batch_size) + 1,
desc="Deleting FundingPackages"):
# --------------------------
# FundingPackages
# --------------------------
fp_chunks = list(chunked(plan_ids, batch_size))
total = len(fp_chunks)
for i, chunk in enumerate(fp_chunks, start=1):
print_progress("Deleting FundingPackages", i, total)
session.execute(
delete(FundingPackage)
.where(FundingPackage.plan_id.in_(chunk))
)
# --------------------------
# Plans
for chunk in tqdm(chunked(plan_ids, batch_size),
total=(len(plan_ids) // batch_size) + 1,
desc="Deleting Plans"):
# --------------------------
plan_chunks = list(chunked(plan_ids, batch_size))
total = len(plan_chunks)
for i, chunk in enumerate(plan_chunks, start=1):
print_progress("Deleting Plans", i, total)
session.execute(
delete(Plan)
.where(Plan.id.in_(chunk))
)
# Scenarios (no chunks needed)
tqdm.write("Deleting Scenarios…")
session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id))
# --------------------------
# Scenarios
# --------------------------
print("Deleting Scenarios…")
session.execute(
delete(Scenario)
.where(Scenario.portfolio_id == portfolio_id)
)
# Recommendations - fast delete
for chunk in tqdm(chunked(property_ids, batch_size),
total=(len(property_ids) // batch_size) + 1,
desc="Deleting Recommendations"):
# --------------------------
# Recommendations (fast delete)
# --------------------------
rec_chunks = list(chunked(property_ids, batch_size))
total = len(rec_chunks)
for i, chunk in enumerate(rec_chunks, start=1):
print_progress("Deleting Recommendations", i, total)
fast_delete_recommendations(session, chunk)
# --------------------------
# Inspections
for chunk in tqdm(chunked(property_ids, batch_size),
total=(len(property_ids) // batch_size) + 1,
desc="Deleting Inspections"):
# --------------------------
insp_chunks = list(chunked(property_ids, batch_size))
total = len(insp_chunks)
for i, chunk in enumerate(insp_chunks, start=1):
print_progress("Deleting Inspections", i, total)
session.execute(
delete(InspectionModel)
.where(InspectionModel.property_id.in_(chunk))
)
# Property-related detail tables
tqdm.write("Deleting PropertyTargetsModel…")
# --------------------------
# PropertyTargetsModel
# --------------------------
print("Deleting PropertyTargetsModel…")
session.execute(
delete(PropertyTargetsModel)
.where(PropertyTargetsModel.portfolio_id == portfolio_id)
)
tqdm.write("Deleting PropertyDetailsEpcModel…")
# --------------------------
# PropertyDetailsEpcModel
# --------------------------
print("Deleting PropertyDetailsEpcModel…")
session.execute(
delete(PropertyDetailsEpcModel)
.where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)
)
# --------------------------
# Properties
for chunk in tqdm(chunked(property_ids, batch_size),
total=(len(property_ids) // batch_size) + 1,
desc="Deleting Properties"):
# --------------------------
prop_chunks = list(chunked(property_ids, batch_size))
total = len(prop_chunks)
for i, chunk in enumerate(prop_chunks, start=1):
print_progress("Deleting Properties", i, total)
session.execute(
delete(PropertyModel)
.where(PropertyModel.id.in_(chunk))
)
session.commit()
tqdm.write("Portfolio cleared.")
print("Portfolio cleared.")