put in some placeholder code for fast deletes

This commit is contained in:
Khalim Conn-Kowlessar 2025-11-30 21:40:10 +00:00
parent ce75f5e9b4
commit b84dc3a6a8

View file

@ -1,5 +1,5 @@
from tqdm import tqdm
from sqlalchemy import insert, delete
from sqlalchemy import insert, delete, text
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from backend.app.db.models.recommendations import (
@ -175,6 +175,34 @@ def chunked(iterable, size=100):
yield iterable[i:i + size]
def fast_delete_recommendations(session, chunk):
values = ",".join(f"({pid})" for pid in chunk)
sql = text(f"""
WITH ids(property_id) AS (
VALUES {values}
)
DELETE FROM recommendation r
USING ids
WHERE r.property_id = ids.property_id;
""")
session.execute(sql)
# Note; we may be able to go even faster like this:
# def delete_with_temp_table(session, chunk):
# session.execute(text("CREATE TEMP TABLE tmp_ids (id bigint) ON COMMIT DROP;"))
#
# insert_sql = "INSERT INTO tmp_ids (id) VALUES " + ",".join(f"({i})" for i in chunk)
# session.execute(text(insert_sql))
#
# session.execute(text("""
# DELETE FROM recommendation r
# USING tmp_ids t
# WHERE r.property_id = t.id;
# """))
def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
# --------------------------
# Collect IDs up-front
@ -252,14 +280,11 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
tqdm.write("Deleting Scenarios…")
session.execute(delete(Scenario).where(Scenario.portfolio_id == portfolio_id))
# Recommendations
# Recommendations - fast delete
for chunk in tqdm(chunked(property_ids, batch_size),
total=(len(property_ids) // batch_size) + 1,
desc="Deleting Recommendations"):
session.execute(
delete(Recommendation)
.where(Recommendation.property_id.in_(chunk))
)
fast_delete_recommendations(session, chunk)
# Inspections
for chunk in tqdm(chunked(property_ids, batch_size),