diff --git a/backend/app/db/functions/epc_functions.py b/backend/app/db/functions/epc_functions.py index 27fb9da4..defc24c9 100644 --- a/backend/app/db/functions/epc_functions.py +++ b/backend/app/db/functions/epc_functions.py @@ -1,8 +1,9 @@ from typing import List from datetime import datetime, timedelta, timezone -from sqlalchemy.orm import Session from sqlalchemy.exc import SQLAlchemyError from backend.app.db.models.epc import EpcStore +from sqlmodel import Session +from sqlalchemy.dialects.postgresql import insert class EpcStoreService: @@ -191,3 +192,38 @@ class EpcStoreService: except SQLAlchemyError as e: raise e + + @classmethod + def bulk_upsert_epc_data(cls, session: Session, rows_to_insert: list[dict]): + if not rows_to_insert: + return + + now = datetime.now(timezone.utc) + + values = [ + { + "uprn": row["uprn"], + "epc_api": row["epc_api"], + "epc_api_created_at": now, + "epc_page": row["epc_page"], + "epc_page_rrn": row["epc_page_rrn"], + "epc_page_created_at": now if row["epc_page"] else None, + } + for row in rows_to_insert + ] + + insert_stmt = insert(EpcStore).values(values) + + stmt = insert_stmt.on_conflict_do_update( + index_elements=["uprn"], + set_={ + "epc_api": insert_stmt.excluded.epc_api, + "epc_api_created_at": insert_stmt.excluded.epc_api_created_at, + "epc_page": insert_stmt.excluded.epc_page, + "epc_page_rrn": insert_stmt.excluded.epc_page_rrn, + "epc_page_created_at": insert_stmt.excluded.epc_page_created_at, + }, + ) + + session.execute(stmt) + session.commit() diff --git a/backend/engine/engine.py b/backend/engine/engine.py index c19cf28f..cf147fcd 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -716,7 +716,7 @@ async def model_engine(body: PlanTriggerRequest): if landlord_property_id: property_lookup[("landlord_property_id", landlord_property_id)] = prop_id - input_properties, inspections_map, eco_packages = [], {}, {} + input_properties, inspections_map, eco_packages, epc_upserts = [], {}, {}, [] for addr, config in tqdm( zip(addresses, plan_input), total=len(addresses), @@ -838,25 +838,25 @@ async def model_engine(body: PlanTriggerRequest): # 2) A real EPC # 3) A UPRN (meaning that a UPRN could be fetched against that property) # We store this data - # TODO: Upload in bulk - with db_session() as session: - if db_funcs.epc_functions.EpcStoreService.check_insert_needed( - epc_cache, epc_searcher.newest_epc.get("estimated"), epc_searcher.uprn - ): - # We store the EPC data we have found for this property - db_funcs.epc_functions.EpcStoreService.upsert_epc_data( - session=session, - uprn=epc_searcher.uprn, - epc_api=epc_searcher.data, - epc_page=epc_page_source.get("page_source"), - epc_page_rrn=epc_page_source.get("rrn"), - ) + if db_funcs.epc_functions.EpcStoreService.check_insert_needed( + epc_cache, epc_searcher.newest_epc.get("estimated"), epc_searcher.uprn, + ): + epc_upserts.append({ + "uprn": epc_searcher.uprn, + "epc_api": epc_searcher.data, + "epc_page": epc_page_source.get("page_source"), + "epc_page_rrn": epc_page_source.get("rrn"), + }) if not input_properties: return Response(status_code=204) check_duplicate_property_ids(input_properties) + # We now bulk upload all of the EPC data + with db_session() as session: + db_funcs.epc_functions.EpcStoreService.bulk_upsert_epc_data(session, epc_upserts) + # We check if we have inspections data and store it in the database if so. We'll update or create # aginst each property if if inspections_map: