junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

implenenting bulk EPC data upload

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-12-21 11:05:01 +08:00
parent 9c34e202bc
commit 9c48bdfbd6
2 changed files with 51 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
backend/app/db/functions/epc_functions.py
View file

@ -1,8 +1,9 @@
from typing import List
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from backend.app.db.models.epc import EpcStore
from sqlmodel import Session
from sqlalchemy.dialects.postgresql import insert
class EpcStoreService:
@ -191,3 +192,38 @@ class EpcStoreService:
except SQLAlchemyError as e:
raise e
@classmethod
def bulk_upsert_epc_data(cls, session: Session, rows_to_insert: list[dict]):
if not rows_to_insert:
return
now = datetime.now(timezone.utc)
values = [
{
"uprn": row["uprn"],
"epc_api": row["epc_api"],
"epc_api_created_at": now,
"epc_page": row["epc_page"],
"epc_page_rrn": row["epc_page_rrn"],
"epc_page_created_at": now if row["epc_page"] else None,
}
for row in rows_to_insert
]
insert_stmt = insert(EpcStore).values(values)
stmt = insert_stmt.on_conflict_do_update(
index_elements=["uprn"],
set_={
"epc_api": insert_stmt.excluded.epc_api,
"epc_api_created_at": insert_stmt.excluded.epc_api_created_at,
"epc_page": insert_stmt.excluded.epc_page,
"epc_page_rrn": insert_stmt.excluded.epc_page_rrn,
"epc_page_created_at": insert_stmt.excluded.epc_page_created_at,
},
)
session.execute(stmt)
session.commit()

28
backend/engine/engine.py
View file

@ -716,7 +716,7 @@ async def model_engine(body: PlanTriggerRequest):
if landlord_property_id:
property_lookup[("landlord_property_id", landlord_property_id)] = prop_id
input_properties, inspections_map, eco_packages = [], {}, {}
input_properties, inspections_map, eco_packages, epc_upserts = [], {}, {}, []
for addr, config in tqdm(
zip(addresses, plan_input),
total=len(addresses),
@ -838,25 +838,25 @@ async def model_engine(body: PlanTriggerRequest):
# 2) A real EPC
# 3) A UPRN (meaning that a UPRN could be fetched against that property)
# We store this data
# TODO: Upload in bulk
with db_session() as session:
if db_funcs.epc_functions.EpcStoreService.check_insert_needed(
epc_cache, epc_searcher.newest_epc.get("estimated"), epc_searcher.uprn
):
# We store the EPC data we have found for this property
db_funcs.epc_functions.EpcStoreService.upsert_epc_data(
session=session,
uprn=epc_searcher.uprn,
epc_api=epc_searcher.data,
epc_page=epc_page_source.get("page_source"),
epc_page_rrn=epc_page_source.get("rrn"),
)
if db_funcs.epc_functions.EpcStoreService.check_insert_needed(
epc_cache, epc_searcher.newest_epc.get("estimated"), epc_searcher.uprn,
):
epc_upserts.append({
"uprn": epc_searcher.uprn,
"epc_api": epc_searcher.data,
"epc_page": epc_page_source.get("page_source"),
"epc_page_rrn": epc_page_source.get("rrn"),
})
if not input_properties:
return Response(status_code=204)
check_duplicate_property_ids(input_properties)
# We now bulk upload all of the EPC data
with db_session() as session:
db_funcs.epc_functions.EpcStoreService.bulk_upsert_epc_data(session, epc_upserts)
# We check if we have inspections data and store it in the database if so. We'll update or create
# aginst each property if
if inspections_map: