From 9963151944fc2be087c7099dc02632237ed54bb6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Aug 2025 18:53:43 +0100 Subject: [PATCH] debugging upload of funding data to db --- backend/app/db/functions/funding_functions.py | 26 ++++++++++--------- .../app/db/functions/portfolio_functions.py | 1 + .../db/functions/recommendations_functions.py | 21 ++++++++++++--- backend/app/db/models/funding.py | 5 +++- backend/app/db/models/recommendations.py | 2 ++ backend/engine/engine.py | 20 +++++++++----- 6 files changed, 52 insertions(+), 23 deletions(-) diff --git a/backend/app/db/functions/funding_functions.py b/backend/app/db/functions/funding_functions.py index 2b9f73a3..86611b9f 100644 --- a/backend/app/db/functions/funding_functions.py +++ b/backend/app/db/functions/funding_functions.py @@ -3,17 +3,17 @@ from sqlalchemy.exc import SQLAlchemyError from backend.app.db.models.funding import FundingPackage, FundingPackageMeasures -def upload_funding(session: Session, p, plan_id, property_recommendations): +def upload_funding(session: Session, p, plan_id, recommendations_to_upload): try: # Prepare data for bulk insert for Recommendation funding_package_data = { "plan_id": plan_id, "scheme": p.scheme, - "full_project_funding": p.full_project_funding, - "total_uplift": p.total_uplift, - "full_project_score": p.full_project_score, - "partial_project_score": p.partial_project_score, - "uplift_project_score": p.uplift_project_score + "project_funding": float(p.project_funding), + "total_uplift": float(p.total_uplift), + "full_project_score": float(p.full_project_score), + "partial_project_score": float(p.partial_project_score), + "uplift_project_score": float(p.uplift_project_score) } # upload the funding package data and get back the ID @@ -29,7 +29,7 @@ def upload_funding(session: Session, p, plan_id, property_recommendations): for part in p.funded_measures: recommendation_id = part["id"] recommendation = next( - (x for x in property_recommendations if x["recommendation_id"] == recommendation_id), {} + (x for x in recommendations_to_upload if x["recommendation_id"] == recommendation_id), {} ) material_id = None if recommendation["parts"]: @@ -43,13 +43,15 @@ def upload_funding(session: Session, p, plan_id, property_recommendations): "uplift_project_score": float(part["uplift_project_score"]) }) - session.bulk_insert_mappings(FundingPackageMeasures, funding_measures_data) + # Bulk insert the funding measures data + if funding_measures_data: + session.bulk_insert_mappings(FundingPackageMeasures, funding_measures_data) - # flush the changes to get the newly created IDs - session.flush() + # flush the changes to get the newly created IDs + session.flush() - # Commit the transaction - session.commit() + # Commit the transaction + session.commit() return True diff --git a/backend/app/db/functions/portfolio_functions.py b/backend/app/db/functions/portfolio_functions.py index ac340ab5..fa97c206 100644 --- a/backend/app/db/functions/portfolio_functions.py +++ b/backend/app/db/functions/portfolio_functions.py @@ -29,6 +29,7 @@ def aggregate_portfolio_recommendations( .one() ) + # Contingeny and funding are in the aggregated data aggregates_dict = { "cost": aggregates.cost or 0, "total_work_hours": aggregates.total_work_hours or 0, diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 41501270..f42f66e1 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -7,6 +7,7 @@ from backend.app.db.models.recommendations import ( from backend.app.db.models.portfolio import ( PropertyModel, PropertyTargetsModel, PropertyDetailsEpcModel ) +from backend.app.db.models.funding import FundingPackageMeasures, FundingPackage def create_plan(session: Session, plan): @@ -138,9 +139,9 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "recommendation_id": recommendation_id, "material_id": part["id"], "depth": int(part["depth"]) if part["depth"] else None, - "quantity": float(part["quantity"]), - "quantity_unit": part["quantity_unit"], - "estimated_cost": part["total"], + "quantity": float(part["quantity"]) if part.get("quantity") else None, + "quantity_unit": part.get("quantity_unit", None), + "estimated_cost": float(part.get("total", part.get("total_cost"))), } for rec, recommendation_id in zip(recommendations_to_upload, uploaded_recommendation_ids) for part in rec["parts"] @@ -176,6 +177,10 @@ def clear_portfolio(session: Session, portfolio_id: int): recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all() recommendation_ids = [r.id for r in recommendation_ids] + # Fetch all plan IDs associated with the portfolio + plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all() + plan_ids = [p.id for p in plan_ids] + # Delete all entries from RecommendationMaterials for these recommendations session.execute( delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) @@ -186,6 +191,16 @@ def clear_portfolio(session: Session, portfolio_id: int): session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar() ))) + # Delete FundingPackageMeasures → FundingPackage → Plan + session.execute( + delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_( + session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids)) + )) + ) + session.execute( + delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids)) + ) + # Delete all Plans associated with the portfolio session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id)) diff --git a/backend/app/db/models/funding.py b/backend/app/db/models/funding.py index 78ba7ff1..6ea8364e 100644 --- a/backend/app/db/models/funding.py +++ b/backend/app/db/models/funding.py @@ -21,7 +21,10 @@ class FundingPackage(Base): id = Column(Integer, primary_key=True, autoincrement=True) plan_id = Column(BigInteger, ForeignKey(Plan.id), nullable=False) - scheme = Column(String, nullable=False) # Assuming Scheme is a string representation + scheme = Column( + Enum(SchemeEnum, values_callable=lambda x: [e.value for e in x], create_constraint=False), + nullable=False + ) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) project_funding = Column(Float) total_uplift = Column(Float) diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index 54a876d7..bd5c4e20 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -91,6 +91,8 @@ class Scenario(Base): # Add in the fields we need, which were previously sitting at the portfolio level cost = Column(Float) + contingency = Column(Float) + funding = Column(Float) total_work_hours = Column(Float) energy_savings = Column(Float) co2_equivalent_savings = Column(Float) diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 108f5091..86294b07 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -23,6 +23,7 @@ from backend.app.db.functions.property_functions import ( from backend.app.db.functions.recommendations_functions import ( create_plan, upload_recommendations, create_scenario ) +from backend.app.db.functions.funding_functions import upload_funding from backend.app.db.functions.energy_assessment_functions import get_latest_assessment_by_uprn from backend.app.db.models.portfolio import rating_lookup from backend.app.plan.schemas import PlanTriggerRequest, WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES @@ -166,7 +167,9 @@ def extract_portfolio_aggregation_data( "sap_point_improvement": sap_point_improvement, "lower_bound_valuation_uplift": lower_bound_valuation_uplift, "upper_bound_valuation_uplift": upper_bound_valuation_uplift, - "has_recommendations": has_recommendations + "has_recommendations": has_recommendations, + "funding": float(p.project_funding) if p.project_funding is not None else 0, + "contingency": float(sum([x.get("contingency", 0) for x in default_recommendations])) }) agg_data = pd.DataFrame(agg_data) @@ -214,6 +217,9 @@ def extract_portfolio_aggregation_data( cost_per_sap_point = agg_data["cost"].sum() / total_sap_points if total_sap_points > 0 else 0 cost_per_sap_point = format_money(cost_per_sap_point) + total_funding = agg_data["funding"].sum() + total_contingency = agg_data["contingency"].sum() + aggregation_data = { "epc_breakdown_pre_retrofit": json.dumps( reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict()) @@ -236,6 +242,8 @@ def extract_portfolio_aggregation_data( "cost_per_co2_saved": cost_per_co2_saved, "cost_per_sap_point": cost_per_sap_point, "valuation_return_on_investment": valuation_return_on_investment, + "funding": float(total_funding), + "contingency": float(total_contingency) } return aggregation_data @@ -932,14 +940,14 @@ async def model_engine(body: PlanTriggerRequest): optimal_solution = solutions.iloc[0] # This is the list of measures that we will recommend - funded_measures = optimal_solution["items"] - solution = funded_measures + optimal_solution["unfunded_items"] + scheme = optimal_solution["scheme"] + funded_measures = optimal_solution["items"] if scheme != "none" else [] + solution = optimal_solution["items"] + optimal_solution["unfunded_items"] # This is the total amount of funding that the project will produce (including uplifts) (£) project_funding = optimal_solution["full_project_funding"] # This is the total amount of funding associated to the uplift (£) total_uplift = optimal_solution["total_uplift"] # This is the funding scheme selected - scheme = optimal_solution["scheme"] # This is the full project ABS full_project_score = optimal_solution["full_project_funding"] # This is the partial project ABS @@ -1143,9 +1151,7 @@ async def model_engine(body: PlanTriggerRequest): session, recommendations_to_upload, p.id, new_plan_id ) - upload_funding( - session, - ) + upload_funding(session, p, new_plan_id, recommendations_to_upload) property_valuation_increases.append( valuations["average_increased_value"] - valuations["current_value"]