implementing funding optimiser

backend/Funding.py

@@ -1056,17 +1056,20 @@ class Funding:
 
         if self.tenure == "Private":
             # We return ECO4 rates
-            return innovation_uplift * (
+            rate = (
                 self.eco4_private_cavity_abs_rate if is_cavity
                 else self.eco4_private_solid_abs_rate
             )
 
+            return pps, pps * rate, innovation_uplift * rate
+
         if self.tenure == "Social":
             # We return ECO4 rates
-            return innovation_uplift * (
+            rate = (
                 self.eco4_social_cavity_abs_rate if is_cavity
                 else self.eco4_social_solid_abs_rate
             )
+            return pps, pps * rate, innovation_uplift * rate
 
         raise ValueError("Invalid tenure type for innovation uplift calculation: {}".format(self.tenure))
 

backend/app/plan/schemas.py

@@ -107,7 +107,6 @@ class PlanTriggerRequest(BaseModel):
     scenario_name: Optional[str] = ""
     scenario_id: Optional[str | int] = None  # Used to utilise and existing scenario for a engine run
     multi_plan: Optional[bool] = False
-    optimise: Optional[bool] = True
     default_u_values: Optional[bool] = True
 
     ashp_cop: Optional[float] = 2.8

backend/engine/engine.py

@@ -870,13 +870,16 @@ async def model_engine(body: PlanTriggerRequest):
                         is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"],
                     )
 
+                # TODO: Turn this into a function and store the innovaiton uplift
                 for group in measures_to_optimise_with_uplift:
                     for r in group:
                         if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating"]:
                             r["innovation_uplift"] = 0
                             continue
 
-                        r["innovation_uplift"] = funding.get_innovation_uplift(
+                         (
+                             r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"]
+                         ) = funding.get_innovation_uplift(
                             measure=r,
                             starting_sap=p.data["current-energy-efficiency"],
                             floor_area=p.floor_area,
@@ -903,26 +906,26 @@ async def model_engine(body: PlanTriggerRequest):
                 )
 
                 # Given the solutions we select the optimal one
-                solutions["cost_less_full_project_funding"] = solutions["total_cost"] - solutions[
+                solutions["cost_less_full_project_funding"] = (
-                    "eco4_full_project_funding"]
+                    solutions["total_cost"] - solutions["eco4_full_project_funding"] - solutions["total_uplift"]
+                )
                 solutions = solutions.sort_values("cost_less_full_project_funding", ascending=True)
 
                 if solutions["meets_upgrade_target"].any():
                     # If we have a solution that meets the upgrade target, we select that one
                     optimal_solution = solutions[solutions["meets_upgrade_target"]].iloc[0]
                 else:
-                    optimal_solution = optimal_solution.iloc[0]
+                    optimal_solution = solutions.iloc[0]
 
-                # optimal_solution =
+                solution = optimal_solution["items"] + optimal_solution["unfunded_items"]
+                full_project_funding = optimal_solution["eco4_full_project_funding"]
+                scheme = optimal_solution["scheme"]
 
-            if not body.optimise:
-                if body.goal != "Increasing EPC":
-                    raise NotImplementedError("Only EPC optimisation is currently supported")
-                solution = [max(sub_list, key=lambda x: (x['gain'], -x['cost'])) for sub_list in input_measures]
             else:
+                # We optimise and then we determine eligibility for funding, based on the measures selected
                 optimiser = (
                     GainOptimiser(
-                        input_measures, max_cost=body.budget, max_gain=gain, allow_slack=body.goal == "Increasing EPC"
+                        input_measures, max_cost=body.budget, max_gain=gain, allow_slack=False
                     ) if body.budget else CostOptimiser(input_measures, min_gain=gain)
                 )
                 optimiser.setup()

recommendations/optimiser/funding_optimiser.py

@@ -172,10 +172,8 @@ def _ensure_unfunded_costs(groups):
     for grp in groups:
         for opt in grp:
             base = opt.get("cost_minus_uplift")
-            upl = opt.get("innovation_uplift", 0.0)
             if base is not None:
-                opt["cost"] = float(base) + float(upl)
+                opt["cost"] = opt["raw_cost"]
-            # else: assume opt["cost"] already includes uplift
     return groups
 
 
@@ -250,7 +248,8 @@ def optimise_with_funding_paths(p, input_measures, housing_type, funding: Fundin
                     "total_gain": sub_gain,
                     "path": path_spec,
                     "scheme": scheme,
-                    "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], sub_gain)
+                    "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], sub_gain),
+                    "unfunded_items": []
                 }
             )
 
@@ -318,6 +317,43 @@ def optimise_with_funding_paths(p, input_measures, housing_type, funding: Fundin
 
             scheme = _path_scheme(path_spec)
 
+            unfunded_picked = []
+            if total_gain - target_gain < -0.1:
+                # In this case, we have a funded package that does not meet the target gain, so we look at the remaining
+                # measures and see if we can include them
+                picked_types = {opt["type"] for opt in total_picks}
+
+                # We find the indexes of the picked types
+                picked_group_index = {}
+                for pt in picked_types:
+                    for gi, grp in enumerate(optimisation_input_measures):
+                        if any(pt in opt["type"] for opt in grp):
+                            picked_group_index[pt] = gi
+                            break
+                # We get the remaining types
+                # ECO4 case
+                remaining = []
+                for i, grp in enumerate(optimisation_input_measures):
+                    if i in picked_group_index.values():
+                        continue
+                    keep = [x for x in grp if x["type"] not in picked_types]
+                    if keep:
+                        for x in keep:
+                            # Adjust to raw cost (without funding)
+                            x["cost"] = x["raw_cost"]
+                        remaining.append(keep)
+
+                if remaining:
+                    # If we have remaining measures we can optimise, we run them down an unfunded route
+                    unfunded_picked, unfunded_cost, unfunded_gain = run_optimizer(
+                        remaining,
+                        budget - total_cost if budget is not None else None,
+                        sub_target_gain=target_gain - total_gain if target_gain is not None else None
+                    )
+
+                    total_cost += unfunded_cost
+                    total_gain += unfunded_gain
+
             solutions.append({
                 "fixed_ids": fixed_ids,
                 "items": total_picks,
@@ -325,14 +361,15 @@ def optimise_with_funding_paths(p, input_measures, housing_type, funding: Fundin
                 "total_gain": total_gain,
                 "path": path_spec,
                 "scheme": scheme,
-                "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], total_gain)
+                "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], total_gain),
+                "unfunded_items": unfunded_picked,
             })
 
     solutions = pd.DataFrame(solutions)
 
     # Given the scheme, we now check if the packages are eligible. If they *are* eligible, but they don't meet the
     # final upgrade target, we then look to perform a final optimisation pass to meet the target gain.
-    solutions["meets_upgrade_target"] = solutions["total_gain"] >= target_gain
+    solutions["meets_upgrade_target"] = solutions["total_gain"] >= target_gain - 0.1
 
     # If we have packages that are fundable, but do not meet the upgrade target, we can run a final optimisation pass
     if not solutions[solutions["is_eligible"] & ~solutions["meets_upgrade_target"]].empty:
@@ -358,12 +395,19 @@ def optimise_with_funding_paths(p, input_measures, housing_type, funding: Fundin
     # if the scheme is not ECO4, we set the funding to 0 with iloc
     solutions.loc[solutions["scheme"] != "eco4", "eco4_full_project_funding"] = 0.0
 
+    # We pull out uplifts
+    solutions["total_uplift"] = solutions.apply(lambda x: get_total_uplift(x), axis=1)
+
     return solutions
 
 
 # ---- helpers -------------------------------------------------------------
 
 
+def get_total_uplift(x):
+    return sum([y["innovation_uplift"] for y in x["items"]])
+
+
 def sum_cost_gain(items):
     c = sum(float(x['cost']) for x in items)
     g = sum(float(x['gain']) for x in items)

recommendations/optimiser/optimiser_functions.py

@@ -78,6 +78,9 @@ def prepare_input_measures(property_recommendations, goal, needs_ventilation, fu
         # Build enriched measure data
         to_append = []
         for rec in recs:
+
+            raw_cost = rec["total"]
+
             if funding:
                 total = (
                     rec["total"] - rec["innovation_uplift"] + ventilation_recommendation["total"]
@@ -108,9 +111,12 @@ def prepare_input_measures(property_recommendations, goal, needs_ventilation, fu
             # We also include the innovation uplift
             to_append.append(
                 {
-                    "id": rec["recommendation_id"], "cost": non_negative_total, "gain": gain, "type": rec_type,
+                    "id": rec["recommendation_id"],
+                    "cost": non_negative_total,
+                    "gain": gain, "type": rec_type,
                     "innovation_uplift": rec["innovation_uplift"] if funding else 0,
-                    "cost_minus_uplift": total
+                    "cost_minus_uplift": total,
+                    "raw_cost": raw_cost
                 }
             )