Merge pull request #731 from Hestia-Homes/feature/categorisation-improvements

Categorisation of works improvements

backend/app/db/functions/recommendations_functions.py

@@ -625,11 +625,60 @@ def get_plans_by_portfolio_id(portfolio_id: int) -> List[PlanModel]:
         return session_any.exec(stmt).scalars().all()
 
 
-def get_scenario(scenario_id: int) -> Optional[ScenarioModel]:
+def get_plans_by_scenario_ids(ids: List[int]) -> List[PlanModel]:
-    stmt = select(ScenarioModel).where(ScenarioModel.id == scenario_id)
+    stmt = select(PlanModel).where(PlanModel.scenario_id.in_(ids))
     with db_read_session() as session:
         session_any: Any = session  # Typehint as Any to satisfy Pylance...
-        return session_any.exec(stmt).scalar_one_or_none()
+        return session_any.exec(stmt).scalars().all()
+
+
+def get_plan_ids_by_scenario_ids(scenario_ids: List[int]) -> List[int]:
+    stmt = select(PlanModel.id).where(PlanModel.scenario_id.in_(scenario_ids))
+    with db_read_session() as session:
+        session_any: Any = session  # Typehint as Any to satisfy Pylance...
+        return session_any.exec(stmt).scalars().all()
+
+
+def get_scenarios_by_portfolio_id(portfolio_id: int) -> List[ScenarioModel]:
+    stmt = select(ScenarioModel).where(ScenarioModel.portfolio_id == portfolio_id)
+    with db_read_session() as session:
+        session_any: Any = session  # Typehint as Any to satisfy Pylance...
+        return session_any.exec(stmt).scalars().all()
+
+
+def get_default_scenario_ids_for_portfolio(portfolio_id: int) -> List[int]:
+    # This should in reality always return exactly 1 ID, but there's currently
+    # no database constraint to enforce that, so account for 0 or >1
+    stmt = select(ScenarioModel.id).where(
+        (ScenarioModel.portfolio_id == portfolio_id)
+        & (ScenarioModel.is_default == True)
+    )
+    with db_read_session() as session:
+        session_any: Any = session  # Typehint as Any to satisfy Pylance...
+        return session_any.exec(stmt).scalars().all()
+
+
+def set_plan_and_scenario_default(plan_id: int, default: bool) -> bool:
+    with db_session() as session:
+        plan: PlanModel = session.get(PlanModel, plan_id)
+        if not plan:
+            return False
+
+        scenario_id = plan.scenario_id
+
+        plan_mapper: Mapper[Any] = inspect(PlanModel)
+        scenario_mapper: Mapper[Any] = inspect(ScenarioModel)
+
+        plan_mappings: List[Dict[str, Any]] = [{"id": plan.id, "is_default": default}]
+        scenario_mappings: List[Dict[str, Any]] = [
+            {"id": scenario_id, "is_default": default}
+        ]
+
+        session.bulk_update_mappings(plan_mapper, plan_mappings)
+        session.bulk_update_mappings(scenario_mapper, scenario_mappings)
+        session.commit()
+
+        return True
 
 
 def bulk_update_plans(

backend/categorisation/categorisation_trigger_request.py

@@ -1,5 +1,12 @@
+from typing import List, Optional
 from pydantic import BaseModel
 
 
 class CategorisationTriggerRequest(BaseModel):
     portfolio_id: int
+
+    scenarios_to_consider: Optional[List[int]] = None
+    scenario_priority_order: Optional[List[int]] = None
+
+
+# {"portfolio_id": 556, "plans_to_consider": [1589319,1589320], "plan_priority_order": [1589319,1589320]}

backend/categorisation/handler/handler.py

@@ -1,5 +1,6 @@
 import json
 from typing import Any, Mapping
+
 from backend.categorisation.categorisation_trigger_request import (
     CategorisationTriggerRequest,
 )
@@ -12,6 +13,10 @@ logger = setup_logger()
 
 def handler(event: Mapping[str, Any], context: Any) -> None:
 
+    logger.info("Received message")
+
+    logger.info(f"Number of events: {len(event.get('Records', []))}")
+
     for record in event.get("Records", []):
         try:
             body_dict = json.loads(record["body"])
@@ -20,7 +25,12 @@ def handler(event: Mapping[str, Any], context: Any) -> None:
 
             logger.debug("Successfully validated request body")
 
-            process_portfolio(payload.portfolio_id)
+            process_portfolio(
+                payload.portfolio_id,
+                payload.scenarios_to_consider,
+                payload.scenario_priority_order,
+            )
 
         except Exception as e:
+            logger.info("Handler exception")
             logger.error(f"Failed to process record: {e}")

backend/categorisation/local_handler/docker-compose.yml

@@ -0,0 +1,11 @@
+version: "3.9"
+
+services:
+  categorisation-lambda:
+    build:
+      context: ../../../
+      dockerfile: backend/categorisation/handler/Dockerfile
+    ports:
+      - "9000:8080"
+    env_file:
+      - ../../../.env

backend/categorisation/local_handler/invoke_local_lambda.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import json
+import requests
+
+LAMBDA_URL = "http://localhost:9000/2015-03-31/functions/function/invocations"
+
+payload = {
+    "Records": [
+        {
+            "body": json.dumps(
+                {
+                    "portfolio_id": 556,
+                    "scenarios_to_consider": [1039, 1041],
+                    "scenarios_priority_order": [],
+                }
+            )
+        }
+    ]
+}
+
+response = requests.post(LAMBDA_URL, json=payload)
+
+print("Status code:", response.status_code)
+print("Response:")
+print(response.text)

backend/categorisation/processor.py

@@ -1,10 +1,14 @@
 from collections import defaultdict
-from typing import Dict, List
+from typing import Dict, List, Optional, Tuple
 
 from backend.app.db.functions.recommendations_functions import (
     bulk_update_plans,
+    get_default_scenario_ids_for_portfolio,
+    get_plan_ids_by_scenario_ids,
     get_plans_by_portfolio_id,
-    get_scenario,
+    get_plans_by_scenario_ids,
+    get_scenarios_by_portfolio_id,
+    set_plan_and_scenario_default,
 )
 from backend.app.db.models.recommendations import PlanModel, ScenarioModel
 from backend.app.domain.classes.plan import Plan
@@ -14,37 +18,152 @@ from utils.logger import setup_logger
 logger = setup_logger()
 
 
-def process_portfolio(portfolio_id: int) -> None:
+def process_portfolio(
-    print(f"Processing portfolio {portfolio_id}")
+    portfolio_id: int,
-    plans: List[Plan] = _load_plans_for_portfolio(portfolio_id)
+    scenarios_to_consider: Optional[List[int]] = None,
+    scenario_priority_order: Optional[List[int]] = None,
+) -> None:
+    logger.info(f"Processing portfolio {portfolio_id}")
+
+    if scenarios_to_consider:
+        if len(scenarios_to_consider) < 2:
+            raise ValueError(
+                "Cannot run auto categorisation for fewer than 2 scenarios"
+            )
+
+    if scenarios_to_consider is not None:
+        _unset_defaults_for_scenarios_not_being_considered(
+            portfolio_id, scenarios_to_consider
+        )
+
+    plans: List[Plan] = _load_plans_for_portfolio(portfolio_id, scenarios_to_consider)
+
     plans_by_property: Dict[int, List[Plan]] = _group_plans_by_property(plans)
 
-    for uprn, property_plans in plans_by_property.items():
+    updated_plan_models: List[PlanModel] = []
+    updated_scenario_models: List[ScenarioModel] = []
+
+    for property_id, property_plans in plans_by_property.items():
 
         if not property_plans:
-            raise ValueError(f"No plans for property {uprn}")
+            raise ValueError(f"No plans for property {property_id}")
 
-        cheapest_plan = _choose_cheapest_relevant_plan(property_plans)
+        cheapest_plan = choose_cheapest_relevant_plan(
-        _update_default_flags(property_plans, cheapest_plan)
+            property_plans, scenario_priority_order
+        )
+
+        updated_property_plan_models, updated_property_scenario_models = (
+            _update_plan_and_scenario_objects(property_plans, cheapest_plan)
+        )
+
+        updated_plan_models.extend(updated_property_plan_models)
+        updated_scenario_models.extend(updated_property_scenario_models)
+
+    if len(updated_plan_models) > 0:
+        bulk_update_plans(updated_plan_models, updated_scenario_models)
+        logger.info("Successfully updated Plan default values in database")
 
 
-def _load_plans_for_portfolio(portfolio_id: int) -> List[Plan]:
+def choose_cheapest_relevant_plan(
-    plan_models = get_plans_by_portfolio_id(portfolio_id)
+    plans: List[Plan], scenario_priority_order: Optional[List[int]] = None
-    print(f"Got {len(plan_models)} plans from database")
+) -> Plan:
+    scenario_priority_order = scenario_priority_order or []
+
+    eligible_plans: List[Plan] = [plan for plan in plans if plan.is_compliant] or plans
+    if not eligible_plans:
+        raise ValueError("No plans available to choose from.")
+
+    for plan in eligible_plans:
+        if plan.id is None:
+            # This should never actually happen, but plan.id is optional to cater
+            # for new plans. We are only working with already persisted plans here
+            raise ValueError(
+                f"All plans must have an ID, but found a plan with no ID: {plan}"
+            )
+
+    min_cost: float = min(
+        (
+            plan.record.cost_of_works
+            if plan.record.cost_of_works is not None
+            else float("inf")
+        )
+        for plan in eligible_plans
+    )
+
+    cheapest_plans: List[Plan] = [
+        plan
+        for plan in eligible_plans
+        if (plan.record.cost_of_works or float("inf")) == min_cost
+    ]
+
+    for priority_scenario_id in scenario_priority_order:
+        for plan in cheapest_plans:
+            if plan.scenario.id == priority_scenario_id:
+                return plan
+
+    return cheapest_plans[0]
+
+
+def _unset_defaults_for_scenarios_not_being_considered(
+    portfolio_id: int, scenarios_to_consider: List[int]
+) -> None:
+    default_scenario_ids: List[int] = get_default_scenario_ids_for_portfolio(
+        portfolio_id
+    )
+    scenarios_to_unset_default: List[int] = []
+
+    for id in default_scenario_ids:
+        if id not in scenarios_to_consider:
+            scenarios_to_unset_default.append(id)
+
+    logger.info(
+        f"Unsetting {scenarios_to_unset_default} as default scenario(s) as not included in provided list of scenarios to consider"
+    )
+
+    if len(scenarios_to_unset_default) > 0:
+        plans_to_unset_default: List[int] = get_plan_ids_by_scenario_ids(
+            scenarios_to_unset_default
+        )
+        for plan_id in plans_to_unset_default:
+            set_plan_and_scenario_default(plan_id, False)  # TODO: do this in batch
+
+
+def _load_plans_for_portfolio(
+    portfolio_id: int, scenarios_to_consider: Optional[List[int]] = None
+) -> List[Plan]:
+
+    if scenarios_to_consider:
+        logger.info(f"Getting {len(scenarios_to_consider)} plans")
+        plan_models: List[PlanModel] = get_plans_by_scenario_ids(scenarios_to_consider)
+
+    else:
+        logger.info(
+            f"No list of Plans to consider provided. Getting all Plans for portfolio {portfolio_id}"
+        )
+        plan_models: List[PlanModel] = get_plans_by_portfolio_id(portfolio_id)
 
     plans: List[Plan] = []
 
+    scenarios: List[ScenarioModel] = get_scenarios_by_portfolio_id(portfolio_id)
+
+    if not scenarios:
+        raise Exception(f"No scenarios found for Portfolio {portfolio_id}")
+
     for model in plan_models:
-        if not model.scenario_id:
+
+        scenario_model = next((s for s in scenarios if s.id == model.scenario_id))
+        if not scenario_model:
             logger.info(f"No Scenario associated with Plan of ID {model.id}")
             continue
 
-        scenario_model = get_scenario(model.scenario_id)
         plans.append(
             Plan.from_sqlalchemy(model, Scenario.from_sqlalchemy(scenario_model))
         )
-        print("Successfully mapped plan and scenario to domain object")
+        logger.debug(
+            f"Successfully mapped plan {model.id} and scenario {scenario_model.id} to domain object"
+        )
 
+    logger.debug(f"Got {len(plans)} plans from database")
     return plans
 
 
@@ -57,37 +176,26 @@ def _group_plans_by_property(plans: List[Plan]) -> Dict[int, List[Plan]]:
     return grouped
 
 
-def _choose_cheapest_relevant_plan(plans: List[Plan]) -> Plan:
+def _update_plan_and_scenario_objects(
-    plans_to_consider: List[Plan] = [p for p in plans if p.is_compliant] or plans
+    plans: List[Plan], cheapest_plan: Plan
-
+) -> Tuple[List[PlanModel], List[ScenarioModel]]:
-    def plan_cost(plan: Plan) -> float:
-        return (
-            plan.record.cost_of_works
-            if plan.record.cost_of_works is not None
-            else float("inf")
-        )
-
-    cheapest_plan = min(plans_to_consider, key=plan_cost)
-
-    return cheapest_plan
-
-
-def _update_default_flags(plans: List[Plan], cheapest_plan: Plan) -> None:
     plans_to_update: List[Plan] = []
 
     for plan in plans:
         should_be_default: bool = plan.id == cheapest_plan.id
         if plan.record.is_default != should_be_default:
+            logger.info(
+                f"Setting Plan {plan.id} (Scenario Name: {plan.scenario.record.name}) to is_default: {should_be_default}"
+            )
             plan.set_default(should_be_default)
             plans_to_update.append(plan)
 
-    if plans_to_update:
+    plan_models: List[PlanModel] = []
-        plan_models: List[PlanModel] = []
+    scenario_models: List[ScenarioModel] = []
-        scenario_models: List[ScenarioModel] = []
 
-        for plan in plans_to_update:
+    for plan in plans_to_update:
-            plan_model, scenario_model = plan.to_sqlalchemy()
+        plan_model, scenario_model = plan.to_sqlalchemy()
-            plan_models.append(plan_model)
+        plan_models.append(plan_model)
-            scenario_models.append(scenario_model)
+        scenario_models.append(scenario_model)
 
-        bulk_update_plans(plan_models, scenario_models)
+    return (plan_models, scenario_models)

backend/categorisation/tests/test_prioritised_plan_selected.py

@@ -0,0 +1,94 @@
+from datetime import datetime
+from typing import List
+import pytest
+
+from backend.app.domain.classes.plan import Plan
+from backend.app.domain.classes.scenario import Scenario
+from backend.app.domain.records.plan_record import PlanRecord
+from backend.app.domain.records.scenario_record import ScenarioRecord
+from backend.app.db.models.portfolio import Epc, PortfolioGoal
+from backend.categorisation.processor import choose_cheapest_relevant_plan
+
+
+@pytest.fixture
+def created_at_datetime() -> datetime:
+    return datetime.now()
+
+
+def make_plan_record(
+    created_at: datetime, default: bool, cost_of_works: float = 500.0
+) -> PlanRecord:
+    return PlanRecord(
+        property_id=1,
+        portfolio_id=1,
+        created_at=created_at,
+        is_default=default,
+        post_epc_rating=Epc.C,
+        cost_of_works=cost_of_works,
+    )
+
+
+def make_scenario(name: str, created_at: datetime, is_default: bool) -> Scenario:
+    record = ScenarioRecord(
+        name=name,
+        created_at=created_at,
+        housing_type="",
+        goal=PortfolioGoal.INCREASING_EPC,
+        goal_value="C",
+        trigger_file_path="",
+        multi_plan=False,
+        is_default=is_default,
+    )
+    return Scenario(record=record, id=3 if is_default else 4)
+
+
+def make_plan(
+    created_at: datetime, default: bool, cost_of_works: float = 500.0, name: str = ""
+) -> Plan:
+    scenario = make_scenario(name, created_at, default)
+    plan_id = 1 if default else 2
+    return Plan(
+        record=make_plan_record(created_at, default, cost_of_works),
+        scenario=scenario,
+        id=plan_id,
+    )
+
+
+def test_prioritised_scenario_selected(created_at_datetime: datetime) -> None:
+    # arrange
+    epc_c_plan = make_plan(created_at_datetime, True, name="EPC C")
+    minor_works_plan = make_plan(created_at_datetime, False, name="EPC C - Minor Works")
+    scenario_priority_order: List[int] = [4, 3]
+    expected_default_plan_id = 2
+
+    # act
+    actual_default_plan = choose_cheapest_relevant_plan(
+        plans=[epc_c_plan, minor_works_plan],
+        scenario_priority_order=scenario_priority_order,
+    )
+
+    # assert
+    assert actual_default_plan.id == expected_default_plan_id
+
+
+def test_cheapest_plan_returned_if_not_in_priority_list(
+    created_at_datetime: datetime,
+) -> None:
+    # arrange
+    epc_c_plan = make_plan(
+        created_at_datetime, True, cost_of_works=1000.0, name="EPC C"
+    )
+    minor_works_plan = make_plan(
+        created_at_datetime, False, cost_of_works=100.0, name="EPC C - Minor Works"
+    )
+    scenario_priority_order: List[int] = [3, 5]
+    expected_default_plan_id = 2
+
+    # act
+    actual_default_plan = choose_cheapest_relevant_plan(
+        plans=[epc_c_plan, minor_works_plan],
+        scenario_priority_order=scenario_priority_order,
+    )
+
+    # assert
+    assert actual_default_plan.id == expected_default_plan_id

backend/docker-compose-local-lambdas.yml

@@ -0,0 +1,11 @@
+version: "3.9"
+
+services:
+  categorisation-lambda:
+    build:
+      context: ../
+      dockerfile: backend/categorisation/handler/Dockerfile
+    ports:
+      - "9000:8080"
+    env_file:
+      - ../.env