diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index e690991a..7ffcf603 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -625,11 +625,60 @@ def get_plans_by_portfolio_id(portfolio_id: int) -> List[PlanModel]: return session_any.exec(stmt).scalars().all() -def get_scenario(scenario_id: int) -> Optional[ScenarioModel]: - stmt = select(ScenarioModel).where(ScenarioModel.id == scenario_id) +def get_plans_by_scenario_ids(ids: List[int]) -> List[PlanModel]: + stmt = select(PlanModel).where(PlanModel.scenario_id.in_(ids)) with db_read_session() as session: session_any: Any = session # Typehint as Any to satisfy Pylance... - return session_any.exec(stmt).scalar_one_or_none() + return session_any.exec(stmt).scalars().all() + + +def get_plan_ids_by_scenario_ids(scenario_ids: List[int]) -> List[int]: + stmt = select(PlanModel.id).where(PlanModel.scenario_id.in_(scenario_ids)) + with db_read_session() as session: + session_any: Any = session # Typehint as Any to satisfy Pylance... + return session_any.exec(stmt).scalars().all() + + +def get_scenarios_by_portfolio_id(portfolio_id: int) -> List[ScenarioModel]: + stmt = select(ScenarioModel).where(ScenarioModel.portfolio_id == portfolio_id) + with db_read_session() as session: + session_any: Any = session # Typehint as Any to satisfy Pylance... + return session_any.exec(stmt).scalars().all() + + +def get_default_scenario_ids_for_portfolio(portfolio_id: int) -> List[int]: + # This should in reality always return exactly 1 ID, but there's currently + # no database constraint to enforce that, so account for 0 or >1 + stmt = select(ScenarioModel.id).where( + (ScenarioModel.portfolio_id == portfolio_id) + & (ScenarioModel.is_default == True) + ) + with db_read_session() as session: + session_any: Any = session # Typehint as Any to satisfy Pylance... + return session_any.exec(stmt).scalars().all() + + +def set_plan_and_scenario_default(plan_id: int, default: bool) -> bool: + with db_session() as session: + plan: PlanModel = session.get(PlanModel, plan_id) + if not plan: + return False + + scenario_id = plan.scenario_id + + plan_mapper: Mapper[Any] = inspect(PlanModel) + scenario_mapper: Mapper[Any] = inspect(ScenarioModel) + + plan_mappings: List[Dict[str, Any]] = [{"id": plan.id, "is_default": default}] + scenario_mappings: List[Dict[str, Any]] = [ + {"id": scenario_id, "is_default": default} + ] + + session.bulk_update_mappings(plan_mapper, plan_mappings) + session.bulk_update_mappings(scenario_mapper, scenario_mappings) + session.commit() + + return True def bulk_update_plans( diff --git a/backend/categorisation/categorisation_trigger_request.py b/backend/categorisation/categorisation_trigger_request.py index 9ef1d106..fbc2328b 100644 --- a/backend/categorisation/categorisation_trigger_request.py +++ b/backend/categorisation/categorisation_trigger_request.py @@ -1,5 +1,12 @@ +from typing import List, Optional from pydantic import BaseModel class CategorisationTriggerRequest(BaseModel): portfolio_id: int + + scenarios_to_consider: Optional[List[int]] = None + scenario_priority_order: Optional[List[int]] = None + + +# {"portfolio_id": 556, "plans_to_consider": [1589319,1589320], "plan_priority_order": [1589319,1589320]} diff --git a/backend/categorisation/handler/handler.py b/backend/categorisation/handler/handler.py index 20076613..9fb235d5 100644 --- a/backend/categorisation/handler/handler.py +++ b/backend/categorisation/handler/handler.py @@ -1,5 +1,6 @@ import json from typing import Any, Mapping + from backend.categorisation.categorisation_trigger_request import ( CategorisationTriggerRequest, ) @@ -12,6 +13,10 @@ logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: + logger.info("Received message") + + logger.info(f"Number of events: {len(event.get('Records', []))}") + for record in event.get("Records", []): try: body_dict = json.loads(record["body"]) @@ -20,7 +25,12 @@ def handler(event: Mapping[str, Any], context: Any) -> None: logger.debug("Successfully validated request body") - process_portfolio(payload.portfolio_id) + process_portfolio( + payload.portfolio_id, + payload.scenarios_to_consider, + payload.scenario_priority_order, + ) except Exception as e: + logger.info("Handler exception") logger.error(f"Failed to process record: {e}") diff --git a/backend/categorisation/local_handler/docker-compose.yml b/backend/categorisation/local_handler/docker-compose.yml new file mode 100644 index 00000000..9529fdb2 --- /dev/null +++ b/backend/categorisation/local_handler/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3.9" + +services: + categorisation-lambda: + build: + context: ../../../ + dockerfile: backend/categorisation/handler/Dockerfile + ports: + - "9000:8080" + env_file: + - ../../../.env \ No newline at end of file diff --git a/backend/categorisation/local_handler/invoke_local_lambda.py b/backend/categorisation/local_handler/invoke_local_lambda.py new file mode 100644 index 00000000..127d2575 --- /dev/null +++ b/backend/categorisation/local_handler/invoke_local_lambda.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import json +import requests + +LAMBDA_URL = "http://localhost:9000/2015-03-31/functions/function/invocations" + +payload = { + "Records": [ + { + "body": json.dumps( + { + "portfolio_id": 556, + "scenarios_to_consider": [1039, 1041], + "scenarios_priority_order": [], + } + ) + } + ] +} + +response = requests.post(LAMBDA_URL, json=payload) + +print("Status code:", response.status_code) +print("Response:") +print(response.text) diff --git a/backend/categorisation/processor.py b/backend/categorisation/processor.py index 7c5698b7..966ecbf5 100644 --- a/backend/categorisation/processor.py +++ b/backend/categorisation/processor.py @@ -1,10 +1,14 @@ from collections import defaultdict -from typing import Dict, List +from typing import Dict, List, Optional, Tuple from backend.app.db.functions.recommendations_functions import ( bulk_update_plans, + get_default_scenario_ids_for_portfolio, + get_plan_ids_by_scenario_ids, get_plans_by_portfolio_id, - get_scenario, + get_plans_by_scenario_ids, + get_scenarios_by_portfolio_id, + set_plan_and_scenario_default, ) from backend.app.db.models.recommendations import PlanModel, ScenarioModel from backend.app.domain.classes.plan import Plan @@ -14,37 +18,152 @@ from utils.logger import setup_logger logger = setup_logger() -def process_portfolio(portfolio_id: int) -> None: - print(f"Processing portfolio {portfolio_id}") - plans: List[Plan] = _load_plans_for_portfolio(portfolio_id) +def process_portfolio( + portfolio_id: int, + scenarios_to_consider: Optional[List[int]] = None, + scenario_priority_order: Optional[List[int]] = None, +) -> None: + logger.info(f"Processing portfolio {portfolio_id}") + + if scenarios_to_consider: + if len(scenarios_to_consider) < 2: + raise ValueError( + "Cannot run auto categorisation for fewer than 2 scenarios" + ) + + if scenarios_to_consider is not None: + _unset_defaults_for_scenarios_not_being_considered( + portfolio_id, scenarios_to_consider + ) + + plans: List[Plan] = _load_plans_for_portfolio(portfolio_id, scenarios_to_consider) + plans_by_property: Dict[int, List[Plan]] = _group_plans_by_property(plans) - for uprn, property_plans in plans_by_property.items(): + updated_plan_models: List[PlanModel] = [] + updated_scenario_models: List[ScenarioModel] = [] + + for property_id, property_plans in plans_by_property.items(): if not property_plans: - raise ValueError(f"No plans for property {uprn}") + raise ValueError(f"No plans for property {property_id}") - cheapest_plan = _choose_cheapest_relevant_plan(property_plans) - _update_default_flags(property_plans, cheapest_plan) + cheapest_plan = choose_cheapest_relevant_plan( + property_plans, scenario_priority_order + ) + + updated_property_plan_models, updated_property_scenario_models = ( + _update_plan_and_scenario_objects(property_plans, cheapest_plan) + ) + + updated_plan_models.extend(updated_property_plan_models) + updated_scenario_models.extend(updated_property_scenario_models) + + if len(updated_plan_models) > 0: + bulk_update_plans(updated_plan_models, updated_scenario_models) + logger.info("Successfully updated Plan default values in database") -def _load_plans_for_portfolio(portfolio_id: int) -> List[Plan]: - plan_models = get_plans_by_portfolio_id(portfolio_id) - print(f"Got {len(plan_models)} plans from database") +def choose_cheapest_relevant_plan( + plans: List[Plan], scenario_priority_order: Optional[List[int]] = None +) -> Plan: + scenario_priority_order = scenario_priority_order or [] + + eligible_plans: List[Plan] = [plan for plan in plans if plan.is_compliant] or plans + if not eligible_plans: + raise ValueError("No plans available to choose from.") + + for plan in eligible_plans: + if plan.id is None: + # This should never actually happen, but plan.id is optional to cater + # for new plans. We are only working with already persisted plans here + raise ValueError( + f"All plans must have an ID, but found a plan with no ID: {plan}" + ) + + min_cost: float = min( + ( + plan.record.cost_of_works + if plan.record.cost_of_works is not None + else float("inf") + ) + for plan in eligible_plans + ) + + cheapest_plans: List[Plan] = [ + plan + for plan in eligible_plans + if (plan.record.cost_of_works or float("inf")) == min_cost + ] + + for priority_scenario_id in scenario_priority_order: + for plan in cheapest_plans: + if plan.scenario.id == priority_scenario_id: + return plan + + return cheapest_plans[0] + + +def _unset_defaults_for_scenarios_not_being_considered( + portfolio_id: int, scenarios_to_consider: List[int] +) -> None: + default_scenario_ids: List[int] = get_default_scenario_ids_for_portfolio( + portfolio_id + ) + scenarios_to_unset_default: List[int] = [] + + for id in default_scenario_ids: + if id not in scenarios_to_consider: + scenarios_to_unset_default.append(id) + + logger.info( + f"Unsetting {scenarios_to_unset_default} as default scenario(s) as not included in provided list of scenarios to consider" + ) + + if len(scenarios_to_unset_default) > 0: + plans_to_unset_default: List[int] = get_plan_ids_by_scenario_ids( + scenarios_to_unset_default + ) + for plan_id in plans_to_unset_default: + set_plan_and_scenario_default(plan_id, False) # TODO: do this in batch + + +def _load_plans_for_portfolio( + portfolio_id: int, scenarios_to_consider: Optional[List[int]] = None +) -> List[Plan]: + + if scenarios_to_consider: + logger.info(f"Getting {len(scenarios_to_consider)} plans") + plan_models: List[PlanModel] = get_plans_by_scenario_ids(scenarios_to_consider) + + else: + logger.info( + f"No list of Plans to consider provided. Getting all Plans for portfolio {portfolio_id}" + ) + plan_models: List[PlanModel] = get_plans_by_portfolio_id(portfolio_id) plans: List[Plan] = [] + scenarios: List[ScenarioModel] = get_scenarios_by_portfolio_id(portfolio_id) + + if not scenarios: + raise Exception(f"No scenarios found for Portfolio {portfolio_id}") + for model in plan_models: - if not model.scenario_id: + + scenario_model = next((s for s in scenarios if s.id == model.scenario_id)) + if not scenario_model: logger.info(f"No Scenario associated with Plan of ID {model.id}") continue - scenario_model = get_scenario(model.scenario_id) plans.append( Plan.from_sqlalchemy(model, Scenario.from_sqlalchemy(scenario_model)) ) - print("Successfully mapped plan and scenario to domain object") + logger.debug( + f"Successfully mapped plan {model.id} and scenario {scenario_model.id} to domain object" + ) + logger.debug(f"Got {len(plans)} plans from database") return plans @@ -57,37 +176,26 @@ def _group_plans_by_property(plans: List[Plan]) -> Dict[int, List[Plan]]: return grouped -def _choose_cheapest_relevant_plan(plans: List[Plan]) -> Plan: - plans_to_consider: List[Plan] = [p for p in plans if p.is_compliant] or plans - - def plan_cost(plan: Plan) -> float: - return ( - plan.record.cost_of_works - if plan.record.cost_of_works is not None - else float("inf") - ) - - cheapest_plan = min(plans_to_consider, key=plan_cost) - - return cheapest_plan - - -def _update_default_flags(plans: List[Plan], cheapest_plan: Plan) -> None: +def _update_plan_and_scenario_objects( + plans: List[Plan], cheapest_plan: Plan +) -> Tuple[List[PlanModel], List[ScenarioModel]]: plans_to_update: List[Plan] = [] for plan in plans: should_be_default: bool = plan.id == cheapest_plan.id if plan.record.is_default != should_be_default: + logger.info( + f"Setting Plan {plan.id} (Scenario Name: {plan.scenario.record.name}) to is_default: {should_be_default}" + ) plan.set_default(should_be_default) plans_to_update.append(plan) - if plans_to_update: - plan_models: List[PlanModel] = [] - scenario_models: List[ScenarioModel] = [] + plan_models: List[PlanModel] = [] + scenario_models: List[ScenarioModel] = [] - for plan in plans_to_update: - plan_model, scenario_model = plan.to_sqlalchemy() - plan_models.append(plan_model) - scenario_models.append(scenario_model) + for plan in plans_to_update: + plan_model, scenario_model = plan.to_sqlalchemy() + plan_models.append(plan_model) + scenario_models.append(scenario_model) - bulk_update_plans(plan_models, scenario_models) + return (plan_models, scenario_models) diff --git a/backend/categorisation/tests/test_prioritised_plan_selected.py b/backend/categorisation/tests/test_prioritised_plan_selected.py new file mode 100644 index 00000000..74eb8c69 --- /dev/null +++ b/backend/categorisation/tests/test_prioritised_plan_selected.py @@ -0,0 +1,94 @@ +from datetime import datetime +from typing import List +import pytest + +from backend.app.domain.classes.plan import Plan +from backend.app.domain.classes.scenario import Scenario +from backend.app.domain.records.plan_record import PlanRecord +from backend.app.domain.records.scenario_record import ScenarioRecord +from backend.app.db.models.portfolio import Epc, PortfolioGoal +from backend.categorisation.processor import choose_cheapest_relevant_plan + + +@pytest.fixture +def created_at_datetime() -> datetime: + return datetime.now() + + +def make_plan_record( + created_at: datetime, default: bool, cost_of_works: float = 500.0 +) -> PlanRecord: + return PlanRecord( + property_id=1, + portfolio_id=1, + created_at=created_at, + is_default=default, + post_epc_rating=Epc.C, + cost_of_works=cost_of_works, + ) + + +def make_scenario(name: str, created_at: datetime, is_default: bool) -> Scenario: + record = ScenarioRecord( + name=name, + created_at=created_at, + housing_type="", + goal=PortfolioGoal.INCREASING_EPC, + goal_value="C", + trigger_file_path="", + multi_plan=False, + is_default=is_default, + ) + return Scenario(record=record, id=3 if is_default else 4) + + +def make_plan( + created_at: datetime, default: bool, cost_of_works: float = 500.0, name: str = "" +) -> Plan: + scenario = make_scenario(name, created_at, default) + plan_id = 1 if default else 2 + return Plan( + record=make_plan_record(created_at, default, cost_of_works), + scenario=scenario, + id=plan_id, + ) + + +def test_prioritised_scenario_selected(created_at_datetime: datetime) -> None: + # arrange + epc_c_plan = make_plan(created_at_datetime, True, name="EPC C") + minor_works_plan = make_plan(created_at_datetime, False, name="EPC C - Minor Works") + scenario_priority_order: List[int] = [4, 3] + expected_default_plan_id = 2 + + # act + actual_default_plan = choose_cheapest_relevant_plan( + plans=[epc_c_plan, minor_works_plan], + scenario_priority_order=scenario_priority_order, + ) + + # assert + assert actual_default_plan.id == expected_default_plan_id + + +def test_cheapest_plan_returned_if_not_in_priority_list( + created_at_datetime: datetime, +) -> None: + # arrange + epc_c_plan = make_plan( + created_at_datetime, True, cost_of_works=1000.0, name="EPC C" + ) + minor_works_plan = make_plan( + created_at_datetime, False, cost_of_works=100.0, name="EPC C - Minor Works" + ) + scenario_priority_order: List[int] = [3, 5] + expected_default_plan_id = 2 + + # act + actual_default_plan = choose_cheapest_relevant_plan( + plans=[epc_c_plan, minor_works_plan], + scenario_priority_order=scenario_priority_order, + ) + + # assert + assert actual_default_plan.id == expected_default_plan_id diff --git a/backend/docker-compose-local-lambdas.yml b/backend/docker-compose-local-lambdas.yml new file mode 100644 index 00000000..50e9193b --- /dev/null +++ b/backend/docker-compose-local-lambdas.yml @@ -0,0 +1,11 @@ +version: "3.9" + +services: + categorisation-lambda: + build: + context: ../ + dockerfile: backend/categorisation/handler/Dockerfile + ports: + - "9000:8080" + env_file: + - ../.env \ No newline at end of file