Merge pull request #1271 from Hestia-Homes/feature/task-handler-clean

Feature/task handler clean

backend/epc_api/json_samples/real_life_examples/RdSAP-Schema-20.0.0/uprn_22086693/epc.json

@@ -0,0 +1,438 @@
+{
+  "uprn": 22086693,
+  "roofs": [
+    {
+      "description": {
+        "value": "Pitched, 200 mm loft insulation",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    },
+    {
+      "description": {
+        "value": "Flat, insulated (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 3
+    }
+  ],
+  "walls": [
+    {
+      "description": {
+        "value": "Cavity wall, as built, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 2,
+      "environmental_efficiency_rating": 2
+    },
+    {
+      "description": {
+        "value": "Cavity wall, as built, insulated (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "floors": [
+    {
+      "description": {
+        "value": "Suspended, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    },
+    {
+      "description": {
+        "value": "Solid, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "status": "entered",
+  "tenure": 2,
+  "window": {
+    "description": {
+      "value": "Fully double glazed",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 3,
+    "environmental_efficiency_rating": 3
+  },
+  "addendum": {
+    "addendum_numbers": [
+      8
+    ],
+    "cavity_fill_recommended": "true"
+  },
+  "lighting": {
+    "description": {
+      "value": "Low energy lighting in all fixed outlets",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "postcode": "BN2 9ZN",
+  "hot_water": {
+    "description": {
+      "value": "From main system",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "post_town": "BRIGHTON",
+  "built_form": 2,
+  "created_at": "2022-03-22 14:09:56",
+  "door_count": 1,
+  "glazed_area": 1,
+  "glazing_gap": "16+",
+  "region_code": 14,
+  "report_type": 2,
+  "sap_heating": {
+    "cylinder_size": 1,
+    "water_heating_code": 901,
+    "water_heating_fuel": 26,
+    "instantaneous_wwhrs": {
+      "rooms_with_bath_and_or_shower": 1,
+      "rooms_with_mixer_shower_no_bath": 0,
+      "rooms_with_bath_and_mixer_shower": 0
+    },
+    "secondary_fuel_type": 29,
+    "main_heating_details": [
+      {
+        "has_fghrs": "N",
+        "main_fuel_type": 26,
+        "boiler_flue_type": 2,
+        "fan_flue_present": "Y",
+        "heat_emitter_type": 1,
+        "emitter_temperature": 0,
+        "main_heating_number": 1,
+        "main_heating_control": 2106,
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "sap_main_heating_code": 113,
+        "central_heating_pump_age": 0,
+        "main_heating_data_source": 2
+      }
+    ],
+    "immersion_heating_type": "NA",
+    "secondary_heating_type": 691,
+    "has_fixed_air_conditioning": "false"
+  },
+  "sap_version": 9.94,
+  "schema_type": "RdSAP-Schema-20.0.0",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": {
+        "value": "Boiler and radiators, mains gas",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "dwelling_type": {
+    "value": "Semi-detached house",
+    "language": "1"
+  },
+  "language_code": 1,
+  "property_type": 0,
+  "address_line_1": "30 Hallett Road",
+  "assessment_type": "RdSAP",
+  "completion_date": "2022-03-22",
+  "inspection_date": "2022-03-17",
+  "extensions_count": 1,
+  "measurement_type": 1,
+  "total_floor_area": 80,
+  "transaction_type": 8,
+  "conservatory_type": 1,
+  "heated_room_count": 4,
+  "pvc_window_frames": "true",
+  "registration_date": "2022-03-22",
+  "sap_energy_source": {
+    "mains_gas": "Y",
+    "meter_type": 1,
+    "photovoltaic_supply": [
+      [
+        {
+          "pitch": 2,
+          "peak_power": {
+            "value": 1.14,
+            "quantity": "kW"
+          },
+          "orientation": 5,
+          "overshading": 1,
+          "pv_connection": 2
+        }
+      ],
+      [
+        {
+          "pitch": 2,
+          "peak_power": {
+            "value": 1.14,
+            "quantity": "kW"
+          },
+          "orientation": 7,
+          "overshading": 1,
+          "pv_connection": 2
+        }
+      ]
+    ],
+    "wind_turbines_count": 0,
+    "wind_turbines_terrain_type": 2
+  },
+  "secondary_heating": {
+    "description": {
+      "value": "Room heaters, electric",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "lzc_energy_sources": [
+    11
+  ],
+  "sap_building_parts": [
+    {
+      "identifier": "Main Dwelling",
+      "wall_dry_lined": "N",
+      "wall_thickness": 280,
+      "floor_heat_loss": 7,
+      "roof_construction": 4,
+      "wall_construction": 4,
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "floor_insulation": 1,
+          "total_floor_area": {
+            "value": 36.86,
+            "quantity": "square metres"
+          },
+          "party_wall_length": {
+            "value": 6.8,
+            "quantity": "metres"
+          },
+          "floor_construction": 2,
+          "heat_loss_perimeter": {
+            "value": 13.4,
+            "quantity": "metres"
+          }
+        },
+        {
+          "floor": 1,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "total_floor_area": {
+            "value": 36.86,
+            "quantity": "square metres"
+          },
+          "party_wall_length": {
+            "value": 6.8,
+            "quantity": "metres"
+          },
+          "heat_loss_perimeter": {
+            "value": 17.4,
+            "quantity": "metres"
+          }
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "C",
+      "party_wall_construction": 0,
+      "wall_thickness_measured": "Y",
+      "roof_insulation_location": 2,
+      "roof_insulation_thickness": "200mm",
+      "wall_insulation_thickness": "NI",
+      "floor_insulation_thickness": "NI"
+    },
+    {
+      "identifier": "Extension",
+      "wall_dry_lined": "N",
+      "floor_heat_loss": 7,
+      "roof_construction": 1,
+      "wall_construction": 4,
+      "building_part_number": 2,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "floor_insulation": 1,
+          "total_floor_area": {
+            "value": 6,
+            "quantity": "square metres"
+          },
+          "party_wall_length": 0,
+          "floor_construction": 1,
+          "heat_loss_perimeter": {
+            "value": 7,
+            "quantity": "metres"
+          }
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "H",
+      "party_wall_construction": 0,
+      "wall_thickness_measured": "N",
+      "roof_insulation_location": 6,
+      "wall_insulation_thickness": "NI",
+      "floor_insulation_thickness": "NI",
+      "flat_roof_insulation_thickness": "AB"
+    }
+  ],
+  "low_energy_lighting": 100,
+  "solar_water_heating": "N",
+  "habitable_room_count": 4,
+  "heating_cost_current": {
+    "value": 710,
+    "currency": "GBP"
+  },
+  "insulated_door_count": 0,
+  "co2_emissions_current": 2.4,
+  "energy_rating_average": 60,
+  "energy_rating_current": 72,
+  "lighting_cost_current": {
+    "value": 74,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": {
+        "value": "Programmer, room thermostat and TRVs",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "multiple_glazing_type": 3,
+  "open_fireplaces_count": 0,
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 548,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 95,
+    "currency": "GBP"
+  },
+  "mechanical_ventilation": 0,
+  "percent_draughtproofed": 100,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 112,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3500 - \u00a31,500",
+      "improvement_type": "B",
+      "improvement_details": {
+        "improvement_number": 6
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 77,
+      "environmental_impact_rating": 75
+    },
+    {
+      "sequence": 2,
+      "typical_saving": {
+        "value": 49,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3800 - \u00a31,200",
+      "improvement_type": "W1",
+      "improvement_details": {
+        "improvement_number": 57
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 79,
+      "environmental_impact_rating": 78
+    },
+    {
+      "sequence": 3,
+      "typical_saving": {
+        "value": 30,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a34,000 - \u00a36,000",
+      "improvement_type": "N",
+      "improvement_details": {
+        "improvement_number": 19
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 81,
+      "environmental_impact_rating": 80
+    }
+  ],
+  "co2_emissions_potential": 1.5,
+  "energy_rating_potential": 81,
+  "lighting_cost_potential": {
+    "value": 74,
+    "currency": "GBP"
+  },
+  "schema_version_original": "LIG-19.0",
+  "alternative_improvements": [
+    {
+      "improvement": {
+        "sequence": 1,
+        "typical_saving": {
+          "value": 59,
+          "currency": "GBP"
+        },
+        "improvement_type": "Q2",
+        "improvement_details": {
+          "improvement_number": 55
+        },
+        "improvement_category": 6,
+        "energy_performance_rating": 80,
+        "environmental_impact_rating": 78
+      }
+    }
+  ],
+  "hot_water_cost_potential": {
+    "value": 66,
+    "currency": "GBP"
+  },
+  "renewable_heat_incentive": {
+    "water_heating": 2107,
+    "impact_of_cavity_insulation": -1824,
+    "space_heating_existing_dwelling": 9574
+  },
+  "energy_consumption_current": 171,
+  "has_fixed_air_conditioning": "false",
+  "multiple_glazed_proportion": 100,
+  "calculation_software_version": "2.1.0.1",
+  "energy_consumption_potential": 103,
+  "environmental_impact_current": 69,
+  "fixed_lighting_outlets_count": 10,
+  "windows_transmission_details": {
+    "u_value": 2.6,
+    "data_source": 2,
+    "solar_transmittance": 0.76
+  },
+  "current_energy_efficiency_band": "C",
+  "environmental_impact_potential": 80,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "B",
+  "co2_emissions_current_per_floor_area": 30,
+  "low_energy_fixed_lighting_outlets_count": 10
+}

deployment/terraform/lambda/modelling_e2e/variables.tf

@@ -26,7 +26,7 @@ variable "reserved_concurrent_executions" {
 
 variable "maximum_concurrency" {
   type        = number
-  default     = 16
+  default     = 32
   description = "Maximum concurrent Lambda invocations from the SQS trigger."
 }
 

scripts/hyde/elmhurst_download.py

@@ -28,7 +28,7 @@ SESSION_DIR = HERE / ".elmhurst-session"
 SAMPLE_DIR = (
     HERE.parent.parent
     / "backend/epc_api/json_samples/real_life_examples"
-    / "SAP-Schema-16.3/uprn_100061905751"
+    / "SAP-Schema-16.0/uprn_10070004512"
 )
 
 ASSESSMENT_GUID = "B44A0DB4-4C08-4241-B818-86F060172105"

tests/utilities/aws_lambda/test_task_handler.py

@@ -0,0 +1,104 @@
+from collections.abc import Generator, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any
+from uuid import UUID
+
+import pytest
+from sqlalchemy import Engine
+from sqlmodel import Session
+
+from domain.tasks.tasks import Source
+from orchestration.task_orchestrator import TaskOrchestrator
+from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository
+from repositories.tasks.task_postgres_repository import TaskPostgresRepository
+from utilities.aws_lambda.task_handler import task_handler
+
+
+@dataclass
+class Harness:
+    orchestrator: TaskOrchestrator
+    tasks: TaskPostgresRepository
+    subtasks: SubTaskPostgresRepository
+
+    @contextmanager
+    def factory(self) -> Generator[TaskOrchestrator, None, None]:
+        yield self.orchestrator
+
+
+@pytest.fixture
+def harness(db_engine: Engine) -> Iterator[Harness]:
+    with Session(db_engine) as session:
+        tasks = TaskPostgresRepository(session=session)
+        subtasks = SubTaskPostgresRepository(session=session)
+        yield Harness(
+            orchestrator=TaskOrchestrator(task_repo=tasks, subtask_repo=subtasks),
+            tasks=tasks,
+            subtasks=subtasks,
+        )
+
+
+def _direct_event(property_id: str) -> dict[str, Any]:
+    return {"property_id": property_id}
+
+
+def test_task_handler_records_cloudwatch_url_on_subtask(
+    harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # arrange
+    monkeypatch.setenv("AWS_REGION", "eu-west-2")
+    monkeypatch.setenv(
+        "AWS_LAMBDA_LOG_GROUP_NAME", "/aws/lambda/modelling-e2e"
+    )
+    monkeypatch.setenv(
+        "AWS_LAMBDA_LOG_STREAM_NAME", "2026/05/20/[$LATEST]abc123"
+    )
+
+    @task_handler(
+        task_source="modelling_e2e",
+        source=Source.PROPERTY,
+        orchestrator_cm=harness.factory,
+    )
+    def handler(body: dict[str, Any], context: Any) -> None:
+        return None
+
+    # act
+    result = handler(_direct_event("prop-1"), context=None)
+
+    # assert
+    subtask_id = result[0]["subtask_id"]
+    saved_url = harness.subtasks.get(UUID(subtask_id)).cloud_logs_url
+    assert saved_url is not None
+    assert saved_url.startswith(
+        "https://eu-west-2.console.aws.amazon.com/cloudwatch/home"
+    )
+    # Log group / stream are console-encoded ("/" -> "$252F").
+    assert "$252Faws$252Flambda$252Fmodelling-e2e" in saved_url
+    assert "$255B$2524LATEST$255D" in saved_url
+
+
+def test_task_handler_leaves_cloudwatch_url_unset_outside_lambda(
+    harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # arrange
+    for var in (
+        "AWS_REGION",
+        "AWS_LAMBDA_LOG_GROUP_NAME",
+        "AWS_LAMBDA_LOG_STREAM_NAME",
+    ):
+        monkeypatch.delenv(var, raising=False)
+
+    @task_handler(
+        task_source="modelling_e2e",
+        source=Source.PROPERTY,
+        orchestrator_cm=harness.factory,
+    )
+    def handler(body: dict[str, Any], context: Any) -> None:
+        return None
+
+    # act
+    result = handler(_direct_event("prop-1"), context=None)
+
+    # assert
+    subtask_id = result[0]["subtask_id"]
+    assert harness.subtasks.get(UUID(subtask_id)).cloud_logs_url is None

utilities/aws_lambda/cloud_logs.py

@@ -0,0 +1,27 @@
+"""Build a CloudWatch console deep-link for the running Lambda invocation.
+
+Shared by @task_handler and @subtask_handler so both persist the same
+`cloud_logs_url` onto the SubTask they run.
+"""
+
+import os
+from typing import Optional
+from urllib.parse import quote
+
+
+def _console_encode(value: str) -> str:
+    return quote(value, safe="").replace("%", "$25")
+
+
+def cloudwatch_url() -> Optional[str]:
+    """Deep-link to this invocation's log stream, or None outside Lambda."""
+    region = os.environ.get("AWS_REGION")
+    log_group = os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME")
+    log_stream = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME")
+    if not (region and log_group and log_stream):
+        return None
+    return (
+        f"https://{region}.console.aws.amazon.com/cloudwatch/home"
+        f"?region={region}#logsV2:log-groups/log-group/"
+        f"{_console_encode(log_group)}/log-events/{_console_encode(log_stream)}"
+    )

utilities/aws_lambda/subtask_handler.py

@@ -6,12 +6,11 @@ TaskOrchestrator.run_subtask(...) calls.
 
 import json
 import logging
-import os
 from contextlib import AbstractContextManager
 from functools import wraps
 from typing import Any, Callable, Optional, cast
-from urllib.parse import quote
 
+from utilities.aws_lambda.cloud_logs import cloudwatch_url
 from utilities.aws_lambda.default_orchestrator import default_orchestrator
 from utilities.aws_lambda.subtask_trigger_body import SubtaskTriggerBody
 from orchestration.task_orchestrator import TaskOrchestrator
@@ -42,7 +41,7 @@ def subtask_handler(
 
         @wraps(func)
         def wrapper(event: dict[str, Any], context: Any) -> None:
-            cloud_logs_url = _cloudwatch_url()
+            cloud_logs_url = cloudwatch_url()
             with factory() as orchestrator:
                 for record in _records(event):
                     body = _parse_body(record)
@@ -95,20 +94,3 @@ def _records(event: dict[str, Any]) -> list[dict[str, Any]]:
     if isinstance(raw_records, list):
         return [r for r in cast(list[Any], raw_records) if isinstance(r, dict)]
     return [event]
-
-
-def _console_encode(value: str) -> str:
-    return quote(value, safe="").replace("%", "$25")
-
-
-def _cloudwatch_url() -> Optional[str]:
-    region = os.environ.get("AWS_REGION")
-    log_group = os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME")
-    log_stream = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME")
-    if not (region and log_group and log_stream):
-        return None
-    return (
-        f"https://{region}.console.aws.amazon.com/cloudwatch/home"
-        f"?region={region}#logsV2:log-groups/log-group/"
-        f"{_console_encode(log_group)}/log-events/{_console_encode(log_stream)}"
-    )

utilities/aws_lambda/task_handler.py

@@ -10,6 +10,7 @@ from contextlib import AbstractContextManager
 from functools import wraps
 from typing import Any, Callable, Optional, cast
 
+from utilities.aws_lambda.cloud_logs import cloudwatch_url
 from utilities.aws_lambda.default_orchestrator import default_orchestrator
 from domain.tasks.tasks import Source
 from orchestration.task_orchestrator import TaskOrchestrator
@@ -41,6 +42,7 @@ def task_handler(
     def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
         @wraps(func)
         def wrapper(event: dict[str, Any], context: Any) -> Any:
+            cloud_logs_url = cloudwatch_url()
             with factory() as orchestrator:
                 task_ids: list[dict[str, str]] = []
                 failures: list[dict[str, Any]] = []
@@ -66,6 +68,7 @@ def task_handler(
                         orchestrator.run_subtask(
                             subtask.id,
                             work=lambda body=body: func(body, context),
+                            cloud_logs_url=cloud_logs_url,
                         )
                     except Exception:
                         logger.exception(