From 133a1255ebf0a98d9029997ef0f29191fcec6ddb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 27 Nov 2025 17:50:26 +0000 Subject: [PATCH] fixed missing task and subtask for single remote assessments --- BaseUtility.py | 1 - backend/Property.py | 54 +++++++++---------- backend/app/plan/router.py | 8 +++ backend/app/plan/utils.py | 3 +- backend/engine/engine.py | 3 +- etl/epc/settings.py | 2 + .../epc_attributes/FloorAttributes.py | 11 ++-- .../epc_attributes/MainheatAttributes.py | 2 +- .../test_mainheat_attributes_cases.py | 17 +++++- 9 files changed, 62 insertions(+), 39 deletions(-) diff --git a/BaseUtility.py b/BaseUtility.py index 1a31c5d0..fb5d3d67 100644 --- a/BaseUtility.py +++ b/BaseUtility.py @@ -1,5 +1,4 @@ from etl.epc.settings import DATA_ANOMALY_MATCHES as data_anon_matches -from etl.epc.settings import DATA_ANOMALY_MATCHES as data_anon_matches class Definitions: diff --git a/backend/Property.py b/backend/Property.py index d0d85565..c01e4353 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -592,13 +592,21 @@ class Property: :return: """ - if not cleaned: - raise ValueError("Cleaner does not contain cleaned data") + # if not cleaned: + # raise ValueError("Cleaner does not contain cleaned data") if not self.data: raise ValueError("Property does not contain data") - for description, attribute in cleaned.items(): + components = [ + 'floor-description', 'hotwater-description', 'main-fuel', 'mainheat-description', + 'mainheatcont-description', 'roof-description', 'walls-description', 'windows-description', + 'lighting-description' + ] + + for description in components: + + cleaner_cls = all_cleaner_map[description] if self.data[description] in self.DATA_ANOMALY_MATCHES: template = cleaned[description][0] @@ -616,35 +624,22 @@ class Property: ) continue - attributes = [ - x - for x in cleaned[description] - if x["original_description"] == self.data[description] - ] + if description == "lighting-description": + cleaner_cls = cleaner_cls(self.data[description], averages=None) + else: + cleaner_cls = cleaner_cls(self.data[description]) - if len(attributes) > 1: - raise ValueError( - "Either No attributes or multiple found for %s" % description + processed = { + "original_description": self.data[description], + "clean_description": cleaner_cls.description.replace( + "(assumed)", "" ) + .rstrip() + .capitalize(), + **cleaner_cls.process(), + } - if len(attributes) == 0: - # We attempt to perform the clean on the fly - cleaner_cls = all_cleaner_map[description] - if description == "lighting-description": - cleaner_cls = cleaner_cls(self.data[description], averages=None) - else: - cleaner_cls = cleaner_cls(self.data[description]) - processed = { - "original_description": self.data[description], - "clean_description": cleaner_cls.description.replace( - "(assumed)", "" - ) - .rstrip() - .capitalize(), - **cleaner_cls.process(), - } - - attributes = [processed] + attributes = [processed] setattr(self, self.ATTRIBUTE_MAP[description], attributes[0]) @@ -1160,6 +1155,7 @@ class Property: 'has_community_scheme': 'Varied (Community Scheme)', "has_dual_fuel_mineral_and_wood": 'Wood Logs', "has_electricaire": 'Electricity', + "has_wood_chips": 'Wood Logs' } # Hot water diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d143dc95..5611a53d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -129,6 +129,14 @@ async def trigger_plan_entrypoint(body: PlanTriggerRequest): else: # Fallback: Just send a single message try: + task_id, subtask_id = TasksInterface.create_task( + task_source="backend/plan/router.py:trigger_plan_entrypoint", + service="plan_engine", + inputs=data, + task_only=False + ) + data["task_id"] = task_id + data["subtask_id"] = subtask_id message_body = json.dumps(data) response = sqs_client.send_message( QueueUrl=settings.ENGINE_SQS_URL, diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 569eafd1..c18968f9 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -215,7 +215,7 @@ def parse_eco_packages(config: dict[str, Any], prepared_epc) -> tuple[list[str], return measures, mapped["target_sap"], mapped["plan_type"], already_installed -def build_cloudwatch_log_url(start_ms: int, end_ms: int) -> str: +def build_cloudwatch_log_url(start_ms: int) -> str: """ Build a CloudWatch Logs URL for the current Lambda invocation, including timestamp window from start_ms to end_ms (epoch ms). @@ -235,7 +235,6 @@ def build_cloudwatch_log_url(start_ms: int, end_ms: int) -> str: f"#logsV2:log-groups/log-group/{encoded_group}" f"/log-events/{encoded_stream}" f"$3Fstart={start_ms}" - f"$26end={end_ms}" ) diff --git a/backend/engine/engine.py b/backend/engine/engine.py index ce0505e2..ebb0a6b8 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -1290,8 +1290,7 @@ async def model_engine(body: PlanTriggerRequest): finally: session.close() - end_ms = int(time.time() * 1000) - cloud_logs_url = build_cloudwatch_log_url(start_ms, end_ms) + cloud_logs_url = build_cloudwatch_log_url(start_ms) # Mark the subtask as successful SubTaskInterface().update_subtask_status( subtask_id=UUID(body.subtask_id), status="complete", cloud_logs_url=cloud_logs_url diff --git a/etl/epc/settings.py b/etl/epc/settings.py index 16619fa2..d453080e 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -51,6 +51,8 @@ DATA_ANOMALY_MATCHES = { "UNKNOWN", # "Unknown", + # Observed error case + "(error), (error)", } # Add the post_sap10 date to indicate if the epc is post sap10 diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 23c7dd8e..62767638 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -55,6 +55,7 @@ class FloorAttributes(Definitions): or (description in self.DATA_ANOMALY_MATCHES) or (description in self.OBSERVED_ERRORS) or (self.description == "sap05:floor") + or not self.description ) # Try and perform a translation, incase it's in welsh @@ -63,8 +64,8 @@ class FloorAttributes(Definitions): if not self.nodata and not any( rt in self.description for rt in self.FLOOR_TYPES - + self.DWELLING_BELOW - + ["average thermal transmittance"] + + self.DWELLING_BELOW + + ["average thermal transmittance"] ): raise ValueError("Invalid description") @@ -97,7 +98,11 @@ class FloorAttributes(Definitions): def process(self) -> Dict[str, Union[str, bool, int, None]]: if self.nodata: - return {"no_data": True} + return { + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': True, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': True, 'is_solid': False, + 'another_property_below': False, 'insulation_thickness': 'none', 'no_data': True + } result: Dict[str, Union[float, str, bool, None]] = {} description = self.description diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 312fa9fe..d20d9290 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -20,7 +20,7 @@ class MainHeatAttributes(Definitions): ] FUEL_TYPES = ["electric", "mains gas", "wood logs", "coal", "oil", "wood pellets", "anthracite", "dual fuel mineral and wood", "smokeless fuel", "lpg", "b30k", "mineral and wood", - "dual fuel appliance"] + "dual fuel appliance", "wood chips"] DISTRIBUTION_SYSTEMS = ["radiators", "fan coil units", "pipes in screed above insulation", "pipes in insulated timber floor", "pipes in concrete slab"] OTHERS = ["assumed", "electricaire", "assumed for most rooms"] diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 45994b1d..e1939a7d 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1752,6 +1752,21 @@ mainheat_cases = [ 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_mineral_and_wood': False, 'has_dual_fuel_appliance': False, 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False - + }, + { + 'original_description': 'Boiler and radiators, wood chips', + 'has_radiators': True, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_hot-water-only': False, 'has_electric': False, 'has_mains_gas': False, + 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_mineral_and_wood': False, 'has_dual_fuel_appliance': False, 'has_wood_chips': True, 'has_assumed': False, + 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False } + ]