From 981b6358c47193a14869bbecbb5034aa385b9e8e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Feb 2026 15:14:26 +0000 Subject: [PATCH] plug everything into in handler --- .../condition/condition_trigger_request.py | 6 ++++-- backend/condition/handler.py | 20 ++++++++++--------- backend/condition/lookups/uprn_lookup_s3.py | 12 ++++++++--- backend/condition/parsing/lbwf_parser.py | 1 - backend/condition/parsing/parser.py | 1 - 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/backend/condition/condition_trigger_request.py b/backend/condition/condition_trigger_request.py index 180259dc..1bea6a0d 100644 --- a/backend/condition/condition_trigger_request.py +++ b/backend/condition/condition_trigger_request.py @@ -10,5 +10,7 @@ class ConditionFileType(Enum): class ConditionTriggerRequest(BaseModel): file_type: ConditionFileType - trigger_file_path: str # TODO: split into bucket/prefix? - uprn_lookup_file_path: str # TODO: split into bucket/prefix? + trigger_file_bucket: str # TODO: get this from settings + trigger_file_key: str + uprn_lookup_file_bucket: str # TODO: get this from settings + uprn_lookup_file_key: str diff --git a/backend/condition/handler.py b/backend/condition/handler.py index a3a92d70..4aef0eac 100644 --- a/backend/condition/handler.py +++ b/backend/condition/handler.py @@ -1,19 +1,18 @@ -import asyncio import json from typing import Mapping, Any from io import BytesIO from backend.condition.condition_trigger_request import ConditionTriggerRequest from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 -from utils.logger import setup_logger from backend.condition.processor import process_file +from utils.logger import setup_logger +from utils.s3 import read_io_from_s3 logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: - # Temporary stub for PoC wiring uprn_lookup = UprnLookupS3( bucket="", key="" ) # TODO: replace with postgres implementation @@ -23,13 +22,16 @@ def handler(event: Mapping[str, Any], context: Any) -> None: body_dict = json.loads(record["body"]) payload = ConditionTriggerRequest.model_validate(body_dict) - # fetch file from s3 + file_bytes: BytesIO = read_io_from_s3( + bucket_name=payload.trigger_file_bucket, + file_key=payload.trigger_file_key, + ) - # open file and send bytes to processor + process_file( + file_stream=file_bytes, + file_type=payload.file_type, + uprn_lookup=uprn_lookup, + ) except Exception as e: logger.error(f"Failed to process record: {e}") - - # dummy_stream = BytesIO(b"") - - # process_file(dummy_stream, source_key) diff --git a/backend/condition/lookups/uprn_lookup_s3.py b/backend/condition/lookups/uprn_lookup_s3.py index 1eb68aae..da725a2f 100644 --- a/backend/condition/lookups/uprn_lookup_s3.py +++ b/backend/condition/lookups/uprn_lookup_s3.py @@ -1,7 +1,9 @@ import csv -from io import TextIOWrapper +from io import BytesIO, TextIOWrapper from typing import BinaryIO, Dict, TextIO + from backend.condition.lookups.uprn_lookup import UprnLookup +from utils.s3 import read_io_from_s3 class UprnLookupS3(UprnLookup): @@ -10,9 +12,13 @@ class UprnLookupS3(UprnLookup): self.key = key def get_property_ref_to_uprn_lookup(self) -> Dict[str, int]: - raise NotImplementedError() + file_bytes: BytesIO = read_io_from_s3( + bucket_name=self.bucket, file_key=self.key + ) - def _parse_csv(self, file_stream: BinaryIO) -> Dict[str, int]: + return self._parse_csv_bytes(file_bytes) + + def _parse_csv_bytes(self, file_stream: BinaryIO) -> Dict[str, int]: text_stream: TextIO = TextIOWrapper(file_stream, encoding="utf-8") mapping: Dict[str, int] = {} reader = csv.DictReader(text_stream) diff --git a/backend/condition/parsing/lbwf_parser.py b/backend/condition/parsing/lbwf_parser.py index 3a23d028..a713b1ef 100644 --- a/backend/condition/parsing/lbwf_parser.py +++ b/backend/condition/parsing/lbwf_parser.py @@ -18,7 +18,6 @@ class LbwfParser(Parser): def parse( self, file_stream: BinaryIO, - location_ref_to_uprn_map: Optional[Dict[str, int]] = None, ) -> Any: wb: Workbook = load_workbook(file_stream) address_to_uprn_map: Dict[str, int] = LbwfParser._generate_address_to_uprn_dict( diff --git a/backend/condition/parsing/parser.py b/backend/condition/parsing/parser.py index 825abcd5..b160b217 100644 --- a/backend/condition/parsing/parser.py +++ b/backend/condition/parsing/parser.py @@ -8,6 +8,5 @@ class Parser(ABC): def parse( self, file_stream: BinaryIO, - location_ref_to_uprn_map: Optional[Dict[str, int]] = None, ) -> Any: pass