From 371874380115febec25753cec19189efdbfa16ae Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Jul 2026 13:26:49 +0000 Subject: [PATCH 1/2] Carry the EPC certificate number through address2uprn to property EpcClientService.search_by_postcode already returns the matched certificate number alongside the UPRN, but it was dropped before persistence. Thread it through get_epc_data_with_postcode -> get_uprn_with_epc_df / get_uprn_from_historic_epc (using the historic dataset's lmk_key) -> the address2uprn_certificate_number result column -> PropertyIdentityInsert -> the property table's new certificate_number column (assessment-model PR #362). Co-Authored-By: Claude Sonnet 5 --- backend/address2UPRN/main.py | 40 +++++++++++++------ infrastructure/postgres/property_table.py | 1 + .../bulk_upload_finaliser_orchestrator.py | 7 ++++ .../historic_epc/historic_epc_resolver.py | 9 +++-- .../property/property_postgres_repository.py | 1 + repositories/property/property_repository.py | 7 +++- ...test_bulk_upload_finaliser_orchestrator.py | 4 ++ .../test_historic_epc_resolver.py | 3 +- 8 files changed, 52 insertions(+), 20 deletions(-) diff --git a/backend/address2UPRN/main.py b/backend/address2UPRN/main.py index 599cbaa6b..acebe03ca 100644 --- a/backend/address2UPRN/main.py +++ b/backend/address2UPRN/main.py @@ -34,22 +34,29 @@ def get_epc_data_with_postcode(postcode: str) -> pd.DataFrame: service = EpcClientService(auth_token=token) results = service.search_by_postcode(postcode) return pd.DataFrame( - [{"address": r.address_line_1, "uprn": r.uprn} for r in results] + [ + { + "address": r.address_line_1, + "uprn": r.uprn, + "certificate_number": r.certificate_number, + } + for r in results + ] ) def get_uprn_from_historic_epc( user_inputed_address: str, postcode: str, -) -> Optional[tuple[str, str, float]]: +) -> Optional[tuple[str, str, float, Optional[str]]]: """Resolve a UPRN via historic EPC S3 data. - Returns (uprn, address, lexiscore) when the historic dataset agrees on a - single rank-1 UPRN, None otherwise (no stored data, zero score, or - ambiguous top rank). The score gate is `unambiguous_uprn`'s own (score > 0); - the 0.7 heuristic used for the new-EPC source isn't applied here because - historic addresses use a more verbose format that systematically depresses - lexiscores. + Returns (uprn, address, lexiscore, certificate_number) when the historic + dataset agrees on a single rank-1 UPRN, None otherwise (no stored data, + zero score, or ambiguous top rank). The score gate is `unambiguous_uprn`'s + own (score > 0); the 0.7 heuristic used for the new-EPC source isn't + applied here because historic addresses use a more verbose format that + systematically depresses lexiscores. """ repo = HistoricEpcS3Repository.with_default_s3_client() return HistoricEpcResolver(repo).resolve_uprn(user_inputed_address, postcode) @@ -59,7 +66,7 @@ def get_uprn_with_epc_df( user_inputed_address: str, epc_df: pd.DataFrame, verbose: bool = False, -) -> Optional[str | tuple[str, str, float]]: +) -> Optional[str | tuple[str, str, float, Optional[str]]]: """ Return uprn (str) using a pre-fetched EPC dataframe. This avoids calling the API multiple times for the same postcode. @@ -88,6 +95,7 @@ def get_uprn_with_epc_df( address = top_rank_df["address"].values[0] score = float(top_rank_df["lexiscore"].values[0]) + certificate_number = top_rank_df["certificate_number"].values[0] logger.info(f"Address found to be: {address}, with lexiscore {score}") # Safe to return the agreed UPRN @@ -98,7 +106,7 @@ def get_uprn_with_epc_df( return None if verbose: - return (found_uprn, address, score) + return (found_uprn, address, score, certificate_number) else: return found_uprn @@ -120,7 +128,7 @@ def get_uprn( """ df = get_epc_data_with_postcode(postcode=postcode) - result: Optional[tuple[str, str, float]] = get_uprn_with_epc_df( + result: Optional[tuple[str, str, float, Optional[str]]] = get_uprn_with_epc_df( user_inputed_address=user_inputed_address, epc_df=df, verbose=True, @@ -393,6 +401,7 @@ def handler(event, context, local=False): "address2uprn_uprn": "invalid postcode", "address2uprn_address": "invalid postcode", "address2uprn_lexiscore": "invalid postcode", + "address2uprn_certificate_number": "invalid postcode", } ) continue @@ -428,7 +437,9 @@ def handler(event, context, local=False): continue # Get UPRN using the pre-fetched EPC data with all return options - result: Optional[tuple[str, str, float]] = get_uprn_with_epc_df( + result: Optional[ + tuple[str, str, float, Optional[str]] + ] = get_uprn_with_epc_df( user_inputed_address=address2uprn_user_input, epc_df=epc_df, verbose=True, @@ -453,7 +464,7 @@ def handler(event, context, local=False): # Parse result tuple if successful if result: - uprn, found_address, score = result + uprn, found_address, score, certificate_number = result logger.info( f"Found UPRN for {address2uprn_user_input} in {postcode}: {uprn} (score: {score})" ) @@ -464,6 +475,7 @@ def handler(event, context, local=False): "address2uprn_uprn": uprn, "address2uprn_address": found_address, "address2uprn_lexiscore": score, + "address2uprn_certificate_number": certificate_number, } ) else: @@ -476,6 +488,7 @@ def handler(event, context, local=False): "address2uprn_uprn": None, "address2uprn_address": None, "address2uprn_lexiscore": None, + "address2uprn_certificate_number": None, } ) @@ -490,6 +503,7 @@ def handler(event, context, local=False): "address2uprn_uprn": None, "address2uprn_address": None, "address2uprn_lexiscore": None, + "address2uprn_certificate_number": None, "error": str(e), } ) diff --git a/infrastructure/postgres/property_table.py b/infrastructure/postgres/property_table.py index 56d0b2fa7..9664b7885 100644 --- a/infrastructure/postgres/property_table.py +++ b/infrastructure/postgres/property_table.py @@ -49,6 +49,7 @@ class PropertyRow(SQLModel, table=True): user_inputted_address: Optional[str] = Field(default=None) user_inputted_postcode: Optional[str] = Field(default=None) lexiscore: Optional[float] = Field(default=None) + certificate_number: Optional[str] = Field(default=None) # FE-owned columns the modelling pipeline now WRITES to record a run: the old # engine set `has_recommendations` (engine.py); we mirror that, and bump diff --git a/orchestration/bulk_upload_finaliser_orchestrator.py b/orchestration/bulk_upload_finaliser_orchestrator.py index 1d707a8d7..534835e20 100644 --- a/orchestration/bulk_upload_finaliser_orchestrator.py +++ b/orchestration/bulk_upload_finaliser_orchestrator.py @@ -41,6 +41,7 @@ INTERNAL_REF_COL = "Internal Reference" UPRN_COL = "address2uprn_uprn" MATCHED_ADDRESS_COL = "address2uprn_address" LEXISCORE_COL = "address2uprn_lexiscore" +CERTIFICATE_NUMBER_COL = "address2uprn_certificate_number" MISSING_SENTINEL = "invalid postcode" UK_POSTCODE_RE = re.compile(r"[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}", re.IGNORECASE) @@ -362,6 +363,11 @@ class BulkUploadFinaliserOrchestrator: internal_ref = _normalize(raw.get(INTERNAL_REF_COL)) or None lexiscore = _parse_lexiscore(raw.get(LEXISCORE_COL)) + certificate_number_raw = _normalize(raw.get(CERTIFICATE_NUMBER_COL)) + certificate_number = ( + None if _is_missing(certificate_number_raw) else certificate_number_raw + ) + return PropertyIdentityInsert( portfolio_id=portfolio_id, uprn=uprn, @@ -371,5 +377,6 @@ class BulkUploadFinaliserOrchestrator: user_inputted_address=user_inputted_address, user_inputted_postcode=user_inputted_postcode, lexiscore=lexiscore, + certificate_number=certificate_number, creation_status="READY", ) diff --git a/repositories/historic_epc/historic_epc_resolver.py b/repositories/historic_epc/historic_epc_resolver.py index 21a7457be..37413f6c7 100644 --- a/repositories/historic_epc/historic_epc_resolver.py +++ b/repositories/historic_epc/historic_epc_resolver.py @@ -49,9 +49,10 @@ class HistoricEpcResolver: def resolve_uprn( self, user_address: str, postcode: str - ) -> Optional[tuple[str, str, float]]: - """``(uprn, matched_address, lexiscore)`` for an unambiguous rank-1 - match, else None (no data / ambiguous tie / zero score).""" + ) -> Optional[tuple[str, str, float, Optional[str]]]: + """``(uprn, matched_address, lexiscore, certificate_number)`` for an + unambiguous rank-1 match, else None (no data / ambiguous tie / zero + score). ``certificate_number`` is the historic dataset's ``lmk_key``.""" matches: HistoricEpcMatches = self.match(user_address, postcode) uprn: Optional[str] = matches.unambiguous_uprn() if not uprn or uprn == "nan": @@ -59,4 +60,4 @@ class HistoricEpcResolver: top: Optional[ScoredHistoricEpc] = matches.top() if top is None: return None - return uprn, top.record.address, top.lexiscore + return uprn, top.record.address, top.lexiscore, top.record.lmk_key diff --git a/repositories/property/property_postgres_repository.py b/repositories/property/property_postgres_repository.py index 1072c18d8..57fb9ef54 100644 --- a/repositories/property/property_postgres_repository.py +++ b/repositories/property/property_postgres_repository.py @@ -162,6 +162,7 @@ class PropertyPostgresRepository(PropertyRepository): "user_inputted_address": r.user_inputted_address, "user_inputted_postcode": r.user_inputted_postcode, "lexiscore": r.lexiscore, + "certificate_number": r.certificate_number, } for r in rows ] diff --git a/repositories/property/property_repository.py b/repositories/property/property_repository.py index e2f8284ae..3911ab3ca 100644 --- a/repositories/property/property_repository.py +++ b/repositories/property/property_repository.py @@ -13,8 +13,10 @@ class PropertyIdentityInsert: """One row inserted into the FE-owned ``property`` table at Finalise (ADR-0013). Mirrors the exact column set today's Next.js ``/finalize`` writes: nine fields - plus ``creation_status='READY'``. ``address``/``postcode`` are the resolved - (matched ?? user-inputted) values and may be ``None``. + plus ``creation_status='READY'``, plus ``certificate_number`` (the EPC + certificate the address2uprn lookup matched, not written by the old + Next.js route). ``address``/``postcode`` are the resolved (matched ?? + user-inputted) values and may be ``None``. """ portfolio_id: int @@ -25,6 +27,7 @@ class PropertyIdentityInsert: user_inputted_address: Optional[str] user_inputted_postcode: Optional[str] lexiscore: Optional[float] + certificate_number: Optional[str] = None creation_status: str = "READY" diff --git a/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py b/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py index 335a3e919..0462e9792 100644 --- a/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py +++ b/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py @@ -100,6 +100,7 @@ def test_finalise_inserts_resolved_rows_and_marks_complete() -> None: "address2uprn_uprn": "100023", "address2uprn_address": "1 SOME STREET, TOWN, SW1A 1AA", "address2uprn_lexiscore": "0.95", + "address2uprn_certificate_number": "1234-5678-9012-3456-7890", }, ] @@ -114,6 +115,7 @@ def test_finalise_inserts_resolved_rows_and_marks_complete() -> None: assert row.postcode == "SW1A 1AA" # extracted from the matched address assert row.user_inputted_address == "1 Some Street" assert row.lexiscore == 0.95 + assert row.certificate_number == "1234-5678-9012-3456-7890" assert row.creation_status == "READY" # The upload is marked complete via the injected status writer. assert status.calls == [(task_id, "complete")] @@ -128,6 +130,7 @@ def test_finalise_falls_back_to_user_input_when_unmatched() -> None: "address2uprn_uprn": "invalid postcode", "address2uprn_address": "invalid postcode", "address2uprn_lexiscore": "", + "address2uprn_certificate_number": "invalid postcode", }, ] @@ -138,6 +141,7 @@ def test_finalise_falls_back_to_user_input_when_unmatched() -> None: assert row.address == "2 Other Road" # falls back to user input assert row.postcode == "B1 1BB" # falls back to user-inputted postcode assert row.lexiscore is None + assert row.certificate_number is None # missing sentinel -> None def test_finalise_parses_pandas_float_uprn() -> None: diff --git a/tests/repositories/historic_epc/test_historic_epc_resolver.py b/tests/repositories/historic_epc/test_historic_epc_resolver.py index c4b05f8b7..f28787c1b 100644 --- a/tests/repositories/historic_epc/test_historic_epc_resolver.py +++ b/tests/repositories/historic_epc/test_historic_epc_resolver.py @@ -73,10 +73,11 @@ def test_resolve_uprn_returns_unambiguous_match(): # Assert assert result is not None - uprn, address, score = result + uprn, address, score, certificate_number = result assert uprn == "100" assert address == "47 GORDON ROAD" assert score > 0 + assert certificate_number == "" def test_resolve_uprn_is_none_when_postcode_has_no_data(): From 7abc71ed95973285094569fa33ddc3436614c152 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Jul 2026 14:16:55 +0000 Subject: [PATCH 2/2] Drop property-table wiring for certificate_number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On reflection the certificate number only needs to travel through the address2uprn pipeline (result CSV, S3 output) as an internal value — not persisted to property. Reverts the PropertyIdentityInsert / property_table.py / property_postgres_repository.py changes; keeps certificate_number flowing through get_epc_data_with_postcode, get_uprn_with_epc_df, get_uprn_from_historic_epc, and the address2uprn_certificate_number result column. Co-Authored-By: Claude Sonnet 5 --- infrastructure/postgres/property_table.py | 1 - orchestration/bulk_upload_finaliser_orchestrator.py | 7 ------- repositories/property/property_postgres_repository.py | 1 - repositories/property/property_repository.py | 7 ++----- .../test_bulk_upload_finaliser_orchestrator.py | 4 ---- 5 files changed, 2 insertions(+), 18 deletions(-) diff --git a/infrastructure/postgres/property_table.py b/infrastructure/postgres/property_table.py index 9664b7885..56d0b2fa7 100644 --- a/infrastructure/postgres/property_table.py +++ b/infrastructure/postgres/property_table.py @@ -49,7 +49,6 @@ class PropertyRow(SQLModel, table=True): user_inputted_address: Optional[str] = Field(default=None) user_inputted_postcode: Optional[str] = Field(default=None) lexiscore: Optional[float] = Field(default=None) - certificate_number: Optional[str] = Field(default=None) # FE-owned columns the modelling pipeline now WRITES to record a run: the old # engine set `has_recommendations` (engine.py); we mirror that, and bump diff --git a/orchestration/bulk_upload_finaliser_orchestrator.py b/orchestration/bulk_upload_finaliser_orchestrator.py index 534835e20..1d707a8d7 100644 --- a/orchestration/bulk_upload_finaliser_orchestrator.py +++ b/orchestration/bulk_upload_finaliser_orchestrator.py @@ -41,7 +41,6 @@ INTERNAL_REF_COL = "Internal Reference" UPRN_COL = "address2uprn_uprn" MATCHED_ADDRESS_COL = "address2uprn_address" LEXISCORE_COL = "address2uprn_lexiscore" -CERTIFICATE_NUMBER_COL = "address2uprn_certificate_number" MISSING_SENTINEL = "invalid postcode" UK_POSTCODE_RE = re.compile(r"[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}", re.IGNORECASE) @@ -363,11 +362,6 @@ class BulkUploadFinaliserOrchestrator: internal_ref = _normalize(raw.get(INTERNAL_REF_COL)) or None lexiscore = _parse_lexiscore(raw.get(LEXISCORE_COL)) - certificate_number_raw = _normalize(raw.get(CERTIFICATE_NUMBER_COL)) - certificate_number = ( - None if _is_missing(certificate_number_raw) else certificate_number_raw - ) - return PropertyIdentityInsert( portfolio_id=portfolio_id, uprn=uprn, @@ -377,6 +371,5 @@ class BulkUploadFinaliserOrchestrator: user_inputted_address=user_inputted_address, user_inputted_postcode=user_inputted_postcode, lexiscore=lexiscore, - certificate_number=certificate_number, creation_status="READY", ) diff --git a/repositories/property/property_postgres_repository.py b/repositories/property/property_postgres_repository.py index 57fb9ef54..1072c18d8 100644 --- a/repositories/property/property_postgres_repository.py +++ b/repositories/property/property_postgres_repository.py @@ -162,7 +162,6 @@ class PropertyPostgresRepository(PropertyRepository): "user_inputted_address": r.user_inputted_address, "user_inputted_postcode": r.user_inputted_postcode, "lexiscore": r.lexiscore, - "certificate_number": r.certificate_number, } for r in rows ] diff --git a/repositories/property/property_repository.py b/repositories/property/property_repository.py index 3911ab3ca..e2f8284ae 100644 --- a/repositories/property/property_repository.py +++ b/repositories/property/property_repository.py @@ -13,10 +13,8 @@ class PropertyIdentityInsert: """One row inserted into the FE-owned ``property`` table at Finalise (ADR-0013). Mirrors the exact column set today's Next.js ``/finalize`` writes: nine fields - plus ``creation_status='READY'``, plus ``certificate_number`` (the EPC - certificate the address2uprn lookup matched, not written by the old - Next.js route). ``address``/``postcode`` are the resolved (matched ?? - user-inputted) values and may be ``None``. + plus ``creation_status='READY'``. ``address``/``postcode`` are the resolved + (matched ?? user-inputted) values and may be ``None``. """ portfolio_id: int @@ -27,7 +25,6 @@ class PropertyIdentityInsert: user_inputted_address: Optional[str] user_inputted_postcode: Optional[str] lexiscore: Optional[float] - certificate_number: Optional[str] = None creation_status: str = "READY" diff --git a/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py b/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py index 0462e9792..335a3e919 100644 --- a/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py +++ b/tests/orchestration/test_bulk_upload_finaliser_orchestrator.py @@ -100,7 +100,6 @@ def test_finalise_inserts_resolved_rows_and_marks_complete() -> None: "address2uprn_uprn": "100023", "address2uprn_address": "1 SOME STREET, TOWN, SW1A 1AA", "address2uprn_lexiscore": "0.95", - "address2uprn_certificate_number": "1234-5678-9012-3456-7890", }, ] @@ -115,7 +114,6 @@ def test_finalise_inserts_resolved_rows_and_marks_complete() -> None: assert row.postcode == "SW1A 1AA" # extracted from the matched address assert row.user_inputted_address == "1 Some Street" assert row.lexiscore == 0.95 - assert row.certificate_number == "1234-5678-9012-3456-7890" assert row.creation_status == "READY" # The upload is marked complete via the injected status writer. assert status.calls == [(task_id, "complete")] @@ -130,7 +128,6 @@ def test_finalise_falls_back_to_user_input_when_unmatched() -> None: "address2uprn_uprn": "invalid postcode", "address2uprn_address": "invalid postcode", "address2uprn_lexiscore": "", - "address2uprn_certificate_number": "invalid postcode", }, ] @@ -141,7 +138,6 @@ def test_finalise_falls_back_to_user_input_when_unmatched() -> None: assert row.address == "2 Other Road" # falls back to user input assert row.postcode == "B1 1BB" # falls back to user-inputted postcode assert row.lexiscore is None - assert row.certificate_number is None # missing sentinel -> None def test_finalise_parses_pandas_float_uprn() -> None: