Model/domain/epc_prediction/comparable_properties.py
Khalim Conn-Kowlessar fa11df56c2 fix(epc-prediction): dedupe re-lodgements + leak-free leave-one-out (ADR-0029)
The register lists every historical lodgement, so a postcode cohort
contains the same physical address many times (LS61AA: 15 certs / 11
addresses; NG71AA: 15 / 9 — "FLAT 3" appears 3x in each). Two
consequences:

  - Production: a re-lodged neighbour was counting up to 3x towards the
    cohort mode. select_comparables now dedupes candidates to the latest
    cert per address (one comparable per real neighbour) — Comparable
    gains address + registration_date (the register metadata its docstring
    already anticipated, read straight off the cached payload).

  - Validation: leave-one-out leaked — predicting a flat from a near-
    identical re-lodgement of itself. The harness now holds out a whole
    address (excludes every sibling cert) and evaluates on the latest cert
    per address (the best ground truth).

Removing the leak gives the honest numbers (19 distinct addresses):
  wall_construction      93.1% -> 89.5%
  construction_age_band  65.5% -> 52.6%
  roof_construction      79.3% -> 68.4%
  floor_area mean|.|     37.9  -> 52.6 m2
The earlier figures were inflated by self-leakage; these are the real
accuracy to beat.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-14 00:40:23 +00:00

135 lines
4.9 KiB
Python

"""Comparable Properties selection for EPC Prediction (ADR-0029).
Given a `PredictionTarget` (the known inputs for an EPC-less Property) and the
raw postcode cohort of candidate `Comparable`s, `select_comparables` chooses the
reference cohort EPC Prediction synthesises from. Pure domain logic — the cohort
IO (postcode search → per-cert fetch) lives behind a repository port.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import date
from typing import Callable, Optional, Union
from datatypes.epc.domain.epc_property_data import EpcPropertyData
# Default floor on the cohort: a conditioning filter (built form, a known
# override) is applied only while at least this many comparables survive it,
# else it is relaxed (ADR-0029 filter-then-relax ladder).
_DEFAULT_MINIMUM_COHORT = 5
@dataclass(frozen=True)
class Comparable:
"""One candidate neighbour: its structured `EpcPropertyData` picture plus the
register metadata not carried on the cert (identity for leave-one-out
exclusion; recency + address for weighting + re-lodgement dedup)."""
epc: EpcPropertyData
certificate_number: str
address: Optional[str] = None
registration_date: Optional[date] = None
@dataclass(frozen=True)
class PredictionTarget:
"""The known inputs for the Property whose EPC we are predicting — the fields
guaranteed at ingestion (plus any Landlord Overrides, added as they're used).
`built_form` is often but not always known.
"""
postcode: str
property_type: str
built_form: Optional[str] = None
# A known Landlord Override (e.g. solid brick) conditions cohort selection —
# matching comparables are emphasised while enough remain (ADR-0029).
wall_construction: Optional[Union[int, str]] = None
@dataclass(frozen=True)
class ComparableProperties:
"""The selected reference cohort for a `PredictionTarget`."""
members: tuple[Comparable, ...]
def _maybe_filter(
cohort: list[Comparable],
predicate: Callable[[Comparable], bool],
*,
active: bool,
minimum_cohort: int,
) -> list[Comparable]:
"""Apply a conditioning filter only while it leaves at least
`minimum_cohort` comparables; otherwise relax it (keep the pre-filter
cohort) — the filter-then-relax ladder (ADR-0029)."""
if not active:
return cohort
filtered = [c for c in cohort if predicate(c)]
return filtered if len(filtered) >= minimum_cohort else cohort
def select_comparables(
target: PredictionTarget,
candidates: list[Comparable],
*,
minimum_cohort: int = _DEFAULT_MINIMUM_COHORT,
) -> ComparableProperties:
"""Select the Comparable Properties for `target` from the raw postcode
cohort. The register lists every historical lodgement, so first dedupe each
address to its latest cert (one comparable per real neighbour); then property
type is an always-hard filter (a flat is never a comparable for a house) and
built form is a conditioning filter on the relax ladder."""
cohort = _dedupe_to_latest_per_address(candidates)
cohort = [
c for c in cohort if c.epc.property_type == target.property_type
]
cohort = _maybe_filter(
cohort,
lambda c: c.epc.built_form == target.built_form,
active=target.built_form is not None,
minimum_cohort=minimum_cohort,
)
cohort = _maybe_filter(
cohort,
lambda c: _main_wall_construction(c) == target.wall_construction,
active=target.wall_construction is not None,
minimum_cohort=minimum_cohort,
)
return ComparableProperties(members=tuple(cohort))
def _dedupe_to_latest_per_address(
candidates: list[Comparable],
) -> list[Comparable]:
"""Collapse the register's re-lodgements: keep one comparable per address —
the latest by registration date (ties broken by certificate number, for
determinism) — so a re-lodged neighbour does not count more than once.
Candidates with no address are passed through untouched (each is its own
neighbour). Input order is otherwise preserved."""
latest: dict[str, Comparable] = {}
passthrough: list[Comparable] = []
for c in candidates:
if c.address is None:
passthrough.append(c)
continue
incumbent = latest.get(c.address)
if incumbent is None or _recency_key(c) > _recency_key(incumbent):
latest[c.address] = c
return list(latest.values()) + passthrough
def _recency_key(comparable: Comparable) -> tuple[date, str]:
"""Sort key making the most recent (then highest cert number) win. A missing
registration date sorts oldest."""
return (
comparable.registration_date or date.min,
comparable.certificate_number,
)
def _main_wall_construction(comparable: Comparable) -> object:
"""The main building part's wall construction, or None when no part lodged."""
parts = comparable.epc.sap_building_parts
return parts[0].wall_construction if parts else None