mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-27 23:35:01 +00:00
Merge pull request #1650 from Hestia-Homes/fix/pashub-sitenotes-boiler-thermostat-gable
PasHub Wythenshawe: fix 6 from_site_notes defects + re-pin harness to DB oracle (#1649)
This commit is contained in:
KhalimCK
GitHub
commit
08dc81332c
9 changed files with 1177 additions and 220 deletions
|
|
@ -115,6 +115,16 @@ one-line reason. Absent ≠ zero everywhere — note which.
|
|||
- **Water heating** 903 = Electric immersion off-peak → Elmhurst "Water Heater"
|
||||
category, not "Boiler Circulator" (901 = from main system).
|
||||
- **Windows** are synthesised from `glazed_area` band × TFA — not real geometry.
|
||||
- **Room height — give the RAW surveyed room height, NOT the engine's storey
|
||||
height.** Elmhurst adds the RdSAP +0.25 m upper-storey (joist-void) allowance
|
||||
itself. Our mapper already applies that +0.25 to upper floors
|
||||
(`_UPPER_FLOOR_HEIGHT_ADD_M`), so `sap_floor_dimensions[*].room_height_m` for
|
||||
a first floor is *already* `raw + 0.25`. Reading that value onto the sheet
|
||||
makes Elmhurst add a SECOND 0.25 → storey height too tall → ~4-5 m² extra wall
|
||||
+ volume → **~1 SAP too low** (it under-scored 6 Barry Road 60→59 until the
|
||||
first-floor height was corrected from 2.76 back to the raw 2.51). Pull heights
|
||||
from the survey/raw `building_measurements`, not the transformed
|
||||
`sap_floor_dimensions`; the ground/lowest floor takes the raw value either way.
|
||||
- **Age band — pick by construction YEAR, not the engine's band letter.** The
|
||||
live RdSAP-10 tool bands differently from `mapping.md`'s older table: it
|
||||
offers `…K 2007-2011`, **`L 2012-2022`**, **`M 2023 onwards`**. A 2020 build
|
||||
|
|
|
|||
|
|
@ -481,7 +481,9 @@ class PasHubRdSapSiteNotesExtractor:
|
|||
|
||||
def extract_room_count_elements(self) -> RoomCountElements:
|
||||
rce_section = self._section("Room Count Elements", "Customer Response")
|
||||
heated_rooms_raw = self._get_in(rce_section, "Number of heated rooms?")
|
||||
heated_rooms_raw = self._get_in(
|
||||
rce_section, "Please enter the number of HEATED rooms:"
|
||||
)
|
||||
return RoomCountElements(
|
||||
number_of_habitable_rooms=int(
|
||||
self._get_in(rce_section, "Number of habitable rooms?") or 0
|
||||
|
|
@ -624,7 +626,11 @@ class PasHubRdSapSiteNotesExtractor:
|
|||
condensing=self._bool_in(data, "Condensing"),
|
||||
year=self._get_in(data, "Year") or "",
|
||||
mount=self._get_in(data, "Mount") or "",
|
||||
open_flue=self._get_in(data, "Open Flue") or "",
|
||||
# PCDF-Search "Open Flue" column wins; a Manual-Entry boiler instead
|
||||
# lodges the flue under "Is there an open flue?" (Yes/No) (#1649 A).
|
||||
open_flue=self._get_in(data, "Open Flue")
|
||||
or self._get_in(data, "Is there an open flue?")
|
||||
or "",
|
||||
fan_assist=self._bool_in(data, "Fan Assist"),
|
||||
status=self._get_in(data, "Status") or "",
|
||||
central_heating_pump_age=self._get_in(data, "Central heating pump age:")
|
||||
|
|
@ -636,6 +642,12 @@ class PasHubRdSapSiteNotesExtractor:
|
|||
weather_compensator=self._bool_in(data, "Is there a weather compensator?"),
|
||||
emitter=self._get_in(data, "Emitter:") or "",
|
||||
emitter_temperature=self._get_in(data, "Emitter Temperature:") or "",
|
||||
# Manual-Entry boiler descriptor (blank on a PCDF-Search boiler):
|
||||
# "Type of boiler:" and "Heating System (Boiler):" (the latter
|
||||
# carries the pre/post-1998 age band) → the mapper's Table 4b
|
||||
# efficiency lookup (#1649 A).
|
||||
boiler_type=self._get_in(data, "Type of boiler:") or "",
|
||||
boiler_age_band=self._get_in(data, "Heating System (Boiler):") or "",
|
||||
)
|
||||
|
||||
def _parse_secondary_heating(self, data: List[str]) -> SecondaryHeating:
|
||||
|
|
@ -865,4 +877,9 @@ class PasHubRdSapSiteNotesExtractor:
|
|||
or "",
|
||||
wall_thickness_mm=self._wall_thickness_in(data),
|
||||
dry_lined=self._optional_bool_in(data, "Wall Dry-Lined?"),
|
||||
# Surveyed alt-wall insulation thickness ("50 mm") → the mapper's
|
||||
# §5.4 thickness input; dropped, the cascade assumes 100 mm (#1649 D).
|
||||
insulation_thickness_mm=self._parse_insulation_thickness(
|
||||
self._get_in(data, "Wall insulation thickness:")
|
||||
)[0],
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -329,6 +329,22 @@ class TestAlternativeWalls:
|
|||
).extract_alternative_walls()
|
||||
assert walls == []
|
||||
|
||||
def test_insulation_thickness_parses_to_mm(self) -> None:
|
||||
# The surveyed "Wall insulation thickness:" line must reach
|
||||
# `AlternativeWall.insulation_thickness_mm`; dropped, the cascade takes
|
||||
# the RdSAP 10 §5.4 100 mm favourable default (#1649 D).
|
||||
block = list(self._BLOCK)
|
||||
i = block.index("Wall Dry-Lined?")
|
||||
block[i:i] = ["Wall insulation thickness:", "50 mm"]
|
||||
walls = PasHubRdSapSiteNotesExtractor(block).extract_alternative_walls()
|
||||
assert walls[0].insulation_thickness_mm == 50
|
||||
|
||||
def test_no_insulation_thickness_line_lodges_none(self) -> None:
|
||||
walls = PasHubRdSapSiteNotesExtractor(
|
||||
list(self._BLOCK)
|
||||
).extract_alternative_walls()
|
||||
assert walls[0].insulation_thickness_mm is None
|
||||
|
||||
|
||||
class TestBuildingMeasurements:
|
||||
@pytest.fixture
|
||||
|
|
@ -1407,3 +1423,83 @@ class TestSolidMasonryPartyWall:
|
|||
bc.main_building.party_wall_construction_type
|
||||
== "Solid Masonry, Timber Frame, or System Built"
|
||||
)
|
||||
|
||||
|
||||
class TestManualEntryBoilerDescriptor:
|
||||
"""A Manual-Entry (non-PCDF) boiler lodges its type/age via the labels
|
||||
"Type of boiler:", "Heating System (Boiler):" (which carries the pre/post-
|
||||
1998 age band) and "Is there an open flue?" — not the PCDF-Search columns
|
||||
("Type"/"Year"/"Open Flue"). The extractor must read the manual labels so
|
||||
the mapper can resolve the SAP 10.2 Table 4b efficiency; dropped, the boiler
|
||||
descriptor is lost and the mapper falls to the generic 0.80 default
|
||||
(#1649 A). Tokens spliced verbatim from deal 500796377318 (4a Hollyhedge)."""
|
||||
|
||||
_SECTION = [
|
||||
"Heating & Hot Water",
|
||||
"How would you like to select the Heating System?",
|
||||
"Manual Entry",
|
||||
"System type:",
|
||||
"Boiler with radiators or underfloor heating",
|
||||
"Fuel:",
|
||||
"Mains Gas",
|
||||
"Type of boiler:",
|
||||
"Back boiler",
|
||||
"Heating System (Boiler):",
|
||||
"1998 or later - Back boiler",
|
||||
"Is there an open flue?",
|
||||
"Yes",
|
||||
"Ventilation",
|
||||
]
|
||||
|
||||
def test_manual_boiler_descriptor_captured(self) -> None:
|
||||
# Act
|
||||
main = PasHubRdSapSiteNotesExtractor(
|
||||
self._SECTION
|
||||
).extract_heating_and_hot_water().main_heating
|
||||
|
||||
# Assert
|
||||
assert main.selection_method == "Manual Entry"
|
||||
assert main.product_id == 0
|
||||
assert main.boiler_type == "Back boiler"
|
||||
assert main.boiler_age_band == "1998 or later - Back boiler"
|
||||
assert main.open_flue == "Yes"
|
||||
|
||||
def test_pcdf_open_flue_column_still_wins(self) -> None:
|
||||
# A PCDF-Search boiler carries the "Open Flue" column ("Room-sealed");
|
||||
# the manual "Is there an open flue?" fallback must not clobber it.
|
||||
section = [
|
||||
"Heating & Hot Water",
|
||||
"System type:",
|
||||
"Boiler with radiators or underfloor heating",
|
||||
"Open Flue",
|
||||
"Room-sealed",
|
||||
"Ventilation",
|
||||
]
|
||||
main = PasHubRdSapSiteNotesExtractor(
|
||||
section
|
||||
).extract_heating_and_hot_water().main_heating
|
||||
assert main.open_flue == "Room-sealed"
|
||||
|
||||
|
||||
class TestHeatedRoomsLabel:
|
||||
"""The PasHub PDF lodges the heated-room count under "Please enter the
|
||||
number of HEATED rooms:", not "Number of heated rooms?" — the old query
|
||||
silently dropped it to None (#1649 E)."""
|
||||
|
||||
def _section(self) -> list[str]:
|
||||
return [
|
||||
"Room Count Elements",
|
||||
"Number of habitable rooms?",
|
||||
"5",
|
||||
"Are any of these rooms unheated?",
|
||||
"Yes",
|
||||
"Please enter the number of HEATED rooms:",
|
||||
"3",
|
||||
"Customer Response",
|
||||
]
|
||||
|
||||
def test_heated_rooms_label_captured(self) -> None:
|
||||
rce = PasHubRdSapSiteNotesExtractor(
|
||||
self._section()
|
||||
).extract_room_count_elements()
|
||||
assert rce.number_of_heated_rooms == 3
|
||||
|
|
|
|||
|
|
@ -1,17 +1,30 @@
|
|||
"""End-to-end accuracy test for the pashub site-notes extractor — WYTHENSHAWE.
|
||||
|
||||
Sibling of `test_pashub_sap_accuracy.py` (the Guinness GMCA cohort); see that
|
||||
module's docstring for the full chain and the meaning of the hybrid `pre_sap`
|
||||
gauge. This module points at the `pashub_accuracy_wythenshawe/` fixtures built by
|
||||
module's docstring for the full chain. This module points at the
|
||||
`pashub_accuracy_wythenshawe/` fixtures built by
|
||||
|
||||
python scripts/build_pashub_accuracy_fixtures.py \\
|
||||
--project-code "[Wythenshawe WH:SHF RAs - WH:SF - 240426 - 494493718748]" \\
|
||||
--out-dir pashub_accuracy_wythenshawe
|
||||
|
||||
from `hubspot_deal_data` (the project_code above) + the `rd_sap_site_note` PDFs
|
||||
in S3. The ratchets below are calibrated to THIS cohort — do not copy the
|
||||
Guinness constants across; each portfolio discovers its own floor/ceiling by
|
||||
running.
|
||||
in S3.
|
||||
|
||||
ORACLE: this module grades against the **DB PasHub-assessment SAP** — the
|
||||
`epc_property_energy_performance.energy_rating_current` (`ep.source='lodged'`)
|
||||
that PasHub's own assessment lodged for the dwelling — NOT the manifest
|
||||
`pre_sap` (`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently
|
||||
wrong; grading against it gave a misleading 79.2% / MAE 0.75 while the engine
|
||||
actually matched the DB in every pre_sap≠DB case). Each fixture's oracle SAP is
|
||||
resolved (by UPRN or postcode+house-number) and baked into `manifest.json` as
|
||||
`oracle_sap` by `scripts/enrich_pashub_wythenshawe_oracle.py`, so CI needs no DB
|
||||
access. 20 fixtures have no DB-oracle match yet (`oracle_sap` null) and are
|
||||
excluded from the accuracy aggregate until PasHub sources their SAP.
|
||||
|
||||
The ratchets below are calibrated to THIS cohort against the DB oracle — do not
|
||||
copy the Guinness constants across; each portfolio discovers its own
|
||||
floor/ceiling by running.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -35,39 +48,35 @@ _KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel
|
|||
_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy_wythenshawe"
|
||||
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"
|
||||
|
||||
# --- Ratchets (never loosen at fixed coverage), mirroring the Guinness module.
|
||||
# First full run (2026-07-16): **149 computed / 19 blocked / 0 errored**
|
||||
# (within-0.5 76.5% = 114/149, MAE 0.665). The 19 blocks are a spread of
|
||||
# separately-ticketed mapper gaps — floor type 'Same dwelling below' (×10),
|
||||
# room-in-roof gable 'None' (×2), 'No thermostatic control...' on 'Room heaters'
|
||||
# (×2), plus one-off secondary-heating / secondary-fuel / room-heater / HHR
|
||||
# storage-control (2404-on-402) labels — each its own field-by-field follow-up,
|
||||
# not tuning. The floor/ceiling below are the observed baseline with a hair of
|
||||
# headroom; treat them as a regression tripwire, NOT a cohort accuracy figure,
|
||||
# and re-baseline (coverage growth is not loosening) as further mapper fixes
|
||||
# unblock fixtures.
|
||||
# 2026-07-16 (#1622/#1624): room-heater control 2601 unblocked the two 'No
|
||||
# thermostatic control...' fixtures → **151 computed / 17 blocked** (within-0.5
|
||||
# 76.2% = 115/151, MAE 0.716; 500767954156 lands within 0.5, 500797465814 is a
|
||||
# −8.8 tail entry). MAE ceiling re-baselined 0.67 → 0.72 per the
|
||||
# coverage-growth convention above.
|
||||
# 2026-07-16 (#1622/#1625): floor type 'Same dwelling below' → "(another
|
||||
# dwelling below)" party-floor suppression (gov-API code-8 precedent) unblocked
|
||||
# all 10 fixtures → **161 computed / 7 blocked** (within-0.5 75.8% = 122/161,
|
||||
# MAE 0.740). 7/10 newly-computed land within 0.5 — the treatment validates;
|
||||
# −3.3/−5.5 tail entries dilute the aggregates. Floor 0.76 → 0.75, ceiling
|
||||
# 0.72 → 0.75.
|
||||
# 2026-07-16 (#1622/#1626-#1631): the remaining one-off labels (HHR-control
|
||||
# 2404→2402 downgrade, 'Loose jacket' cylinder, secondary 601/609, secondary
|
||||
# dual fuel, gas room-heater 603) → **165 computed / 3 blocked** (within-0.5
|
||||
# 75.2% = 124/165, MAE 0.844). The MAE step is one accepted pre_sap
|
||||
# divergence: 500797805807 (90 Brookfield Gardens) scores 49.5 vs pre_sap 31
|
||||
# (+18.5) — its own 2025 accredited cert 6890-0597-0722-0196-3943 lodges the
|
||||
# identical heating codes (603/26/2601) at band E, agreeing with US; the
|
||||
# PasHub pre_sap instead matches the dwelling's superseded electric
|
||||
# room-heater certs (2017/2023, band F). Ceiling 0.75 → 0.85.
|
||||
_MIN_WITHIN_HALF: float = 0.75
|
||||
_MAX_SAP_MAE: float = 0.85
|
||||
# --- Ratchets (never loosen at fixed coverage), against the DB oracle.
|
||||
# Re-pinned to the DB PasHub-assessment SAP (#1649): grading the earlier
|
||||
# pre_sap-based history (which floated 75-76% / MAE 0.67-0.85 purely on
|
||||
# surveyor data-entry noise) is retired — see the module docstring.
|
||||
# 2026-07-19 (#1649 A-F): with the six from_site_notes fixes in place, the
|
||||
# **148 DB-oracle-matched fixtures** (39 UPRN + 109 postcode+house-number; 20
|
||||
# unmatched, excluded) score **within-0.5 89.2% (132/148), MAE 0.297**. The
|
||||
# residual tail is NOT fixable engine defects — two independent causes, both
|
||||
# established, not inferred:
|
||||
# 1. Integer-oracle rounding: the DB oracle is a *rounded integer* SAP graded
|
||||
# against our continuous engine, so a band of spec-correct dwellings sits
|
||||
# just over a rounding boundary (within-1.0 is 98%).
|
||||
# 2. Input-provenance drift between the site-notes survey and the lodged cert.
|
||||
# The two worst under-raters, 6 Barry Road (ours 60.12, oracle 62) and 87
|
||||
# Alderue (ours 61.80, oracle 63), were re-keyed into accredited Elmhurst
|
||||
# on our exact inputs: Elmhurst returned **60 and 62** — matching our
|
||||
# engine, and both *below* the lodged oracle. So our engine computes
|
||||
# correct RdSAP; the lodged cert simply carried a lighter input set (it
|
||||
# dropped the 17.4 m² solid-brick alternative wall the surveyor recorded
|
||||
# via brick-bond evidence, +1.7 SAP). Accredited engines agree on identical
|
||||
# inputs — a 2-point gap is an input difference, not a calculation one.
|
||||
# So do NOT chase these as engine bugs, and do NOT tune the engine to close them
|
||||
# (that would move the gov-API corpus off 78.9% and diverge from accredited
|
||||
# Elmhurst). Treat floor/ceiling as a regression tripwire, not a target;
|
||||
# re-baseline (coverage growth is not loosening) as PasHub sources the 20
|
||||
# unmatched SAPs. The clean accuracy proof is engine-vs-accredited on identical
|
||||
# inputs (done for 6 Barry / 87 Alderue), not engine-vs-lodged-integer.
|
||||
_MIN_WITHIN_HALF: float = 0.88
|
||||
_MAX_SAP_MAE: float = 0.31
|
||||
|
||||
_KNOWN_GAP_REASON = (
|
||||
"pashub `from_site_notes` mapper does not int-code a main-heating "
|
||||
|
|
@ -82,6 +91,13 @@ class PashubFixture:
|
|||
pre_sap_raw: str
|
||||
pre_sap_score: int
|
||||
pdf: str
|
||||
# DB PasHub-assessment SAP (the grading oracle) —
|
||||
# `epc_property_energy_performance.energy_rating_current`, `source='lodged'`,
|
||||
# matched by UPRN or postcode+house-number and baked into the manifest by
|
||||
# `scripts/enrich_pashub_wythenshawe_oracle.py` (see the module docstring).
|
||||
# None for the 20 address-unmatched fixtures (excluded from the aggregate).
|
||||
oracle_sap: Optional[int] = None
|
||||
oracle_match: Optional[str] = None # "uprn" | "address" | None
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
|
|
@ -95,7 +111,7 @@ class Outcome:
|
|||
blocked: bool # blocked on the known main-heating-control gap (xfail territory)
|
||||
reason: Optional[str]
|
||||
sap_continuous: Optional[float]
|
||||
diff: Optional[float] # abs(our SAP - pre_sap)
|
||||
diff: Optional[float] # abs(our SAP - DB oracle); None when unmatched/blocked
|
||||
|
||||
|
||||
def _load_manifest() -> list[PashubFixture]:
|
||||
|
|
@ -119,7 +135,14 @@ def _evaluate(deal_id: str) -> Outcome:
|
|||
result = Sap10Calculator().calculate(epc)
|
||||
except _KNOWN_GAPS as exc:
|
||||
return Outcome(blocked=True, reason=str(exc), sap_continuous=None, diff=None)
|
||||
diff = abs(result.sap_score_continuous - fixture.pre_sap_score)
|
||||
# Grade against the DB PasHub-assessment oracle, not the surveyor-entered
|
||||
# `pre_sap`. A fixture with no DB-oracle match carries diff None and is
|
||||
# excluded from the accuracy aggregate (it still asserts it computes).
|
||||
diff = (
|
||||
abs(result.sap_score_continuous - fixture.oracle_sap)
|
||||
if fixture.oracle_sap is not None
|
||||
else None
|
||||
)
|
||||
return Outcome(
|
||||
blocked=False,
|
||||
reason=None,
|
||||
|
|
@ -141,10 +164,14 @@ def test_pashub_fixture_computes(deal_id: str) -> None:
|
|||
|
||||
@pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy_wythenshawe fixtures/manifest")
|
||||
def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""SAP within-0.5 of pashub's `pre_sap`, aggregated over computable fixtures."""
|
||||
"""SAP within-0.5 of the DB PasHub-assessment oracle, aggregated over the
|
||||
address/UPRN-matched fixtures. Fixtures with no DB-oracle match are counted
|
||||
(`unmatched`) but excluded from the accuracy figures — they cannot be graded
|
||||
until PasHub sources their SAP."""
|
||||
diffs: list[float] = []
|
||||
blocked = 0
|
||||
errored = 0
|
||||
unmatched = 0
|
||||
for deal_id in _IDS:
|
||||
try:
|
||||
outcome = _evaluate(deal_id)
|
||||
|
|
@ -154,13 +181,15 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No
|
|||
if outcome.blocked:
|
||||
blocked += 1
|
||||
continue
|
||||
assert outcome.diff is not None
|
||||
if outcome.diff is None: # computed, but no DB-oracle SAP to grade against
|
||||
unmatched += 1
|
||||
continue
|
||||
diffs.append(outcome.diff)
|
||||
|
||||
if not diffs:
|
||||
pytest.xfail(
|
||||
f"no fixture computes a SAP score "
|
||||
f"({blocked} blocked on the main-heating-control gap, {errored} errored); "
|
||||
f"no fixture grades against the DB oracle "
|
||||
f"({blocked} blocked, {unmatched} address-unmatched, {errored} errored); "
|
||||
f"{_KNOWN_GAP_REASON}"
|
||||
)
|
||||
|
||||
|
|
@ -170,8 +199,8 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No
|
|||
|
||||
with capsys.disabled():
|
||||
print(
|
||||
f"\n[pashub WYTHENSHAWE | {n} computed / {blocked} blocked / "
|
||||
f"{errored} errored]"
|
||||
f"\n[pashub WYTHENSHAWE | {n} graded / {unmatched} unmatched / "
|
||||
f"{blocked} blocked / {errored} errored | oracle=DB energy_rating_current]"
|
||||
f"\n SAP within-0.5 = {within_half:.1%} MAE = {sap_mae:.3f}"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6124,11 +6124,20 @@ def _pashub_room_in_roof(
|
|||
continue
|
||||
surface_kind = kind
|
||||
if surface_kind is None: # gable — kind from the surveyed type
|
||||
if d.gable_type not in _PASHUB_GABLE_TYPE_TO_KIND:
|
||||
if d.gable_type is None:
|
||||
# A form variant omits the "Gable wall N type:" line, so the
|
||||
# type is a genuine field-absence (not a truncation or a new
|
||||
# label). Default unlodged → party `gable_wall` (U=0.25),
|
||||
# mirroring the gov-API sibling `_api_type_1_gable_kind(None)`
|
||||
# (SAP 10.2 Table 4 p.22 party-wall default) (#1649 B). A
|
||||
# present-but-unrecognised label still strict-raises below.
|
||||
surface_kind = "gable_wall"
|
||||
elif d.gable_type not in _PASHUB_GABLE_TYPE_TO_KIND:
|
||||
raise UnmappedPasHubLabel(
|
||||
"room-in-roof gable type", str(d.gable_type)
|
||||
)
|
||||
surface_kind = _PASHUB_GABLE_TYPE_TO_KIND[d.gable_type]
|
||||
else:
|
||||
surface_kind = _PASHUB_GABLE_TYPE_TO_KIND[d.gable_type]
|
||||
thickness = insulation_thickness_mm
|
||||
if form_asks_insulation_known and not d.insulation_known:
|
||||
thickness = None
|
||||
|
|
@ -6377,6 +6386,15 @@ def _map_pashub_alternative_wall(a: AlternativeWall) -> SapAlternativeWall:
|
|||
),
|
||||
wall_thickness_measured="Y" if a.wall_thickness_mm is not None else "N",
|
||||
wall_thickness_mm=a.wall_thickness_mm,
|
||||
# Surveyed alt-wall insulation thickness → the cascade's
|
||||
# `_parse_thickness_mm`; dropped, RdSAP 10 §5.4 assumes a favourable
|
||||
# 100 mm (#1649 D). Mirror the main-wall `wall_insulation_thickness`
|
||||
# string wiring.
|
||||
wall_insulation_thickness=(
|
||||
str(a.insulation_thickness_mm)
|
||||
if a.insulation_thickness_mm is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -6544,6 +6562,18 @@ def _map_sap_heating(
|
|||
cylinder_insulation_type=_pashub_cylinder_insulation_type_code(
|
||||
heating.water_heating.insulation_type, cylinder_present
|
||||
),
|
||||
# A surveyed cylinder thermostat ("Has thermostat? Yes") must reach the
|
||||
# calculator, which reads `!= "Y"` to apply the SAP 10.2 Table 2b Note
|
||||
# a) ×1.3 absent-thermostat cylinder-temperature-factor penalty (and the
|
||||
# Table 4c(2) boiler interlock). Dropped (None), every PasHub cylinder
|
||||
# dwelling was penalised as thermostat-less (#1649 F). Gated on a lodged
|
||||
# cylinder — a combi/no-cylinder dwelling keeps None. Mirrors the
|
||||
# Elmhurst path (`"Y" if cylinder_thermostat else "N"`).
|
||||
cylinder_thermostat=(
|
||||
("Y" if heating.water_heating.has_thermostat else "N")
|
||||
if cylinder_present
|
||||
else None
|
||||
),
|
||||
cylinder_insulation_thickness_mm=heating.water_heating.insulation_thickness_mm,
|
||||
water_heating_code=water_heating_code,
|
||||
water_heating_fuel=water_heating_fuel,
|
||||
|
|
@ -6702,6 +6732,10 @@ _PASHUB_SECONDARY_HEATING_TYPE_TO_SAP10: Dict[str, int] = {
|
|||
"Fire or wall heater with balanced flue": 609,
|
||||
# Solid fuel (eff 0.32 column B).
|
||||
"Open fire in grate": 631,
|
||||
# Solid-fuel closed room heater — SAP 10.2 Table 4a code 633 (inside the
|
||||
# Elmhurst solid-fuel secondary block {631..636}, so downstream fuel/
|
||||
# category resolution already covers it) (#1649 C).
|
||||
"Closed room heater": 633,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -8154,11 +8188,45 @@ def _pashub_main_heating_control_code(
|
|||
return code
|
||||
|
||||
|
||||
# PasHub Manual-Entry boiler descriptor → SAP 10.2 Table 4b sub-row code
|
||||
# (`table_4b.py`), keyed on the surveyed "Heating System (Boiler):" cell
|
||||
# (`boiler_age_band`), which carries both the boiler type and the pre/post-1998
|
||||
# age band. A Manual-Entry boiler has no PCDB `product_id`, so its efficiency
|
||||
# must come from Table 4b, not the generic 0.80 gas default (#1649 A). Only the
|
||||
# cohort's observed gas/liquid descriptors are mapped; an unmapped descriptor
|
||||
# (or an electric "Direct acting" boiler, which Table 4b does not cover) resolves
|
||||
# to None, preserving the pre-existing behaviour (no block, no regression).
|
||||
_PASHUB_MANUAL_BOILER_AGE_BAND_TO_TABLE_4B: Dict[str, int] = {
|
||||
# Gas boilers 1998 or later (Table 4b 101-109).
|
||||
"1998 or later - Condensing combi": 104, # 84/75 winter/summer
|
||||
"1998 or later - Condensing": 102, # regular condensing, 84/74
|
||||
"1998 or later - Back boiler": 109, # back boiler to radiators, 66/56
|
||||
# Gas pre-1998 (Table 4b 110-119).
|
||||
"Pre 1998 - Back boiler": 119, # back boiler to radiators, 66/56
|
||||
"Pre 1998 - Wall mounted": 115, # regular, wall mounted, 66/56
|
||||
# Liquid fuel boilers (Table 4b 124-132).
|
||||
"Pre 1998 - Standard oil boiler": 125, # standard oil 1985-1997, 71/59
|
||||
}
|
||||
|
||||
|
||||
def _pashub_manual_boiler_table_4b_code(main: PasHubMainHeating) -> Optional[int]:
|
||||
"""Resolve a Manual-Entry boiler's surveyed descriptor to its SAP 10.2
|
||||
Table 4b code (#1649 A). Returns None for a PCDF-Search boiler (product_id
|
||||
lodged — efficiency is the SEDBUK lookup) and for any descriptor the Table
|
||||
4b map does not cover (e.g. an electric boiler), leaving the calculator's
|
||||
existing cascade untouched for those."""
|
||||
if main.product_id > 0:
|
||||
return None
|
||||
return _PASHUB_MANUAL_BOILER_AGE_BAND_TO_TABLE_4B.get(main.boiler_age_band)
|
||||
|
||||
|
||||
def _pashub_main_heating_system_code(main: PasHubMainHeating) -> Optional[int]:
|
||||
"""Resolve the Table 4a system code for a PasHub storage/room-heater
|
||||
lodgement at the mapper boundary (ADR-0015). Heat networks resolve their
|
||||
code via `_pashub_community_heating`; boilers carry None (their efficiency
|
||||
is a SEDBUK/product lookup, not a Table 4a rd row).
|
||||
code via `_pashub_community_heating`. A PCDF-Search boiler carries None (its
|
||||
efficiency is the SEDBUK/product lookup); a Manual-Entry boiler resolves its
|
||||
surveyed descriptor to a SAP 10.2 Table 4b code (#1649 A) so the calculator
|
||||
bills the Table 4b seasonal efficiency instead of the generic 0.80 default.
|
||||
|
||||
The surveyed heater *type* is lodged in the "Heating System (Other)" field
|
||||
(`community_heat_source`) for Manual Entry; a PCDF-Search storage lodgement
|
||||
|
|
@ -8185,6 +8253,11 @@ def _pashub_main_heating_system_code(main: PasHubMainHeating) -> Optional[int]:
|
|||
f"{main.community_heat_source!r} (Heating System (Other))",
|
||||
)
|
||||
return code
|
||||
if system in _PASHUB_BOILER_SYSTEM_TYPES:
|
||||
# PCDF-Search boiler → None (SEDBUK product efficiency); Manual-Entry
|
||||
# boiler → its Table 4b code (or None for an uncovered/non-Table-4b
|
||||
# descriptor, keeping the pre-existing cascade) (#1649 A).
|
||||
return _pashub_manual_boiler_table_4b_code(main)
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1957,6 +1957,7 @@ class TestFromSiteNotesExample1:
|
|||
has_fixed_air_conditioning=False,
|
||||
cylinder_size=2, # "Normal (90-130 litres)" → Table 28 code 2
|
||||
cylinder_insulation_type=1, # "Factory fitted" → foam, SAP10 code 1
|
||||
cylinder_thermostat="Y", # "Has thermostat? Yes" → interlock (#1649 F)
|
||||
cylinder_insulation_thickness_mm=12,
|
||||
water_heating_code=901, # cylinder + "From main heating 1" → WHC 901
|
||||
shower_outlets=ShowerOutlets(
|
||||
|
|
@ -3000,3 +3001,250 @@ class TestPasHubUnmappedHeatEmitter:
|
|||
# Act / Assert
|
||||
with pytest.raises(UnmappedPasHubLabel, match="Skirting heaters"):
|
||||
EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
|
||||
class TestPasHubCylinderThermostat:
|
||||
"""The surveyed hot-water cylinder thermostat ("Has thermostat? Yes") must
|
||||
reach `sap_heating.cylinder_thermostat` — dropped (left None), the
|
||||
calculator reads `!= "Y"` and applies the SAP 10.2 Table 2b Note a) ×1.3
|
||||
absent-thermostat cylinder-temperature-factor penalty (equivalently the
|
||||
Table 4c(2) −5pp boiler-interlock penalty), inflating stored-HW loss and
|
||||
under-rating every PasHub cylinder dwelling. The Elmhurst sibling maps it
|
||||
(`"Y" if cylinder_thermostat else "N"`); `from_site_notes` did not.
|
||||
Evidence: 52 Bucklow Drive (DB 62, ours 59.44 → 61.79) and 13 Kerne Grove
|
||||
(DB 67, ours 64.95 → 67.11) — the two worst DB-oracle under-raters (#1649 F).
|
||||
"""
|
||||
|
||||
def test_surveyed_cylinder_thermostat_yes_maps_to_Y(self) -> None:
|
||||
# Arrange — example1 lodges a cylinder ("Normal (90-130 litres)") with
|
||||
# "Has thermostat? Yes".
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
assert data["heating_and_hot_water"]["water_heating"]["has_thermostat"] is True
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act
|
||||
result = EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
# Assert
|
||||
assert result.sap_heating.cylinder_thermostat == "Y"
|
||||
|
||||
def test_surveyed_cylinder_thermostat_no_maps_to_N(self) -> None:
|
||||
# Arrange
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
data["heating_and_hot_water"]["water_heating"]["has_thermostat"] = False
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act
|
||||
result = EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
# Assert
|
||||
assert result.sap_heating.cylinder_thermostat == "N"
|
||||
|
||||
def test_no_cylinder_leaves_thermostat_none(self) -> None:
|
||||
# Arrange — a combi/no-cylinder dwelling has no cylinder thermostat to
|
||||
# lodge; the field must stay None (not "N"), so the calculator's
|
||||
# cylinder-loss path is not spuriously engaged.
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
data["heating_and_hot_water"]["water_heating"]["cylinder_size"] = "No Cylinder"
|
||||
data["heating_and_hot_water"]["water_heating"]["has_thermostat"] = None
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act
|
||||
result = EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
# Assert
|
||||
assert result.sap_heating.cylinder_thermostat is None
|
||||
|
||||
|
||||
class TestPasHubRoomInRoofGableNoneDefaultsParty:
|
||||
"""A survey-form variant that omits the room-in-roof gable-type field lodges
|
||||
a genuine `gable_type=None` (field-absence, not a truncation or a new
|
||||
label). The mapper must default it to the party `gable_wall` (U=0.25),
|
||||
mirroring the gov-API sibling `_api_type_1_gable_kind(None)` — a
|
||||
present-but-unrecognised label still strict-raises. Left raising, the
|
||||
fixture is uncomputable (500771973367 8 Bucklow Drive, 500759637226 34
|
||||
Bucklow Drive both BLOCKED; #1649 B). Spec: SAP 10.2 Table 4 p.22 (party
|
||||
wall U=0.25).
|
||||
"""
|
||||
|
||||
_RIR_GABLE_NONE = {
|
||||
"age_range": "C: 1930 - 1949",
|
||||
"floor_area_m2": 21.0,
|
||||
# The form variant lodges length/height but no "Gable wall N type:" line
|
||||
# → gable_type is a genuine None.
|
||||
"gables": [
|
||||
{"length_m": 4.76, "height_m": 2.32},
|
||||
{"length_m": 3.16, "height_m": 2.32},
|
||||
],
|
||||
"slopes": [{"length_m": 4.76, "height_m": 2.5}],
|
||||
}
|
||||
|
||||
def test_gable_type_none_defaults_to_party_gable_wall(self) -> None:
|
||||
# Arrange
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
data["roof_space"]["main_building"]["rooms_in_roof"] = True
|
||||
data["roof_space"]["main_building"]["room_in_roof"] = self._RIR_GABLE_NONE
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act — must not raise
|
||||
rir = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[
|
||||
0
|
||||
].sap_room_in_roof
|
||||
|
||||
# Assert — both unlodged-type gables default to the party gable_wall.
|
||||
assert rir is not None
|
||||
assert rir.detailed_surfaces is not None
|
||||
gable_kinds = [
|
||||
s.kind for s in rir.detailed_surfaces if s.kind.startswith("gable_wall")
|
||||
]
|
||||
assert gable_kinds == ["gable_wall", "gable_wall"]
|
||||
|
||||
def test_present_but_unknown_gable_label_still_raises(self) -> None:
|
||||
# Arrange — a lodged-but-unrecognised label is a real coverage gap and
|
||||
# must still fail loud (the None default must not swallow it).
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
rir = {
|
||||
**self._RIR_GABLE_NONE,
|
||||
"gables": [{"length_m": 4.76, "height_m": 2.32, "gable_type": "Mystery"}],
|
||||
}
|
||||
data["roof_space"]["main_building"]["rooms_in_roof"] = True
|
||||
data["roof_space"]["main_building"]["room_in_roof"] = rir
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act / Assert
|
||||
with pytest.raises(UnmappedPasHubLabel, match="Mystery"):
|
||||
EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
|
||||
class TestPasHubSecondaryClosedRoomHeater:
|
||||
"""The secondary-heating label "Closed room heater" must map to SAP 10.2
|
||||
Table 4a code 633 (solid-fuel closed room heater). An uncovered label
|
||||
strict-raises and blocks the fixture (500796700868 8 Brinkshaw Avenue;
|
||||
#1649 C). 633 is already in `_ELMHURST_SECONDARY_SOLID_CODES {631..636}`,
|
||||
so downstream fuel/category resolution already covers it.
|
||||
"""
|
||||
|
||||
def test_closed_room_heater_maps_to_633(self) -> None:
|
||||
# Arrange
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
data["heating_and_hot_water"]["secondary_heating"][
|
||||
"secondary_system"
|
||||
] = "Closed room heater"
|
||||
data["heating_and_hot_water"]["secondary_heating"]["secondary_fuel"] = (
|
||||
"Dual Fuel Appliance (Mineral and Wood)"
|
||||
)
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act — must not raise
|
||||
result = EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
# Assert
|
||||
assert result.sap_heating.secondary_heating_type == 633
|
||||
|
||||
|
||||
class TestPasHubManualBoilerEfficiency:
|
||||
"""A Manual-Entry boiler (no PCDB `product_id`) must resolve its surveyed
|
||||
descriptor to a SAP 10.2 Table 4b code so the calculator bills the Table 4b
|
||||
seasonal efficiency, not the generic 0.80 "no data" gas default. Left None,
|
||||
a back boiler (Table 4b 66%) is credited 80% (500796377318 4a Hollyhedge:
|
||||
DB 53, ours 63.6 → 53.4) and a condensing combi's water-heating summer
|
||||
efficiency (75%) is over-credited (the 84 Button / 91 Alderue / 17 Kennett
|
||||
/ 43 Kenninghall / 156 Gladeside over-raters). #1649 A.
|
||||
|
||||
A PCDF-Search boiler keeps `None` (its efficiency is the SEDBUK product
|
||||
lookup); a non-Table-4b descriptor (e.g. electric) also keeps `None`
|
||||
(unchanged behaviour — no regression).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _manual_boiler(data: Dict[str, Any], *, boiler_type: str, age_band: str) -> Any:
|
||||
mh = data["heating_and_hot_water"]["main_heating"]
|
||||
mh["selection_method"] = "Manual Entry"
|
||||
mh["system_type"] = "Boiler with radiators or underfloor heating"
|
||||
mh["product_id"] = 0
|
||||
mh["fuel"] = "Mains Gas"
|
||||
mh["boiler_type"] = boiler_type
|
||||
mh["boiler_age_band"] = age_band
|
||||
return from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
def test_manual_back_boiler_post_1998_maps_to_109(self) -> None:
|
||||
survey = self._manual_boiler(
|
||||
load("pashub_rdsap_site_notes_example1.json"),
|
||||
boiler_type="Back boiler",
|
||||
age_band="1998 or later - Back boiler",
|
||||
)
|
||||
main = EpcPropertyDataMapper.from_site_notes(
|
||||
survey
|
||||
).sap_heating.main_heating_details[0]
|
||||
assert main.sap_main_heating_code == 109
|
||||
|
||||
def test_manual_condensing_combi_maps_to_104(self) -> None:
|
||||
survey = self._manual_boiler(
|
||||
load("pashub_rdsap_site_notes_example1.json"),
|
||||
boiler_type="Condensing combi",
|
||||
age_band="1998 or later - Condensing combi",
|
||||
)
|
||||
main = EpcPropertyDataMapper.from_site_notes(
|
||||
survey
|
||||
).sap_heating.main_heating_details[0]
|
||||
assert main.sap_main_heating_code == 104
|
||||
|
||||
def test_pcdf_boiler_keeps_none(self) -> None:
|
||||
# example1 as lodged is a PCDF-Search boiler (product_id 18400): its
|
||||
# efficiency is the SEDBUK lookup, so the Table 4a code stays None.
|
||||
survey = from_dict(
|
||||
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
|
||||
)
|
||||
main = EpcPropertyDataMapper.from_site_notes(
|
||||
survey
|
||||
).sap_heating.main_heating_details[0]
|
||||
assert main.sap_main_heating_code is None
|
||||
|
||||
def test_manual_boiler_unmapped_descriptor_keeps_none(self) -> None:
|
||||
# An electric "Direct acting" descriptor is not a Table 4b gas/liquid
|
||||
# boiler — keep None (unchanged 0.80 behaviour, no regression / no block).
|
||||
survey = self._manual_boiler(
|
||||
load("pashub_rdsap_site_notes_example1.json"),
|
||||
boiler_type="Standard",
|
||||
age_band="Direct acting",
|
||||
)
|
||||
main = EpcPropertyDataMapper.from_site_notes(
|
||||
survey
|
||||
).sap_heating.main_heating_details[0]
|
||||
assert main.sap_main_heating_code is None
|
||||
|
||||
|
||||
class TestPasHubAlternativeWallInsulationThickness:
|
||||
"""A surveyed alternative-wall "Wall insulation thickness:" must reach
|
||||
`SapAlternativeWall.wall_insulation_thickness`; dropped, the cascade takes
|
||||
the RdSAP 10 §5.4 "assume 100 mm if unknown" favourable default instead of
|
||||
the surveyed value (#1649 D). Mirrors the gov-API / Elmhurst paths.
|
||||
"""
|
||||
|
||||
def test_alt_wall_insulation_thickness_reaches_sap_alt_wall(self) -> None:
|
||||
# Arrange — a main-building alt wall lodging a 50 mm insulation thickness.
|
||||
data = load("pashub_rdsap_site_notes_example1.json")
|
||||
data["alternative_walls"] = [
|
||||
{
|
||||
"location": "Main Building",
|
||||
"id": 1,
|
||||
"construction_type": "Solid brick",
|
||||
"insulation_type": "External",
|
||||
"area_m2": 13.71,
|
||||
"wall_u_value_known": False,
|
||||
"thermal_conductivity_of_wall_insulation": "Unknown",
|
||||
"wall_thickness_mm": 250,
|
||||
"dry_lined": False,
|
||||
"insulation_thickness_mm": 50,
|
||||
}
|
||||
]
|
||||
survey = from_dict(PasHubRdSapSiteNotes, data)
|
||||
|
||||
# Act
|
||||
result = EpcPropertyDataMapper.from_site_notes(survey)
|
||||
|
||||
# Assert — the surveyed 50 mm reaches the SAP alt wall (string wiring,
|
||||
# like the main wall) instead of the §5.4 100 mm favourable default.
|
||||
alt = result.sap_building_parts[0].sap_alternative_wall_1
|
||||
assert alt is not None
|
||||
assert alt.wall_insulation_thickness == "50"
|
||||
|
|
|
|||
|
|
@ -232,6 +232,16 @@ class MainHeating:
|
|||
# "Water or oil-filled radiators"); blank when the type is fixed elsewhere
|
||||
# (a PCDF-Search product, or a boiler). See #1619.
|
||||
community_heat_source: str = ""
|
||||
# Manual-Entry (non-PCDF) boiler descriptor. A boiler selected by "Manual
|
||||
# Entry" (product_id 0) carries no SEDBUK record, so its efficiency must
|
||||
# come from SAP 10.2 Table 4b keyed on these two surveyed labels rather than
|
||||
# the generic 0.80 gas default (#1649 A). `boiler_type` is the §Heating
|
||||
# "Type of boiler:" cell (e.g. "Condensing combi" / "Back boiler"),
|
||||
# `boiler_age_band` the "Heating System (Boiler):" cell, which carries the
|
||||
# pre/post-1998 age band (e.g. "1998 or later - Condensing combi"). Blank on
|
||||
# a PCDF-Search boiler (efficiency is the product lookup).
|
||||
boiler_type: str = ""
|
||||
boiler_age_band: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -369,6 +379,10 @@ class AlternativeWall:
|
|||
thermal_conductivity_of_wall_insulation: str
|
||||
wall_thickness_mm: Optional[int] = None
|
||||
dry_lined: Optional[bool] = None
|
||||
# Surveyed "Wall insulation thickness:" (mm) for the alternative wall.
|
||||
# Dropped, the cascade takes the RdSAP 10 §5.4 "assume 100 mm if unknown"
|
||||
# favourable default instead of the lodged value (#1649 D).
|
||||
insulation_thickness_mm: Optional[int] = None
|
||||
|
||||
|
||||
def _no_alternative_walls() -> List[AlternativeWall]:
|
||||
|
|
|
|||
134
scripts/enrich_pashub_wythenshawe_oracle.py
Normal file
134
scripts/enrich_pashub_wythenshawe_oracle.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""Enrich the PasHub Wythenshawe fixture manifest with the DB-oracle SAP.
|
||||
|
||||
The accuracy harness must grade against the reliable oracle — the PasHub
|
||||
company-assessment result stored in the DB (`epc_property_energy_performance.
|
||||
energy_rating_current`, `ep.source='lodged'`) — NOT the manifest `pre_sap`
|
||||
(`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently wrong). This
|
||||
one-time / rebuild-able provenance script resolves each fixture to that DB SAP
|
||||
and writes `oracle_sap` (int or null) + `oracle_match` ("uprn" / "address" /
|
||||
null) back into `manifest.json`, so the committed manifest is what the test
|
||||
consumes and CI needs no DB access. Mirrors `build_pashub_accuracy_fixtures.py`.
|
||||
|
||||
Match order: manifest `uprn` (bigint, exact) → else postcode + house-number
|
||||
parsed from the PDF "Property Address:" block. Latest cert per key by `ep.id`.
|
||||
|
||||
Requires DB credentials (reads `backend/.env`); NOT run in CI.
|
||||
|
||||
python scripts/enrich_pashub_wythenshawe_oracle.py [--dry-run]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.e2e_common import build_engine, load_env # noqa: E402
|
||||
from sqlalchemy import text # noqa: E402
|
||||
from sqlalchemy.engine import Connection # noqa: E402
|
||||
|
||||
from backend.documents_parser.parser import pdf_to_text_list # noqa: E402
|
||||
|
||||
_FIXTURES = (
|
||||
REPO_ROOT
|
||||
/ "backend"
|
||||
/ "documents_parser"
|
||||
/ "tests"
|
||||
/ "fixtures"
|
||||
/ "pashub_accuracy_wythenshawe"
|
||||
)
|
||||
_MANIFEST = _FIXTURES / "manifest.json"
|
||||
_POSTCODE_RE = re.compile(r"^[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}$")
|
||||
_HOUSENUM_RE = re.compile(r"^(\d+[A-Za-z]?)\b")
|
||||
|
||||
|
||||
def _address_from_pdf(deal_id: str) -> tuple[Optional[str], Optional[str]]:
|
||||
"""(house_number, postcode) from the PDF "Property Address:" block."""
|
||||
tl = pdf_to_text_list((_FIXTURES / f"{deal_id}.pdf").read_bytes())
|
||||
try:
|
||||
i = next(k for k, l in enumerate(tl) if l.strip() == "Property Address:")
|
||||
except StopIteration:
|
||||
return None, None
|
||||
block = [l.strip().rstrip(",") for l in tl[i + 1 : i + 6] if l.strip()]
|
||||
postcode = next((l for l in block if _POSTCODE_RE.match(l.upper())), None)
|
||||
house = None
|
||||
if block:
|
||||
m = _HOUSENUM_RE.match(block[0])
|
||||
if m:
|
||||
house = m.group(1)
|
||||
return house, postcode
|
||||
|
||||
|
||||
def _resolve(
|
||||
conn: Connection, uprn: Optional[int], deal_id: str
|
||||
) -> tuple[Optional[int], str]:
|
||||
if uprn:
|
||||
row = conn.execute(
|
||||
text(
|
||||
"select eppe.energy_rating_current from epc_property ep "
|
||||
"join epc_property_energy_performance eppe "
|
||||
"on eppe.epc_property_id = ep.id "
|
||||
"where ep.source='lodged' and ep.uprn=:u "
|
||||
"and eppe.energy_rating_current is not null "
|
||||
"order by ep.id desc limit 1"
|
||||
),
|
||||
{"u": int(uprn)},
|
||||
).scalar()
|
||||
if row is not None:
|
||||
return int(row), "uprn"
|
||||
house, postcode = _address_from_pdf(deal_id)
|
||||
if house and postcode:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"select ep.address_line_1, eppe.energy_rating_current "
|
||||
"from epc_property ep join epc_property_energy_performance eppe "
|
||||
"on eppe.epc_property_id = ep.id "
|
||||
"where ep.source='lodged' "
|
||||
"and replace(upper(ep.postcode),' ','')=:pc "
|
||||
"and eppe.energy_rating_current is not null order by ep.id desc"
|
||||
),
|
||||
{"pc": postcode.upper().replace(" ", "")},
|
||||
).fetchall()
|
||||
for row_pair in rows:
|
||||
a1 = str(row_pair[0] or "").strip()
|
||||
m = _HOUSENUM_RE.match(a1)
|
||||
if m and m.group(1).lower() == house.lower():
|
||||
return int(row_pair[1]), "address"
|
||||
return None, "unmatched"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
load_env(REPO_ROOT / "backend" / ".env")
|
||||
engine = build_engine()
|
||||
manifest = json.loads(_MANIFEST.read_text())
|
||||
|
||||
matched = 0
|
||||
with engine.connect() as conn:
|
||||
for fx in manifest["fixtures"]:
|
||||
oracle_sap, kind = _resolve(conn, fx.get("uprn"), fx["deal_id"])
|
||||
fx["oracle_sap"] = oracle_sap
|
||||
fx["oracle_match"] = None if kind == "unmatched" else kind
|
||||
if oracle_sap is not None:
|
||||
matched += 1
|
||||
|
||||
n = len(manifest["fixtures"])
|
||||
print(f"resolved {matched}/{n} fixtures to a DB oracle SAP")
|
||||
if args.dry_run:
|
||||
print("--dry-run: manifest NOT written")
|
||||
return
|
||||
_MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
print(f"wrote {_MANIFEST}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue