diff --git a/.claude/skills/epc-to-elmhurst-rdsap-inputs/SKILL.md b/.claude/skills/epc-to-elmhurst-rdsap-inputs/SKILL.md index 65fa75267..85050e6ed 100644 --- a/.claude/skills/epc-to-elmhurst-rdsap-inputs/SKILL.md +++ b/.claude/skills/epc-to-elmhurst-rdsap-inputs/SKILL.md @@ -115,6 +115,16 @@ one-line reason. Absent ≠ zero everywhere — note which. - **Water heating** 903 = Electric immersion off-peak → Elmhurst "Water Heater" category, not "Boiler Circulator" (901 = from main system). - **Windows** are synthesised from `glazed_area` band × TFA — not real geometry. +- **Room height — give the RAW surveyed room height, NOT the engine's storey + height.** Elmhurst adds the RdSAP +0.25 m upper-storey (joist-void) allowance + itself. Our mapper already applies that +0.25 to upper floors + (`_UPPER_FLOOR_HEIGHT_ADD_M`), so `sap_floor_dimensions[*].room_height_m` for + a first floor is *already* `raw + 0.25`. Reading that value onto the sheet + makes Elmhurst add a SECOND 0.25 → storey height too tall → ~4-5 m² extra wall + + volume → **~1 SAP too low** (it under-scored 6 Barry Road 60→59 until the + first-floor height was corrected from 2.76 back to the raw 2.51). Pull heights + from the survey/raw `building_measurements`, not the transformed + `sap_floor_dimensions`; the ground/lowest floor takes the raw value either way. - **Age band — pick by construction YEAR, not the engine's band letter.** The live RdSAP-10 tool bands differently from `mapping.md`'s older table: it offers `…K 2007-2011`, **`L 2012-2022`**, **`M 2023 onwards`**. A 2020 build diff --git a/backend/documents_parser/extractor.py b/backend/documents_parser/extractor.py index d06017dba..c9f26e254 100644 --- a/backend/documents_parser/extractor.py +++ b/backend/documents_parser/extractor.py @@ -481,7 +481,9 @@ class PasHubRdSapSiteNotesExtractor: def extract_room_count_elements(self) -> RoomCountElements: rce_section = self._section("Room Count Elements", "Customer Response") - heated_rooms_raw = self._get_in(rce_section, "Number of heated rooms?") + heated_rooms_raw = self._get_in( + rce_section, "Please enter the number of HEATED rooms:" + ) return RoomCountElements( number_of_habitable_rooms=int( self._get_in(rce_section, "Number of habitable rooms?") or 0 @@ -624,7 +626,11 @@ class PasHubRdSapSiteNotesExtractor: condensing=self._bool_in(data, "Condensing"), year=self._get_in(data, "Year") or "", mount=self._get_in(data, "Mount") or "", - open_flue=self._get_in(data, "Open Flue") or "", + # PCDF-Search "Open Flue" column wins; a Manual-Entry boiler instead + # lodges the flue under "Is there an open flue?" (Yes/No) (#1649 A). + open_flue=self._get_in(data, "Open Flue") + or self._get_in(data, "Is there an open flue?") + or "", fan_assist=self._bool_in(data, "Fan Assist"), status=self._get_in(data, "Status") or "", central_heating_pump_age=self._get_in(data, "Central heating pump age:") @@ -636,6 +642,12 @@ class PasHubRdSapSiteNotesExtractor: weather_compensator=self._bool_in(data, "Is there a weather compensator?"), emitter=self._get_in(data, "Emitter:") or "", emitter_temperature=self._get_in(data, "Emitter Temperature:") or "", + # Manual-Entry boiler descriptor (blank on a PCDF-Search boiler): + # "Type of boiler:" and "Heating System (Boiler):" (the latter + # carries the pre/post-1998 age band) → the mapper's Table 4b + # efficiency lookup (#1649 A). + boiler_type=self._get_in(data, "Type of boiler:") or "", + boiler_age_band=self._get_in(data, "Heating System (Boiler):") or "", ) def _parse_secondary_heating(self, data: List[str]) -> SecondaryHeating: @@ -865,4 +877,9 @@ class PasHubRdSapSiteNotesExtractor: or "", wall_thickness_mm=self._wall_thickness_in(data), dry_lined=self._optional_bool_in(data, "Wall Dry-Lined?"), + # Surveyed alt-wall insulation thickness ("50 mm") → the mapper's + # §5.4 thickness input; dropped, the cascade assumes 100 mm (#1649 D). + insulation_thickness_mm=self._parse_insulation_thickness( + self._get_in(data, "Wall insulation thickness:") + )[0], ) diff --git a/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json b/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json index ac81df84e..c1a601e57 100644 --- a/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json +++ b/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json @@ -8,1176 +8,1512 @@ "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500671707334.pdf" + "pdf": "500671707334.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500671707335", "uprn": "000077046521", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500671707335.pdf" + "pdf": "500671707335.pdf", + "oracle_sap": 64, + "oracle_match": "uprn" }, { "deal_id": "500671707336", "uprn": "000077053410", "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500671707336.pdf" + "pdf": "500671707336.pdf", + "oracle_sap": 59, + "oracle_match": "uprn" }, { "deal_id": "500671707337", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500671707337.pdf" + "pdf": "500671707337.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500677104834", "uprn": "000077050547", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500677104834.pdf" + "pdf": "500677104834.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500717664451", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500717664451.pdf" + "pdf": "500717664451.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500717664452", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500717664452.pdf" + "pdf": "500717664452.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500717664453", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500717664453.pdf" + "pdf": "500717664453.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500717664454", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500717664454.pdf" + "pdf": "500717664454.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500717664455", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500717664455.pdf" + "pdf": "500717664455.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500719659230", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500719659230.pdf" + "pdf": "500719659230.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500719659233", "uprn": null, "pre_sap_raw": "D56", "pre_sap_score": 56, - "pdf": "500719659233.pdf" + "pdf": "500719659233.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500719659234", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500719659234.pdf" + "pdf": "500719659234.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500719659235", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500719659235.pdf" + "pdf": "500719659235.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500721469635", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500721469635.pdf" + "pdf": "500721469635.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500721469636", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500721469636.pdf" + "pdf": "500721469636.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500738218187", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500738218187.pdf" + "pdf": "500738218187.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500738218188", "uprn": "000077073680", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500738218188.pdf" + "pdf": "500738218188.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500738218190", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500738218190.pdf" + "pdf": "500738218190.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500752190667", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500752190667.pdf" + "pdf": "500752190667.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500752190668", "uprn": "000077051200", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500752190668.pdf" + "pdf": "500752190668.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500752190669", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500752190669.pdf" + "pdf": "500752190669.pdf", + "oracle_sap": 59, + "oracle_match": "address" }, { "deal_id": "500752190670", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500752190670.pdf" + "pdf": "500752190670.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500752190671", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500752190671.pdf" + "pdf": "500752190671.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500752190672", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500752190672.pdf" + "pdf": "500752190672.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500759637226", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500759637226.pdf" + "pdf": "500759637226.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500761275639", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500761275639.pdf" + "pdf": "500761275639.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500761275640", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500761275640.pdf" + "pdf": "500761275640.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500762478811", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500762478811.pdf" + "pdf": "500762478811.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500762478813", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500762478813.pdf" + "pdf": "500762478813.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500764850382", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500764850382.pdf" + "pdf": "500764850382.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500764850384", "uprn": "000077052680", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500764850384.pdf" + "pdf": "500764850384.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500764850386", "uprn": "000077051448", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500764850386.pdf" + "pdf": "500764850386.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500764850389", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500764850389.pdf" + "pdf": "500764850389.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500766759115", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500766759115.pdf" + "pdf": "500766759115.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500766759116", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500766759116.pdf" + "pdf": "500766759116.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500766759117", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500766759117.pdf" + "pdf": "500766759117.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500766759119", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500766759119.pdf" + "pdf": "500766759119.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500766759121", "uprn": "000077045339", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500766759121.pdf" + "pdf": "500766759121.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500767954150", "uprn": "000077046582", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500767954150.pdf" + "pdf": "500767954150.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500767954151", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500767954151.pdf" + "pdf": "500767954151.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500767954152", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500767954152.pdf" + "pdf": "500767954152.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500767954153", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500767954153.pdf" + "pdf": "500767954153.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500767954154", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500767954154.pdf" + "pdf": "500767954154.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500767954156", "uprn": null, "pre_sap_raw": "F30", "pre_sap_score": 30, - "pdf": "500767954156.pdf" + "pdf": "500767954156.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500770361585", "uprn": null, "pre_sap_raw": "D57", "pre_sap_score": 57, - "pdf": "500770361585.pdf" + "pdf": "500770361585.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500770361586", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500770361586.pdf" + "pdf": "500770361586.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500770361587", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500770361587.pdf" + "pdf": "500770361587.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500771973359", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500771973359.pdf" + "pdf": "500771973359.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500771973360", "uprn": "000077051178", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500771973360.pdf" + "pdf": "500771973360.pdf", + "oracle_sap": 64, + "oracle_match": "uprn" }, { "deal_id": "500771973361", "uprn": "000077045449", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500771973361.pdf" + "pdf": "500771973361.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500771973362", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500771973362.pdf" + "pdf": "500771973362.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500771973363", "uprn": "000077044951", "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500771973363.pdf" + "pdf": "500771973363.pdf", + "oracle_sap": 70, + "oracle_match": "uprn" }, { "deal_id": "500771973364", "uprn": "000077053171", "pre_sap_raw": "D57", "pre_sap_score": 57, - "pdf": "500771973364.pdf" + "pdf": "500771973364.pdf", + "oracle_sap": 57, + "oracle_match": "uprn" }, { "deal_id": "500771973365", "uprn": "000077050562", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500771973365.pdf" + "pdf": "500771973365.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500771973366", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500771973366.pdf" + "pdf": "500771973366.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500771973367", "uprn": null, "pre_sap_raw": "E46", "pre_sap_score": 46, - "pdf": "500771973367.pdf" + "pdf": "500771973367.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500774016249", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500774016249.pdf" + "pdf": "500774016249.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500796290249", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796290249.pdf" + "pdf": "500796290249.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500796290250", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500796290250.pdf" + "pdf": "500796290250.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500796377309", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796377309.pdf" + "pdf": "500796377309.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500796377310", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796377310.pdf" + "pdf": "500796377310.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500796377311", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500796377311.pdf" + "pdf": "500796377311.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500796377312", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796377312.pdf" + "pdf": "500796377312.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500796377314", "uprn": "000077042251", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796377314.pdf" + "pdf": "500796377314.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500796377316", "uprn": "000077053166", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377316.pdf" + "pdf": "500796377316.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500796377317", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500796377317.pdf" + "pdf": "500796377317.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500796377318", "uprn": null, "pre_sap_raw": "E53", "pre_sap_score": 53, - "pdf": "500796377318.pdf" + "pdf": "500796377318.pdf", + "oracle_sap": 53, + "oracle_match": "address" }, { "deal_id": "500796377319", "uprn": "000077053551", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377319.pdf" + "pdf": "500796377319.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500796377320", "uprn": "000077051205", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377320.pdf" + "pdf": "500796377320.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500796481781", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500796481781.pdf" + "pdf": "500796481781.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500796646641", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500796646641.pdf" + "pdf": "500796646641.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500796646642", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796646642.pdf" + "pdf": "500796646642.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500796646644", "uprn": null, "pre_sap_raw": "F34", "pre_sap_score": 34, - "pdf": "500796646644.pdf" + "pdf": "500796646644.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500796700866", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500796700866.pdf" + "pdf": "500796700866.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500796700867", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500796700867.pdf" + "pdf": "500796700867.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500796700868", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796700868.pdf" + "pdf": "500796700868.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797048053", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797048053.pdf" + "pdf": "500797048053.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797048054", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797048054.pdf" + "pdf": "500797048054.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200599", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797200599.pdf" + "pdf": "500797200599.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200604", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797200604.pdf" + "pdf": "500797200604.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200605", "uprn": "000077080305", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797200605.pdf" + "pdf": "500797200605.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797312247", "uprn": "000077050541", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797312247.pdf" + "pdf": "500797312247.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797312248", "uprn": "000077053227", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797312248.pdf" + "pdf": "500797312248.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500797312249", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797312249.pdf" + "pdf": "500797312249.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797313211", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797313211.pdf" + "pdf": "500797313211.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500797383868", "uprn": "000077045425", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797383868.pdf" + "pdf": "500797383868.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797383869", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500797383869.pdf" + "pdf": "500797383869.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500797383870", "uprn": "000077051737", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500797383870.pdf" + "pdf": "500797383870.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500797465814", "uprn": null, "pre_sap_raw": "E41", "pre_sap_score": 41, - "pdf": "500797465814.pdf" + "pdf": "500797465814.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797523170", "uprn": null, "pre_sap_raw": "C72", "pre_sap_score": 72, - "pdf": "500797523170.pdf" + "pdf": "500797523170.pdf", + "oracle_sap": 72, + "oracle_match": "address" }, { "deal_id": "500797523171", "uprn": "000077050551", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797523171.pdf" + "pdf": "500797523171.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500797523172", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797523172.pdf" + "pdf": "500797523172.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797523173", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797523173.pdf" + "pdf": "500797523173.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797523174", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797523174.pdf" + "pdf": "500797523174.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797668540", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500797668540.pdf" + "pdf": "500797668540.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500797738231", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797738231.pdf" + "pdf": "500797738231.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797738233", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500797738233.pdf" + "pdf": "500797738233.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500797739194", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500797739194.pdf" + "pdf": "500797739194.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797739195", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797739195.pdf" + "pdf": "500797739195.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500797739196", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739196.pdf" + "pdf": "500797739196.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739197", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739197.pdf" + "pdf": "500797739197.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739198", "uprn": "000077051347", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739198.pdf" + "pdf": "500797739198.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500797739199", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739199.pdf" + "pdf": "500797739199.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739200", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797739200.pdf" + "pdf": "500797739200.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797805805", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500797805805.pdf" + "pdf": "500797805805.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500797805807", "uprn": null, "pre_sap_raw": "F31", "pre_sap_score": 31, - "pdf": "500797805807.pdf" + "pdf": "500797805807.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797805809", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797805809.pdf" + "pdf": "500797805809.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500797805810", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797805810.pdf" + "pdf": "500797805810.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500797805811", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797805811.pdf" + "pdf": "500797805811.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797875440", "uprn": "000077050090", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797875440.pdf" + "pdf": "500797875440.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500797962466", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797962466.pdf" + "pdf": "500797962466.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797962469", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797962469.pdf" + "pdf": "500797962469.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797962470", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797962470.pdf" + "pdf": "500797962470.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797962471", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797962471.pdf" + "pdf": "500797962471.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797995201", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797995201.pdf" + "pdf": "500797995201.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797995202", "uprn": "000077073032", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797995202.pdf" + "pdf": "500797995202.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500797995203", "uprn": "000077051983", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797995203.pdf" + "pdf": "500797995203.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500798076127", "uprn": "000077046459", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500798076127.pdf" + "pdf": "500798076127.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500798076128", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500798076128.pdf" + "pdf": "500798076128.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500798076130", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500798076130.pdf" + "pdf": "500798076130.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500798148835", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798148835.pdf" + "pdf": "500798148835.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500798148836", "uprn": "000077045469", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798148836.pdf" + "pdf": "500798148836.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500798148837", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500798148837.pdf" + "pdf": "500798148837.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500798148839", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500798148839.pdf" + "pdf": "500798148839.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500798148841", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798148841.pdf" + "pdf": "500798148841.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798341369", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798341369.pdf" + "pdf": "500798341369.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500798342331", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500798342331.pdf" + "pdf": "500798342331.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500798527676", "uprn": "000077050055", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500798527676.pdf" + "pdf": "500798527676.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500798527677", "uprn": "000077051819", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500798527677.pdf" + "pdf": "500798527677.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500798527678", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798527678.pdf" + "pdf": "500798527678.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500798620869", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798620869.pdf" + "pdf": "500798620869.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798620872", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500798620872.pdf" + "pdf": "500798620872.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500798620874", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798620874.pdf" + "pdf": "500798620874.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798620876", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500798620876.pdf" + "pdf": "500798620876.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500798745810", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500798745810.pdf" + "pdf": "500798745810.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500798745811", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798745811.pdf" + "pdf": "500798745811.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500798745812", "uprn": "000077045588", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798745812.pdf" + "pdf": "500798745812.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500798782667", "uprn": "000077053134", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798782667.pdf" + "pdf": "500798782667.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500798782668", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798782668.pdf" + "pdf": "500798782668.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801592506", "uprn": null, "pre_sap_raw": "C70", "pre_sap_score": 70, - "pdf": "500801592506.pdf" + "pdf": "500801592506.pdf", + "oracle_sap": 70, + "oracle_match": "address" }, { "deal_id": "500801592509", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500801592509.pdf" + "pdf": "500801592509.pdf", + "oracle_sap": 59, + "oracle_match": "address" }, { "deal_id": "500801665243", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801665243.pdf" + "pdf": "500801665243.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801665245", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500801665245.pdf" + "pdf": "500801665245.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500801726702", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801726702.pdf" + "pdf": "500801726702.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801726703", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801726703.pdf" + "pdf": "500801726703.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801726704", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801726704.pdf" + "pdf": "500801726704.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801726708", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500801726708.pdf" + "pdf": "500801726708.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801726709", "uprn": null, "pre_sap_raw": "C70", "pre_sap_score": 70, - "pdf": "500801726709.pdf" + "pdf": "500801726709.pdf", + "oracle_sap": 70, + "oracle_match": "address" }, { "deal_id": "500801796315", "uprn": "000077046508", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801796315.pdf" + "pdf": "500801796315.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500801796316", "uprn": "000077051312", "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500801796316.pdf" + "pdf": "500801796316.pdf", + "oracle_sap": 59, + "oracle_match": "uprn" }, { "deal_id": "500801796318", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801796318.pdf" + "pdf": "500801796318.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801853637", "uprn": "000077045510", "pre_sap_raw": "C72", "pre_sap_score": 72, - "pdf": "500801853637.pdf" + "pdf": "500801853637.pdf", + "oracle_sap": 72, + "oracle_match": "uprn" }, { "deal_id": "500801853638", "uprn": "000077053376", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801853638.pdf" + "pdf": "500801853638.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801853639", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500801853639.pdf" + "pdf": "500801853639.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801853640", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801853640.pdf" + "pdf": "500801853640.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801853641", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801853641.pdf" + "pdf": "500801853641.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801853642", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801853642.pdf" + "pdf": "500801853642.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801924329", "uprn": "000077042973", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924329.pdf" + "pdf": "500801924329.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500801924330", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924330.pdf" + "pdf": "500801924330.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500801924332", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801924332.pdf" + "pdf": "500801924332.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801924334", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801924334.pdf" + "pdf": "500801924334.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801924336", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924336.pdf" + "pdf": "500801924336.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500801964238", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801964238.pdf" + "pdf": "500801964238.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801964241", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801964241.pdf" + "pdf": "500801964241.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801964242", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500801964242.pdf" + "pdf": "500801964242.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801964243", "uprn": "000077042536", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500801964243.pdf" + "pdf": "500801964243.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500801964244", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801964244.pdf" + "pdf": "500801964244.pdf", + "oracle_sap": 62, + "oracle_match": "address" } ] } diff --git a/backend/documents_parser/tests/test_extractor.py b/backend/documents_parser/tests/test_extractor.py index 3a57fcb56..a855d8016 100644 --- a/backend/documents_parser/tests/test_extractor.py +++ b/backend/documents_parser/tests/test_extractor.py @@ -329,6 +329,22 @@ class TestAlternativeWalls: ).extract_alternative_walls() assert walls == [] + def test_insulation_thickness_parses_to_mm(self) -> None: + # The surveyed "Wall insulation thickness:" line must reach + # `AlternativeWall.insulation_thickness_mm`; dropped, the cascade takes + # the RdSAP 10 §5.4 100 mm favourable default (#1649 D). + block = list(self._BLOCK) + i = block.index("Wall Dry-Lined?") + block[i:i] = ["Wall insulation thickness:", "50 mm"] + walls = PasHubRdSapSiteNotesExtractor(block).extract_alternative_walls() + assert walls[0].insulation_thickness_mm == 50 + + def test_no_insulation_thickness_line_lodges_none(self) -> None: + walls = PasHubRdSapSiteNotesExtractor( + list(self._BLOCK) + ).extract_alternative_walls() + assert walls[0].insulation_thickness_mm is None + class TestBuildingMeasurements: @pytest.fixture @@ -1407,3 +1423,83 @@ class TestSolidMasonryPartyWall: bc.main_building.party_wall_construction_type == "Solid Masonry, Timber Frame, or System Built" ) + + +class TestManualEntryBoilerDescriptor: + """A Manual-Entry (non-PCDF) boiler lodges its type/age via the labels + "Type of boiler:", "Heating System (Boiler):" (which carries the pre/post- + 1998 age band) and "Is there an open flue?" — not the PCDF-Search columns + ("Type"/"Year"/"Open Flue"). The extractor must read the manual labels so + the mapper can resolve the SAP 10.2 Table 4b efficiency; dropped, the boiler + descriptor is lost and the mapper falls to the generic 0.80 default + (#1649 A). Tokens spliced verbatim from deal 500796377318 (4a Hollyhedge).""" + + _SECTION = [ + "Heating & Hot Water", + "How would you like to select the Heating System?", + "Manual Entry", + "System type:", + "Boiler with radiators or underfloor heating", + "Fuel:", + "Mains Gas", + "Type of boiler:", + "Back boiler", + "Heating System (Boiler):", + "1998 or later - Back boiler", + "Is there an open flue?", + "Yes", + "Ventilation", + ] + + def test_manual_boiler_descriptor_captured(self) -> None: + # Act + main = PasHubRdSapSiteNotesExtractor( + self._SECTION + ).extract_heating_and_hot_water().main_heating + + # Assert + assert main.selection_method == "Manual Entry" + assert main.product_id == 0 + assert main.boiler_type == "Back boiler" + assert main.boiler_age_band == "1998 or later - Back boiler" + assert main.open_flue == "Yes" + + def test_pcdf_open_flue_column_still_wins(self) -> None: + # A PCDF-Search boiler carries the "Open Flue" column ("Room-sealed"); + # the manual "Is there an open flue?" fallback must not clobber it. + section = [ + "Heating & Hot Water", + "System type:", + "Boiler with radiators or underfloor heating", + "Open Flue", + "Room-sealed", + "Ventilation", + ] + main = PasHubRdSapSiteNotesExtractor( + section + ).extract_heating_and_hot_water().main_heating + assert main.open_flue == "Room-sealed" + + +class TestHeatedRoomsLabel: + """The PasHub PDF lodges the heated-room count under "Please enter the + number of HEATED rooms:", not "Number of heated rooms?" — the old query + silently dropped it to None (#1649 E).""" + + def _section(self) -> list[str]: + return [ + "Room Count Elements", + "Number of habitable rooms?", + "5", + "Are any of these rooms unheated?", + "Yes", + "Please enter the number of HEATED rooms:", + "3", + "Customer Response", + ] + + def test_heated_rooms_label_captured(self) -> None: + rce = PasHubRdSapSiteNotesExtractor( + self._section() + ).extract_room_count_elements() + assert rce.number_of_heated_rooms == 3 diff --git a/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py b/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py index 350cb7bc6..d32889708 100644 --- a/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py +++ b/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py @@ -1,17 +1,30 @@ """End-to-end accuracy test for the pashub site-notes extractor — WYTHENSHAWE. Sibling of `test_pashub_sap_accuracy.py` (the Guinness GMCA cohort); see that -module's docstring for the full chain and the meaning of the hybrid `pre_sap` -gauge. This module points at the `pashub_accuracy_wythenshawe/` fixtures built by +module's docstring for the full chain. This module points at the +`pashub_accuracy_wythenshawe/` fixtures built by python scripts/build_pashub_accuracy_fixtures.py \\ --project-code "[Wythenshawe WH:SHF RAs - WH:SF - 240426 - 494493718748]" \\ --out-dir pashub_accuracy_wythenshawe from `hubspot_deal_data` (the project_code above) + the `rd_sap_site_note` PDFs -in S3. The ratchets below are calibrated to THIS cohort — do not copy the -Guinness constants across; each portfolio discovers its own floor/ceiling by -running. +in S3. + +ORACLE: this module grades against the **DB PasHub-assessment SAP** — the +`epc_property_energy_performance.energy_rating_current` (`ep.source='lodged'`) +that PasHub's own assessment lodged for the dwelling — NOT the manifest +`pre_sap` (`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently +wrong; grading against it gave a misleading 79.2% / MAE 0.75 while the engine +actually matched the DB in every pre_sap≠DB case). Each fixture's oracle SAP is +resolved (by UPRN or postcode+house-number) and baked into `manifest.json` as +`oracle_sap` by `scripts/enrich_pashub_wythenshawe_oracle.py`, so CI needs no DB +access. 20 fixtures have no DB-oracle match yet (`oracle_sap` null) and are +excluded from the accuracy aggregate until PasHub sources their SAP. + +The ratchets below are calibrated to THIS cohort against the DB oracle — do not +copy the Guinness constants across; each portfolio discovers its own +floor/ceiling by running. """ from __future__ import annotations @@ -35,39 +48,35 @@ _KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel _FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy_wythenshawe" _MANIFEST_PATH = _FIXTURES_DIR / "manifest.json" -# --- Ratchets (never loosen at fixed coverage), mirroring the Guinness module. -# First full run (2026-07-16): **149 computed / 19 blocked / 0 errored** -# (within-0.5 76.5% = 114/149, MAE 0.665). The 19 blocks are a spread of -# separately-ticketed mapper gaps — floor type 'Same dwelling below' (×10), -# room-in-roof gable 'None' (×2), 'No thermostatic control...' on 'Room heaters' -# (×2), plus one-off secondary-heating / secondary-fuel / room-heater / HHR -# storage-control (2404-on-402) labels — each its own field-by-field follow-up, -# not tuning. The floor/ceiling below are the observed baseline with a hair of -# headroom; treat them as a regression tripwire, NOT a cohort accuracy figure, -# and re-baseline (coverage growth is not loosening) as further mapper fixes -# unblock fixtures. -# 2026-07-16 (#1622/#1624): room-heater control 2601 unblocked the two 'No -# thermostatic control...' fixtures → **151 computed / 17 blocked** (within-0.5 -# 76.2% = 115/151, MAE 0.716; 500767954156 lands within 0.5, 500797465814 is a -# −8.8 tail entry). MAE ceiling re-baselined 0.67 → 0.72 per the -# coverage-growth convention above. -# 2026-07-16 (#1622/#1625): floor type 'Same dwelling below' → "(another -# dwelling below)" party-floor suppression (gov-API code-8 precedent) unblocked -# all 10 fixtures → **161 computed / 7 blocked** (within-0.5 75.8% = 122/161, -# MAE 0.740). 7/10 newly-computed land within 0.5 — the treatment validates; -# −3.3/−5.5 tail entries dilute the aggregates. Floor 0.76 → 0.75, ceiling -# 0.72 → 0.75. -# 2026-07-16 (#1622/#1626-#1631): the remaining one-off labels (HHR-control -# 2404→2402 downgrade, 'Loose jacket' cylinder, secondary 601/609, secondary -# dual fuel, gas room-heater 603) → **165 computed / 3 blocked** (within-0.5 -# 75.2% = 124/165, MAE 0.844). The MAE step is one accepted pre_sap -# divergence: 500797805807 (90 Brookfield Gardens) scores 49.5 vs pre_sap 31 -# (+18.5) — its own 2025 accredited cert 6890-0597-0722-0196-3943 lodges the -# identical heating codes (603/26/2601) at band E, agreeing with US; the -# PasHub pre_sap instead matches the dwelling's superseded electric -# room-heater certs (2017/2023, band F). Ceiling 0.75 → 0.85. -_MIN_WITHIN_HALF: float = 0.75 -_MAX_SAP_MAE: float = 0.85 +# --- Ratchets (never loosen at fixed coverage), against the DB oracle. +# Re-pinned to the DB PasHub-assessment SAP (#1649): grading the earlier +# pre_sap-based history (which floated 75-76% / MAE 0.67-0.85 purely on +# surveyor data-entry noise) is retired — see the module docstring. +# 2026-07-19 (#1649 A-F): with the six from_site_notes fixes in place, the +# **148 DB-oracle-matched fixtures** (39 UPRN + 109 postcode+house-number; 20 +# unmatched, excluded) score **within-0.5 89.2% (132/148), MAE 0.297**. The +# residual tail is NOT fixable engine defects — two independent causes, both +# established, not inferred: +# 1. Integer-oracle rounding: the DB oracle is a *rounded integer* SAP graded +# against our continuous engine, so a band of spec-correct dwellings sits +# just over a rounding boundary (within-1.0 is 98%). +# 2. Input-provenance drift between the site-notes survey and the lodged cert. +# The two worst under-raters, 6 Barry Road (ours 60.12, oracle 62) and 87 +# Alderue (ours 61.80, oracle 63), were re-keyed into accredited Elmhurst +# on our exact inputs: Elmhurst returned **60 and 62** — matching our +# engine, and both *below* the lodged oracle. So our engine computes +# correct RdSAP; the lodged cert simply carried a lighter input set (it +# dropped the 17.4 m² solid-brick alternative wall the surveyor recorded +# via brick-bond evidence, +1.7 SAP). Accredited engines agree on identical +# inputs — a 2-point gap is an input difference, not a calculation one. +# So do NOT chase these as engine bugs, and do NOT tune the engine to close them +# (that would move the gov-API corpus off 78.9% and diverge from accredited +# Elmhurst). Treat floor/ceiling as a regression tripwire, not a target; +# re-baseline (coverage growth is not loosening) as PasHub sources the 20 +# unmatched SAPs. The clean accuracy proof is engine-vs-accredited on identical +# inputs (done for 6 Barry / 87 Alderue), not engine-vs-lodged-integer. +_MIN_WITHIN_HALF: float = 0.88 +_MAX_SAP_MAE: float = 0.31 _KNOWN_GAP_REASON = ( "pashub `from_site_notes` mapper does not int-code a main-heating " @@ -82,6 +91,13 @@ class PashubFixture: pre_sap_raw: str pre_sap_score: int pdf: str + # DB PasHub-assessment SAP (the grading oracle) — + # `epc_property_energy_performance.energy_rating_current`, `source='lodged'`, + # matched by UPRN or postcode+house-number and baked into the manifest by + # `scripts/enrich_pashub_wythenshawe_oracle.py` (see the module docstring). + # None for the 20 address-unmatched fixtures (excluded from the aggregate). + oracle_sap: Optional[int] = None + oracle_match: Optional[str] = None # "uprn" | "address" | None @property def path(self) -> Path: @@ -95,7 +111,7 @@ class Outcome: blocked: bool # blocked on the known main-heating-control gap (xfail territory) reason: Optional[str] sap_continuous: Optional[float] - diff: Optional[float] # abs(our SAP - pre_sap) + diff: Optional[float] # abs(our SAP - DB oracle); None when unmatched/blocked def _load_manifest() -> list[PashubFixture]: @@ -119,7 +135,14 @@ def _evaluate(deal_id: str) -> Outcome: result = Sap10Calculator().calculate(epc) except _KNOWN_GAPS as exc: return Outcome(blocked=True, reason=str(exc), sap_continuous=None, diff=None) - diff = abs(result.sap_score_continuous - fixture.pre_sap_score) + # Grade against the DB PasHub-assessment oracle, not the surveyor-entered + # `pre_sap`. A fixture with no DB-oracle match carries diff None and is + # excluded from the accuracy aggregate (it still asserts it computes). + diff = ( + abs(result.sap_score_continuous - fixture.oracle_sap) + if fixture.oracle_sap is not None + else None + ) return Outcome( blocked=False, reason=None, @@ -141,10 +164,14 @@ def test_pashub_fixture_computes(deal_id: str) -> None: @pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy_wythenshawe fixtures/manifest") def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> None: - """SAP within-0.5 of pashub's `pre_sap`, aggregated over computable fixtures.""" + """SAP within-0.5 of the DB PasHub-assessment oracle, aggregated over the + address/UPRN-matched fixtures. Fixtures with no DB-oracle match are counted + (`unmatched`) but excluded from the accuracy figures — they cannot be graded + until PasHub sources their SAP.""" diffs: list[float] = [] blocked = 0 errored = 0 + unmatched = 0 for deal_id in _IDS: try: outcome = _evaluate(deal_id) @@ -154,13 +181,15 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No if outcome.blocked: blocked += 1 continue - assert outcome.diff is not None + if outcome.diff is None: # computed, but no DB-oracle SAP to grade against + unmatched += 1 + continue diffs.append(outcome.diff) if not diffs: pytest.xfail( - f"no fixture computes a SAP score " - f"({blocked} blocked on the main-heating-control gap, {errored} errored); " + f"no fixture grades against the DB oracle " + f"({blocked} blocked, {unmatched} address-unmatched, {errored} errored); " f"{_KNOWN_GAP_REASON}" ) @@ -170,8 +199,8 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No with capsys.disabled(): print( - f"\n[pashub WYTHENSHAWE | {n} computed / {blocked} blocked / " - f"{errored} errored]" + f"\n[pashub WYTHENSHAWE | {n} graded / {unmatched} unmatched / " + f"{blocked} blocked / {errored} errored | oracle=DB energy_rating_current]" f"\n SAP within-0.5 = {within_half:.1%} MAE = {sap_mae:.3f}" ) diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 3d79ab863..8b2c4ef9a 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -6124,11 +6124,20 @@ def _pashub_room_in_roof( continue surface_kind = kind if surface_kind is None: # gable — kind from the surveyed type - if d.gable_type not in _PASHUB_GABLE_TYPE_TO_KIND: + if d.gable_type is None: + # A form variant omits the "Gable wall N type:" line, so the + # type is a genuine field-absence (not a truncation or a new + # label). Default unlodged → party `gable_wall` (U=0.25), + # mirroring the gov-API sibling `_api_type_1_gable_kind(None)` + # (SAP 10.2 Table 4 p.22 party-wall default) (#1649 B). A + # present-but-unrecognised label still strict-raises below. + surface_kind = "gable_wall" + elif d.gable_type not in _PASHUB_GABLE_TYPE_TO_KIND: raise UnmappedPasHubLabel( "room-in-roof gable type", str(d.gable_type) ) - surface_kind = _PASHUB_GABLE_TYPE_TO_KIND[d.gable_type] + else: + surface_kind = _PASHUB_GABLE_TYPE_TO_KIND[d.gable_type] thickness = insulation_thickness_mm if form_asks_insulation_known and not d.insulation_known: thickness = None @@ -6377,6 +6386,15 @@ def _map_pashub_alternative_wall(a: AlternativeWall) -> SapAlternativeWall: ), wall_thickness_measured="Y" if a.wall_thickness_mm is not None else "N", wall_thickness_mm=a.wall_thickness_mm, + # Surveyed alt-wall insulation thickness → the cascade's + # `_parse_thickness_mm`; dropped, RdSAP 10 §5.4 assumes a favourable + # 100 mm (#1649 D). Mirror the main-wall `wall_insulation_thickness` + # string wiring. + wall_insulation_thickness=( + str(a.insulation_thickness_mm) + if a.insulation_thickness_mm is not None + else None + ), ) @@ -6544,6 +6562,18 @@ def _map_sap_heating( cylinder_insulation_type=_pashub_cylinder_insulation_type_code( heating.water_heating.insulation_type, cylinder_present ), + # A surveyed cylinder thermostat ("Has thermostat? Yes") must reach the + # calculator, which reads `!= "Y"` to apply the SAP 10.2 Table 2b Note + # a) ×1.3 absent-thermostat cylinder-temperature-factor penalty (and the + # Table 4c(2) boiler interlock). Dropped (None), every PasHub cylinder + # dwelling was penalised as thermostat-less (#1649 F). Gated on a lodged + # cylinder — a combi/no-cylinder dwelling keeps None. Mirrors the + # Elmhurst path (`"Y" if cylinder_thermostat else "N"`). + cylinder_thermostat=( + ("Y" if heating.water_heating.has_thermostat else "N") + if cylinder_present + else None + ), cylinder_insulation_thickness_mm=heating.water_heating.insulation_thickness_mm, water_heating_code=water_heating_code, water_heating_fuel=water_heating_fuel, @@ -6702,6 +6732,10 @@ _PASHUB_SECONDARY_HEATING_TYPE_TO_SAP10: Dict[str, int] = { "Fire or wall heater with balanced flue": 609, # Solid fuel (eff 0.32 column B). "Open fire in grate": 631, + # Solid-fuel closed room heater — SAP 10.2 Table 4a code 633 (inside the + # Elmhurst solid-fuel secondary block {631..636}, so downstream fuel/ + # category resolution already covers it) (#1649 C). + "Closed room heater": 633, } @@ -8154,11 +8188,45 @@ def _pashub_main_heating_control_code( return code +# PasHub Manual-Entry boiler descriptor → SAP 10.2 Table 4b sub-row code +# (`table_4b.py`), keyed on the surveyed "Heating System (Boiler):" cell +# (`boiler_age_band`), which carries both the boiler type and the pre/post-1998 +# age band. A Manual-Entry boiler has no PCDB `product_id`, so its efficiency +# must come from Table 4b, not the generic 0.80 gas default (#1649 A). Only the +# cohort's observed gas/liquid descriptors are mapped; an unmapped descriptor +# (or an electric "Direct acting" boiler, which Table 4b does not cover) resolves +# to None, preserving the pre-existing behaviour (no block, no regression). +_PASHUB_MANUAL_BOILER_AGE_BAND_TO_TABLE_4B: Dict[str, int] = { + # Gas boilers 1998 or later (Table 4b 101-109). + "1998 or later - Condensing combi": 104, # 84/75 winter/summer + "1998 or later - Condensing": 102, # regular condensing, 84/74 + "1998 or later - Back boiler": 109, # back boiler to radiators, 66/56 + # Gas pre-1998 (Table 4b 110-119). + "Pre 1998 - Back boiler": 119, # back boiler to radiators, 66/56 + "Pre 1998 - Wall mounted": 115, # regular, wall mounted, 66/56 + # Liquid fuel boilers (Table 4b 124-132). + "Pre 1998 - Standard oil boiler": 125, # standard oil 1985-1997, 71/59 +} + + +def _pashub_manual_boiler_table_4b_code(main: PasHubMainHeating) -> Optional[int]: + """Resolve a Manual-Entry boiler's surveyed descriptor to its SAP 10.2 + Table 4b code (#1649 A). Returns None for a PCDF-Search boiler (product_id + lodged — efficiency is the SEDBUK lookup) and for any descriptor the Table + 4b map does not cover (e.g. an electric boiler), leaving the calculator's + existing cascade untouched for those.""" + if main.product_id > 0: + return None + return _PASHUB_MANUAL_BOILER_AGE_BAND_TO_TABLE_4B.get(main.boiler_age_band) + + def _pashub_main_heating_system_code(main: PasHubMainHeating) -> Optional[int]: """Resolve the Table 4a system code for a PasHub storage/room-heater lodgement at the mapper boundary (ADR-0015). Heat networks resolve their - code via `_pashub_community_heating`; boilers carry None (their efficiency - is a SEDBUK/product lookup, not a Table 4a rd row). + code via `_pashub_community_heating`. A PCDF-Search boiler carries None (its + efficiency is the SEDBUK/product lookup); a Manual-Entry boiler resolves its + surveyed descriptor to a SAP 10.2 Table 4b code (#1649 A) so the calculator + bills the Table 4b seasonal efficiency instead of the generic 0.80 default. The surveyed heater *type* is lodged in the "Heating System (Other)" field (`community_heat_source`) for Manual Entry; a PCDF-Search storage lodgement @@ -8185,6 +8253,11 @@ def _pashub_main_heating_system_code(main: PasHubMainHeating) -> Optional[int]: f"{main.community_heat_source!r} (Heating System (Other))", ) return code + if system in _PASHUB_BOILER_SYSTEM_TYPES: + # PCDF-Search boiler → None (SEDBUK product efficiency); Manual-Entry + # boiler → its Table 4b code (or None for an uncovered/non-Table-4b + # descriptor, keeping the pre-existing cascade) (#1649 A). + return _pashub_manual_boiler_table_4b_code(main) return None diff --git a/datatypes/epc/domain/tests/test_from_site_notes.py b/datatypes/epc/domain/tests/test_from_site_notes.py index 7d8dedfc6..07b93ff5f 100644 --- a/datatypes/epc/domain/tests/test_from_site_notes.py +++ b/datatypes/epc/domain/tests/test_from_site_notes.py @@ -1957,6 +1957,7 @@ class TestFromSiteNotesExample1: has_fixed_air_conditioning=False, cylinder_size=2, # "Normal (90-130 litres)" → Table 28 code 2 cylinder_insulation_type=1, # "Factory fitted" → foam, SAP10 code 1 + cylinder_thermostat="Y", # "Has thermostat? Yes" → interlock (#1649 F) cylinder_insulation_thickness_mm=12, water_heating_code=901, # cylinder + "From main heating 1" → WHC 901 shower_outlets=ShowerOutlets( @@ -3000,3 +3001,250 @@ class TestPasHubUnmappedHeatEmitter: # Act / Assert with pytest.raises(UnmappedPasHubLabel, match="Skirting heaters"): EpcPropertyDataMapper.from_site_notes(survey) + + +class TestPasHubCylinderThermostat: + """The surveyed hot-water cylinder thermostat ("Has thermostat? Yes") must + reach `sap_heating.cylinder_thermostat` — dropped (left None), the + calculator reads `!= "Y"` and applies the SAP 10.2 Table 2b Note a) ×1.3 + absent-thermostat cylinder-temperature-factor penalty (equivalently the + Table 4c(2) −5pp boiler-interlock penalty), inflating stored-HW loss and + under-rating every PasHub cylinder dwelling. The Elmhurst sibling maps it + (`"Y" if cylinder_thermostat else "N"`); `from_site_notes` did not. + Evidence: 52 Bucklow Drive (DB 62, ours 59.44 → 61.79) and 13 Kerne Grove + (DB 67, ours 64.95 → 67.11) — the two worst DB-oracle under-raters (#1649 F). + """ + + def test_surveyed_cylinder_thermostat_yes_maps_to_Y(self) -> None: + # Arrange — example1 lodges a cylinder ("Normal (90-130 litres)") with + # "Has thermostat? Yes". + data = load("pashub_rdsap_site_notes_example1.json") + assert data["heating_and_hot_water"]["water_heating"]["has_thermostat"] is True + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act + result = EpcPropertyDataMapper.from_site_notes(survey) + + # Assert + assert result.sap_heating.cylinder_thermostat == "Y" + + def test_surveyed_cylinder_thermostat_no_maps_to_N(self) -> None: + # Arrange + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["water_heating"]["has_thermostat"] = False + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act + result = EpcPropertyDataMapper.from_site_notes(survey) + + # Assert + assert result.sap_heating.cylinder_thermostat == "N" + + def test_no_cylinder_leaves_thermostat_none(self) -> None: + # Arrange — a combi/no-cylinder dwelling has no cylinder thermostat to + # lodge; the field must stay None (not "N"), so the calculator's + # cylinder-loss path is not spuriously engaged. + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["water_heating"]["cylinder_size"] = "No Cylinder" + data["heating_and_hot_water"]["water_heating"]["has_thermostat"] = None + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act + result = EpcPropertyDataMapper.from_site_notes(survey) + + # Assert + assert result.sap_heating.cylinder_thermostat is None + + +class TestPasHubRoomInRoofGableNoneDefaultsParty: + """A survey-form variant that omits the room-in-roof gable-type field lodges + a genuine `gable_type=None` (field-absence, not a truncation or a new + label). The mapper must default it to the party `gable_wall` (U=0.25), + mirroring the gov-API sibling `_api_type_1_gable_kind(None)` — a + present-but-unrecognised label still strict-raises. Left raising, the + fixture is uncomputable (500771973367 8 Bucklow Drive, 500759637226 34 + Bucklow Drive both BLOCKED; #1649 B). Spec: SAP 10.2 Table 4 p.22 (party + wall U=0.25). + """ + + _RIR_GABLE_NONE = { + "age_range": "C: 1930 - 1949", + "floor_area_m2": 21.0, + # The form variant lodges length/height but no "Gable wall N type:" line + # → gable_type is a genuine None. + "gables": [ + {"length_m": 4.76, "height_m": 2.32}, + {"length_m": 3.16, "height_m": 2.32}, + ], + "slopes": [{"length_m": 4.76, "height_m": 2.5}], + } + + def test_gable_type_none_defaults_to_party_gable_wall(self) -> None: + # Arrange + data = load("pashub_rdsap_site_notes_example1.json") + data["roof_space"]["main_building"]["rooms_in_roof"] = True + data["roof_space"]["main_building"]["room_in_roof"] = self._RIR_GABLE_NONE + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act — must not raise + rir = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[ + 0 + ].sap_room_in_roof + + # Assert — both unlodged-type gables default to the party gable_wall. + assert rir is not None + assert rir.detailed_surfaces is not None + gable_kinds = [ + s.kind for s in rir.detailed_surfaces if s.kind.startswith("gable_wall") + ] + assert gable_kinds == ["gable_wall", "gable_wall"] + + def test_present_but_unknown_gable_label_still_raises(self) -> None: + # Arrange — a lodged-but-unrecognised label is a real coverage gap and + # must still fail loud (the None default must not swallow it). + data = load("pashub_rdsap_site_notes_example1.json") + rir = { + **self._RIR_GABLE_NONE, + "gables": [{"length_m": 4.76, "height_m": 2.32, "gable_type": "Mystery"}], + } + data["roof_space"]["main_building"]["rooms_in_roof"] = True + data["roof_space"]["main_building"]["room_in_roof"] = rir + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act / Assert + with pytest.raises(UnmappedPasHubLabel, match="Mystery"): + EpcPropertyDataMapper.from_site_notes(survey) + + +class TestPasHubSecondaryClosedRoomHeater: + """The secondary-heating label "Closed room heater" must map to SAP 10.2 + Table 4a code 633 (solid-fuel closed room heater). An uncovered label + strict-raises and blocks the fixture (500796700868 8 Brinkshaw Avenue; + #1649 C). 633 is already in `_ELMHURST_SECONDARY_SOLID_CODES {631..636}`, + so downstream fuel/category resolution already covers it. + """ + + def test_closed_room_heater_maps_to_633(self) -> None: + # Arrange + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["secondary_heating"][ + "secondary_system" + ] = "Closed room heater" + data["heating_and_hot_water"]["secondary_heating"]["secondary_fuel"] = ( + "Dual Fuel Appliance (Mineral and Wood)" + ) + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act — must not raise + result = EpcPropertyDataMapper.from_site_notes(survey) + + # Assert + assert result.sap_heating.secondary_heating_type == 633 + + +class TestPasHubManualBoilerEfficiency: + """A Manual-Entry boiler (no PCDB `product_id`) must resolve its surveyed + descriptor to a SAP 10.2 Table 4b code so the calculator bills the Table 4b + seasonal efficiency, not the generic 0.80 "no data" gas default. Left None, + a back boiler (Table 4b 66%) is credited 80% (500796377318 4a Hollyhedge: + DB 53, ours 63.6 → 53.4) and a condensing combi's water-heating summer + efficiency (75%) is over-credited (the 84 Button / 91 Alderue / 17 Kennett + / 43 Kenninghall / 156 Gladeside over-raters). #1649 A. + + A PCDF-Search boiler keeps `None` (its efficiency is the SEDBUK product + lookup); a non-Table-4b descriptor (e.g. electric) also keeps `None` + (unchanged behaviour — no regression). + """ + + @staticmethod + def _manual_boiler(data: Dict[str, Any], *, boiler_type: str, age_band: str) -> Any: + mh = data["heating_and_hot_water"]["main_heating"] + mh["selection_method"] = "Manual Entry" + mh["system_type"] = "Boiler with radiators or underfloor heating" + mh["product_id"] = 0 + mh["fuel"] = "Mains Gas" + mh["boiler_type"] = boiler_type + mh["boiler_age_band"] = age_band + return from_dict(PasHubRdSapSiteNotes, data) + + def test_manual_back_boiler_post_1998_maps_to_109(self) -> None: + survey = self._manual_boiler( + load("pashub_rdsap_site_notes_example1.json"), + boiler_type="Back boiler", + age_band="1998 or later - Back boiler", + ) + main = EpcPropertyDataMapper.from_site_notes( + survey + ).sap_heating.main_heating_details[0] + assert main.sap_main_heating_code == 109 + + def test_manual_condensing_combi_maps_to_104(self) -> None: + survey = self._manual_boiler( + load("pashub_rdsap_site_notes_example1.json"), + boiler_type="Condensing combi", + age_band="1998 or later - Condensing combi", + ) + main = EpcPropertyDataMapper.from_site_notes( + survey + ).sap_heating.main_heating_details[0] + assert main.sap_main_heating_code == 104 + + def test_pcdf_boiler_keeps_none(self) -> None: + # example1 as lodged is a PCDF-Search boiler (product_id 18400): its + # efficiency is the SEDBUK lookup, so the Table 4a code stays None. + survey = from_dict( + PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json") + ) + main = EpcPropertyDataMapper.from_site_notes( + survey + ).sap_heating.main_heating_details[0] + assert main.sap_main_heating_code is None + + def test_manual_boiler_unmapped_descriptor_keeps_none(self) -> None: + # An electric "Direct acting" descriptor is not a Table 4b gas/liquid + # boiler — keep None (unchanged 0.80 behaviour, no regression / no block). + survey = self._manual_boiler( + load("pashub_rdsap_site_notes_example1.json"), + boiler_type="Standard", + age_band="Direct acting", + ) + main = EpcPropertyDataMapper.from_site_notes( + survey + ).sap_heating.main_heating_details[0] + assert main.sap_main_heating_code is None + + +class TestPasHubAlternativeWallInsulationThickness: + """A surveyed alternative-wall "Wall insulation thickness:" must reach + `SapAlternativeWall.wall_insulation_thickness`; dropped, the cascade takes + the RdSAP 10 §5.4 "assume 100 mm if unknown" favourable default instead of + the surveyed value (#1649 D). Mirrors the gov-API / Elmhurst paths. + """ + + def test_alt_wall_insulation_thickness_reaches_sap_alt_wall(self) -> None: + # Arrange — a main-building alt wall lodging a 50 mm insulation thickness. + data = load("pashub_rdsap_site_notes_example1.json") + data["alternative_walls"] = [ + { + "location": "Main Building", + "id": 1, + "construction_type": "Solid brick", + "insulation_type": "External", + "area_m2": 13.71, + "wall_u_value_known": False, + "thermal_conductivity_of_wall_insulation": "Unknown", + "wall_thickness_mm": 250, + "dry_lined": False, + "insulation_thickness_mm": 50, + } + ] + survey = from_dict(PasHubRdSapSiteNotes, data) + + # Act + result = EpcPropertyDataMapper.from_site_notes(survey) + + # Assert — the surveyed 50 mm reaches the SAP alt wall (string wiring, + # like the main wall) instead of the §5.4 100 mm favourable default. + alt = result.sap_building_parts[0].sap_alternative_wall_1 + assert alt is not None + assert alt.wall_insulation_thickness == "50" diff --git a/datatypes/epc/surveys/pashub_rdsap_site_notes.py b/datatypes/epc/surveys/pashub_rdsap_site_notes.py index 121b7ca17..988ef01ad 100644 --- a/datatypes/epc/surveys/pashub_rdsap_site_notes.py +++ b/datatypes/epc/surveys/pashub_rdsap_site_notes.py @@ -232,6 +232,16 @@ class MainHeating: # "Water or oil-filled radiators"); blank when the type is fixed elsewhere # (a PCDF-Search product, or a boiler). See #1619. community_heat_source: str = "" + # Manual-Entry (non-PCDF) boiler descriptor. A boiler selected by "Manual + # Entry" (product_id 0) carries no SEDBUK record, so its efficiency must + # come from SAP 10.2 Table 4b keyed on these two surveyed labels rather than + # the generic 0.80 gas default (#1649 A). `boiler_type` is the §Heating + # "Type of boiler:" cell (e.g. "Condensing combi" / "Back boiler"), + # `boiler_age_band` the "Heating System (Boiler):" cell, which carries the + # pre/post-1998 age band (e.g. "1998 or later - Condensing combi"). Blank on + # a PCDF-Search boiler (efficiency is the product lookup). + boiler_type: str = "" + boiler_age_band: str = "" @dataclass @@ -369,6 +379,10 @@ class AlternativeWall: thermal_conductivity_of_wall_insulation: str wall_thickness_mm: Optional[int] = None dry_lined: Optional[bool] = None + # Surveyed "Wall insulation thickness:" (mm) for the alternative wall. + # Dropped, the cascade takes the RdSAP 10 §5.4 "assume 100 mm if unknown" + # favourable default instead of the lodged value (#1649 D). + insulation_thickness_mm: Optional[int] = None def _no_alternative_walls() -> List[AlternativeWall]: diff --git a/scripts/enrich_pashub_wythenshawe_oracle.py b/scripts/enrich_pashub_wythenshawe_oracle.py new file mode 100644 index 000000000..45701c055 --- /dev/null +++ b/scripts/enrich_pashub_wythenshawe_oracle.py @@ -0,0 +1,134 @@ +"""Enrich the PasHub Wythenshawe fixture manifest with the DB-oracle SAP. + +The accuracy harness must grade against the reliable oracle — the PasHub +company-assessment result stored in the DB (`epc_property_energy_performance. +energy_rating_current`, `ep.source='lodged'`) — NOT the manifest `pre_sap` +(`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently wrong). This +one-time / rebuild-able provenance script resolves each fixture to that DB SAP +and writes `oracle_sap` (int or null) + `oracle_match` ("uprn" / "address" / +null) back into `manifest.json`, so the committed manifest is what the test +consumes and CI needs no DB access. Mirrors `build_pashub_accuracy_fixtures.py`. + +Match order: manifest `uprn` (bigint, exact) → else postcode + house-number +parsed from the PDF "Property Address:" block. Latest cert per key by `ep.id`. + +Requires DB credentials (reads `backend/.env`); NOT run in CI. + + python scripts/enrich_pashub_wythenshawe_oracle.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + +from scripts.e2e_common import build_engine, load_env # noqa: E402 +from sqlalchemy import text # noqa: E402 +from sqlalchemy.engine import Connection # noqa: E402 + +from backend.documents_parser.parser import pdf_to_text_list # noqa: E402 + +_FIXTURES = ( + REPO_ROOT + / "backend" + / "documents_parser" + / "tests" + / "fixtures" + / "pashub_accuracy_wythenshawe" +) +_MANIFEST = _FIXTURES / "manifest.json" +_POSTCODE_RE = re.compile(r"^[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}$") +_HOUSENUM_RE = re.compile(r"^(\d+[A-Za-z]?)\b") + + +def _address_from_pdf(deal_id: str) -> tuple[Optional[str], Optional[str]]: + """(house_number, postcode) from the PDF "Property Address:" block.""" + tl = pdf_to_text_list((_FIXTURES / f"{deal_id}.pdf").read_bytes()) + try: + i = next(k for k, l in enumerate(tl) if l.strip() == "Property Address:") + except StopIteration: + return None, None + block = [l.strip().rstrip(",") for l in tl[i + 1 : i + 6] if l.strip()] + postcode = next((l for l in block if _POSTCODE_RE.match(l.upper())), None) + house = None + if block: + m = _HOUSENUM_RE.match(block[0]) + if m: + house = m.group(1) + return house, postcode + + +def _resolve( + conn: Connection, uprn: Optional[int], deal_id: str +) -> tuple[Optional[int], str]: + if uprn: + row = conn.execute( + text( + "select eppe.energy_rating_current from epc_property ep " + "join epc_property_energy_performance eppe " + "on eppe.epc_property_id = ep.id " + "where ep.source='lodged' and ep.uprn=:u " + "and eppe.energy_rating_current is not null " + "order by ep.id desc limit 1" + ), + {"u": int(uprn)}, + ).scalar() + if row is not None: + return int(row), "uprn" + house, postcode = _address_from_pdf(deal_id) + if house and postcode: + rows = conn.execute( + text( + "select ep.address_line_1, eppe.energy_rating_current " + "from epc_property ep join epc_property_energy_performance eppe " + "on eppe.epc_property_id = ep.id " + "where ep.source='lodged' " + "and replace(upper(ep.postcode),' ','')=:pc " + "and eppe.energy_rating_current is not null order by ep.id desc" + ), + {"pc": postcode.upper().replace(" ", "")}, + ).fetchall() + for row_pair in rows: + a1 = str(row_pair[0] or "").strip() + m = _HOUSENUM_RE.match(a1) + if m and m.group(1).lower() == house.lower(): + return int(row_pair[1]), "address" + return None, "unmatched" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + load_env(REPO_ROOT / "backend" / ".env") + engine = build_engine() + manifest = json.loads(_MANIFEST.read_text()) + + matched = 0 + with engine.connect() as conn: + for fx in manifest["fixtures"]: + oracle_sap, kind = _resolve(conn, fx.get("uprn"), fx["deal_id"]) + fx["oracle_sap"] = oracle_sap + fx["oracle_match"] = None if kind == "unmatched" else kind + if oracle_sap is not None: + matched += 1 + + n = len(manifest["fixtures"]) + print(f"resolved {matched}/{n} fixtures to a DB oracle SAP") + if args.dry_run: + print("--dry-run: manifest NOT written") + return + _MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n") + print(f"wrote {_MANIFEST}") + + +if __name__ == "__main__": + main()