landlord override data added

domain/epc/property_overlays/roof_type_overlay.py

@@ -1,12 +1,21 @@
-"""Map a Landlord-Override `RoofType` value to a roof Simulation Overlay (ADR-0032).
+"""Map a Landlord-Override `RoofType` value to a roof Simulation Overlay (ADR-0032/0033).
 
-The calculator derives the roof U-value from the building part's loft-insulation
-depth, so a `roof_type` override moves the score only via
-`BuildingPartOverlay.roof_insulation_thickness` (mm). The resolvable family is
-the explicit `"Pitched, N mm loft insulation"` values — N is parsed out.
-Everything else (flat roofs, room-in-roof, "Unknown loft insulation",
-"Another Premises Above" — a flat with a dwelling above, no roof to insulate) has
-no clean loft depth, so it produces no overlay.
+Two resolvable families:
+
+* `"Pitched, N mm loft insulation"` — the loft depth N maps to
+  `BuildingPartOverlay.roof_insulation_thickness` (mm); the calculator scores the
+  roof from that depth.
+* `"Flat, …"` — a flat roof carries no loft depth, so its U-value comes from the
+  age-band default (`_FLAT_ROOF_BY_AGE`, ADR-0033). The calculator's flat path
+  keys on the `roof_construction_type` *string* (`"flat" in …`), so the overlay
+  sets that to `"Flat"` and leaves thickness `None` for the (separately overlaid)
+  construction age band to drive the U-value. No flat `RoofType` value carries an
+  explicit mm depth, confirmed by the Elmhurst sweep (As Built / Unknown ≡
+  age-band default).
+
+Everything else (room-in-roof, "Unknown loft insulation", party-ceiling adjacency
+markers like "Another Premises Above") has no clean depth or shape correction, so
+it produces no overlay.
 """
 
 from __future__ import annotations
@@ -23,8 +32,8 @@ _LOFT_MM = re.compile(r"(\d+)\+?\s*mm loft insulation")
 def roof_overlay_for(
     roof_type_value: str, building_part: int
 ) -> Optional[EpcSimulation]:
-    match = _LOFT_MM.search(roof_type_value)
-    if match is None:
+    overlay = _overlay_for(roof_type_value)
+    if overlay is None:
         return None
 
     identifier = (
@@ -32,10 +41,28 @@ def roof_overlay_for(
         if building_part == 0
         else BuildingPartIdentifier.extension(building_part)
     )
-    return EpcSimulation(
-        building_parts={
-            identifier: BuildingPartOverlay(
-                roof_insulation_thickness=int(match.group(1))
-            )
-        }
-    )
+    return EpcSimulation(building_parts={identifier: overlay})
+
+
+def _overlay_for(roof_type_value: str) -> Optional[BuildingPartOverlay]:
+    match = _LOFT_MM.search(roof_type_value)
+    if match is not None:
+        return BuildingPartOverlay(roof_insulation_thickness=int(match.group(1)))
+    if roof_type_value.startswith("Flat,"):
+        # Flat roof: U-value is the age-band default; flag the shape and let the
+        # age-band overlay drive `_FLAT_ROOF_BY_AGE` (ADR-0033).
+        return BuildingPartOverlay(roof_construction_type="Flat")
+    if roof_type_value == "Pitched, Unknown loft insulation":
+        # Unknown loft depth: U-value is the pitched age-band default. Assert the
+        # pitched shape (keeps `is_flat_roof`/`is_sloping_ceiling` False) and leave
+        # thickness None so the age-band overlay drives `_ROOF_BY_AGE` (ADR-0033).
+        return BuildingPartOverlay(roof_construction_type="Pitched")
+    if roof_type_value.startswith("Pitched, no insulation"):
+        # Genuinely uninsulated pitched roof → 2.30 (Table 16 row 0). Thickness 0
+        # drives the calculator's thickness path and, unlike "Unknown", overrides
+        # any lodged numeric thickness (ADR-0033). Pitched text is load-bearing
+        # here — it does NOT take the age-band default (rdsap_uvalues §5.11 note).
+        return BuildingPartOverlay(
+            roof_construction_type="Pitched", roof_insulation_thickness=0
+        )
+    return None

domain/epc/property_overlays/wall_type_overlay.py

@@ -34,8 +34,16 @@ _MATERIAL_CONSTRUCTION: dict[str, int] = {
 # RdSAP `wall_insulation_type` codes by insulation-state suffix
 # (domain/sap10_ml/rdsap_uvalues.py): external 1, filled-cavity 2, internal 3,
 # as-built/uninsulated 4, cavity+external 6, cavity+internal 7.
+# All three "as built (assumed)" variants resolve to the same as-built code (4);
+# the construction-age-band overlay supplies the U-value, so "partial" / "insulated"
+# need no distinct code (ADR-0033, confirmed by the full A–M Elmhurst sweep in
+# scripts/hyde/uvalue_probe_walls.csv — cavity As Built ≡ (CAVITY,0) by age band,
+# e.g. band F → 1.0 = "partial"). The bare "as built" covers Cob / Park home.
 _STATE_INSULATION: dict[str, int] = {
+    "as built": 4,
     "as built, no insulation (assumed)": 4,
+    "as built, partial insulation (assumed)": 4,
+    "as built, insulated (assumed)": 4,
     "with internal insulation": 3,
     "with external insulation": 1,
     "filled cavity": 2,

domain/modelling/simulation.py

@@ -41,6 +41,12 @@ class BuildingPartOverlay:
     # IWI (`wall_insulation_type=3`); λ defaults to 0.04 W/m·K in the calculator.
     wall_insulation_thickness: Optional[int] = None
     roof_insulation_thickness: Optional[int] = None
+    # The roof shape string the calculator's flat-roof path keys on
+    # (`heat_transmission.py`: `"flat" in roof_construction_type`). Left `None`
+    # by Measures (insulating a roof doesn't change its shape); set by a Landlord
+    # Override that corrects a roof to flat so the `_FLAT_ROOF_BY_AGE` age-band
+    # default applies (ADR-0033). Folds onto the part via the generic field loop.
+    roof_construction_type: Optional[str] = None
     floor_insulation_thickness: Optional[int] = None
     floor_insulation_type_str: Optional[str] = None
 

repositories/property/landlord_override_overlays.py

@@ -7,18 +7,23 @@ type — `domain/` never imports `repositories/`.
 
 Per-component and partial — an override produces an overlay only where a
 component mapping exists and the value resolves; anything else is left to the
-lodged EPC. All four `override_component`s are mapped:
+lodged EPC. Every `override_component` is mapped:
 
 * `wall_type`  → fabric overlay (`wall_construction` + `wall_insulation_type`)
-* `roof_type`  → fabric overlay (`roof_insulation_thickness`, loft-depth family)
+* `roof_type`  → fabric overlay (`roof_insulation_thickness` / `roof_construction_type`)
+* `construction_age_band` → per-part `construction_age_band` — the U-value lever
+  for the "as built / assumed / Unknown" fabric states (ADR-0033)
 * `property_type` / `built_form_type` → whole-dwelling categorical correction
+* `main_fuel` / `glazing` / `water_heating` / `main_heating_system` → whole-dwelling
 
-Two value families deliberately resolve to *no* overlay rather than a guess: the
-`"(assumed) insulated"` / `"partial insulation (assumed)"` wall states (RdSAP
-infers their U-value from the build-era age band, so there is no single
-`wall_insulation_type` code for them — they need Elmhurst validation, ADR-0032),
-and `"Unknown"` categorical values. Roofs with no clean loft depth (flat,
-room-in-roof, "another premises above") likewise produce no overlay.
+Since ADR-0033 the "as built (assumed)" wall states resolve to the as-built code
+(4) and flat / pitched-Unknown / pitched-no-insulation roofs produce an overlay —
+their U-value comes from the per-part `construction_age_band`, not a per-state
+code. Values that still resolve to *no* overlay (left to the lodged EPC): bare
+`"Unknown"` categoricals, party-ceiling roofs ("another/same dwelling above" — a
+party ceiling has ≈0 heat loss), room-in-roof, and the residue logged in ADR-0033.
+The `audit_override_coverage.py` / `audit_hyde_rows.py` pair reports exactly which
+values are live vs no-op before any write.
 """
 
 from __future__ import annotations

scripts/hyde/build_property_overrides.py

@@ -111,6 +111,21 @@ logger = logging.getLogger("build_property_overrides")
 ORG_REF_COLUMN = "Organisation Reference"
 UNKNOWNS_PATH = "overrides_unknowns.csv"
 
+# A "party ceiling" roof (another/same dwelling or premises above) means the
+# dwelling has no exposed roof, which is only physically valid for a Flat or
+# Maisonette. When the property type says House/Bungalow the two landlord fields
+# contradict (a house has nothing above it). Per Khalim these may be houses split
+# into flats — leave them entirely as-is for joint review, so we SKIP the whole
+# property (write NO overrides for it) and record the org_refs for that review
+# (ADR-0033; surfaced by scripts/hyde/audit_hyde_rows.py).
+_PARTY_CEILING_ROOF_VALUES: frozenset[str] = frozenset({
+    "(another dwelling above)", "(same dwelling above)",
+    "(other premises above)", "(another premises above)",
+    "Another Premises Above",
+})
+_FLAT_PTYPE_PREFIXES: tuple[str, ...] = ("flat", "maisonette")
+SKIPPED_PATH = "skipped_contradictory_properties.csv"
+
 
 @dataclass(frozen=True)
 class ComponentSpec:
@@ -330,15 +345,32 @@ def write(args: argparse.Namespace) -> None:
         org_ref_map = _org_ref_to_property_id(session, args.portfolio_id)
         logger.info("Portfolio %d: %d properties with org_ref.", args.portfolio_id, len(org_ref_map))
 
+        ptype_header = _specs_by_component()["property_type"].excel_header
+        roof_header = _specs_by_component()["roof_type"].excel_header
+        roof_vocab = vocab.get("roof_type", {})
         inserts: list[PropertyOverrideInsert] = []
         unmatched: Counter[str] = Counter()
         unresolved: Counter[str] = Counter()
+        skipped: list[tuple[str, str, str]] = []  # (org_ref, property_type, roof)
         for row in rows:
             org_ref = str(row.get(ORG_REF_COLUMN, "")).strip()
             property_id = org_ref_map.get(org_ref)
             if property_id is None:
                 unmatched[org_ref] += 1
                 continue
+            ptype_raw = str(row.get(ptype_header) or "").strip()
+            roof_raw = str(row.get(roof_header) or "").strip()
+            row_is_flat = ptype_raw.lower().startswith(_FLAT_PTYPE_PREFIXES)
+            roof_resolved = {
+                roof_vocab.get(_norm(e))
+                for e in _split_entries(row.get(roof_header), True)
+            }
+            if not row_is_flat and roof_resolved & _PARTY_CEILING_ROOF_VALUES:
+                # Party-ceiling roof on a non-flat dwelling — contradictory source
+                # data (maybe a house split into flats). Leave the property fully
+                # as-is for joint review: write NO overrides, record it (see above).
+                skipped.append((org_ref, ptype_raw, roof_raw))
+                continue
             for spec in _component_specs():
                 comp_vocab = vocab.get(spec.component, {})
                 for building_part, entry in enumerate(
@@ -352,8 +384,18 @@ def write(args: argparse.Namespace) -> None:
                         building_part=building_part, override_component=spec.component,
                         override_value=value, original_spreadsheet_description=entry))
 
-        logger.info("Built %d rows | %d unmatched org_refs | %d unresolved",
-                    len(inserts), sum(unmatched.values()), sum(unresolved.values()))
+        logger.info("Built %d rows | %d unmatched org_refs | %d unresolved | "
+                    "%d contradictory properties skipped (left as-is)",
+                    len(inserts), sum(unmatched.values()), sum(unresolved.values()),
+                    len(skipped))
+        if skipped:
+            out = os.path.join(os.path.dirname(args.excel) or ".", SKIPPED_PATH)
+            with open(out, "w", newline="") as f:
+                w = csv.writer(f)
+                w.writerow(["org_ref", "property_type", "roof"])
+                w.writerows(sorted(skipped))
+            logger.info("Recorded %d skipped properties -> %s (for joint review)",
+                        len(skipped), out)
         if unresolved:
             logger.info("Top unresolved (need apply-edits): %s", unresolved.most_common(10))
         if not args.apply:

scripts/hyde/elmhurst_lib.py

@@ -49,6 +49,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Callable, Generator, Optional
@@ -162,6 +163,30 @@ def set_select(page: Page, suffix: str, value: str, autopostback: bool = True) -
         loc.select_option(value)
 
 
+def select_options(page: Page, suffix: str) -> list[tuple[str, str]]:
+    """Return [(value, visible_text), …] for the <select> at FP+suffix — lets a
+    sweep enumerate the *real* Elmhurst options instead of hard-coding label
+    strings that drift between RdSAP releases. Skips the blank placeholder."""
+    return page.evaluate(
+        """(id)=>{const s=document.getElementById(id); if(!s)return [];
+        return [...s.options].map(o=>[o.value,(o.text||'').trim()])
+          .filter(([v,t])=>v!=='' && t!=='');}""",
+        f"{FP}{suffix}",
+    )
+
+
+def select_by_contains(page: Page, suffix: str, needle: str, autopostback: bool = True) -> Optional[str]:
+    """Select the option whose visible text contains `needle` (case-insensitive),
+    by its value. Returns the chosen text, or None if no option matched — robust
+    to label drift where the exact string ("CA Cavity" vs "Cavity (CA)") is
+    uncertain until seen live."""
+    for value, text in select_options(page, suffix):
+        if needle.lower() in text.lower():
+            set_select(page, suffix, value, autopostback)
+            return text
+    return None
+
+
 def save_close(page: Page) -> None:
     """Save & Close — commits the whole server session to the DB (so the data
     survives the next fresh-login run). The body button id-suffix avoids the
@@ -319,6 +344,66 @@ def select_boiler(page: Page, query: str, ref: str) -> str:
     return "ok"
 
 
+def read_default_uvalue(
+    page: Page, suffix: Optional[str] = None, label_regex: str = r"Default U-value"
+) -> Optional[float]:
+    """Read the read-only "Default U-value [W/m²K]" the tool computes for the
+    active fabric element (wall/roof/floor), after the type/insulation/thickness
+    postbacks have settled.
+
+    The override checkbox ("U-value Known") is documented as unreliable to toggle,
+    but the *default* field is a plain display (a SPAN whose text IS the value,
+    e.g. `…WebUserControlWallMain_LabelUValueDefault`) that RdSAP recomputes on
+    every postback — that recomputed number is Elmhurst's
+    age-band×insulation×thickness answer, which is exactly what the probe wants.
+
+    Preferred: pass the exact `suffix` (after FP) of that label span. Without it,
+    falls back to LABEL PROXIMITY (match `label_regex`, read the nearest numeric),
+    which is brittle near the thickness box. Returns None if not found (caller
+    should fall back to the screenshot)."""
+    if suffix is not None:
+        raw_text = page.evaluate(
+            "(id)=>{const e=document.getElementById(id); return e?(e.textContent||'').trim():null;}",
+            f"{FP}{suffix}",
+        )
+        if raw_text is not None:
+            m = re.search(r"-?\d+(?:\.\d+)?", str(raw_text))
+            if m:
+                return float(m.group(0))
+        return None
+    raw = page.evaluate(
+        """(rx)=>{
+        const re=new RegExp(rx,'i');
+        // Candidate label nodes: any visible element whose own text matches.
+        const labels=[...document.querySelectorAll('label,span,td,div,th')]
+          .filter(e=>e.offsetParent && re.test((e.textContent||'').trim()));
+        const numFrom=(s)=>{const m=(s||'').match(/-?\\d+(?:\\.\\d+)?/); return m?m[0]:null;};
+        for(const lab of labels){
+          // 1) a sibling/descendant input carrying the value
+          let scope=lab.closest('tr,div,td')||lab.parentElement||lab;
+          const inp=scope.querySelector("input[type='text'],input:not([type])");
+          if(inp && numFrom(inp.value)!==null) return inp.value;
+          // 2) the next cell's text (label | value table layout)
+          let sib=lab.nextElementSibling;
+          for(let i=0;i<3 && sib;i++,sib=sib.nextElementSibling){
+            const n=numFrom(sib.textContent);
+            if(n!==null && !re.test(sib.textContent)) return n;
+          }
+          // 3) trailing number in the label's own container, after the label text
+          const tail=numFrom(scope.textContent.replace(lab.textContent,''));
+          if(tail!==null) return tail;
+        }
+        return null;}""",
+        label_regex,
+    )
+    if raw is None:
+        return None
+    try:
+        return float(str(raw).strip())
+    except ValueError:
+        return None
+
+
 def clear_hot_water_cylinder(page: Page, panel: str = "TabContainer_TabPanelWaterHeating_") -> None:
     """Uncheck the HW cylinder (combi). The checkbox won't uncheck via
     .uncheck()/commit (a regular-boiler default re-asserts it); JS-set

tests/domain/epc/test_roof_type_overlay.py

@@ -45,9 +45,74 @@ def test_each_loft_depth_is_parsed(roof_type_value: str, expected_mm: int) -> No
 @pytest.mark.parametrize(
     "roof_type_value",
     [
-        "Another Premises Above",
-        "Pitched, Unknown loft insulation",
         "Flat, insulated",
+        "Flat, insulated (assumed)",
+        "Flat, no insulation",
+        "Flat, no insulation (assumed)",
+        "Flat, limited insulation",
+        "Flat, limited insulation (assumed)",
+    ],
+)
+def test_flat_roof_sets_construction_type_for_age_band_default(
+    roof_type_value: str,
+) -> None:
+    # ADR-0033: a flat roof carries no loft depth, so the U-value comes from
+    # the age-band default (`_FLAT_ROOF_BY_AGE`). The calculator's flat path
+    # keys on the `roof_construction_type` string ("flat" in it), so the overlay
+    # sets that and leaves thickness None for the age band to drive.
+
+    # Act
+    simulation = roof_overlay_for(roof_type_value, 0)
+
+    # Assert
+    assert simulation is not None
+    overlay = simulation.building_parts[BuildingPartIdentifier.MAIN]
+    assert overlay.roof_construction_type == "Flat"
+    assert overlay.roof_insulation_thickness is None
+
+
+@pytest.mark.parametrize(
+    "roof_type_value",
+    ["Pitched, no insulation", "Pitched, no insulation (assumed)"],
+)
+def test_pitched_no_insulation_forces_the_uninsulated_thickness(
+    roof_type_value: str,
+) -> None:
+    # ADR-0033: a pitched "no insulation" roof is genuinely uninsulated (2.30 =
+    # Table 16 row 0). Set thickness 0 — the calculator's thickness path returns
+    # 2.30, and unlike "Unknown" a 0 *can* override a lodged numeric thickness.
+
+    # Act
+    simulation = roof_overlay_for(roof_type_value, 0)
+
+    # Assert
+    assert simulation is not None
+    overlay = simulation.building_parts[BuildingPartIdentifier.MAIN]
+    assert overlay.roof_insulation_thickness == 0
+
+
+def test_pitched_unknown_loft_drives_the_pitched_age_band_default() -> None:
+    # ADR-0033: "Pitched, Unknown loft insulation" carries no depth, so its
+    # U-value is the pitched age-band default (`_ROOF_BY_AGE`). Mirror the flat
+    # case — assert the pitched shape (so `is_flat_roof` stays False) and leave
+    # thickness None for the construction age band to drive. ~3,613 Hyde rows
+    # previously silently produced no overlay.
+
+    # Act
+    simulation = roof_overlay_for("Pitched, Unknown loft insulation", 0)
+
+    # Assert
+    assert simulation is not None
+    overlay = simulation.building_parts[BuildingPartIdentifier.MAIN]
+    assert overlay.roof_construction_type == "Pitched"
+    assert overlay.roof_insulation_thickness is None
+
+
+@pytest.mark.parametrize(
+    "roof_type_value",
+    [
+        "Another Premises Above",
+        "Roof room(s), insulated",
         "",
     ],
 )

tests/domain/epc/test_wall_type_overlay.py

@@ -29,6 +29,14 @@ def test_solid_brick_with_internal_insulation_overlays_main_wall() -> None:
     ("wall_type_value", "construction", "insulation"),
     [
         ("Cavity wall, as built, no insulation (assumed)", 4, 4),
+        # ADR-0033: the three "as built (assumed)" states all resolve to the
+        # as-built insulation code (4); the age band supplies the U-value, so
+        # "partial"/"insulated" need no distinct code. Elmhurst sweep confirms
+        # cavity As Built = (CAVITY,0) by age (e.g. band F → 1.0 = "partial").
+        ("Cavity wall, as built, partial insulation (assumed)", 4, 4),
+        ("Cavity wall, as built, insulated (assumed)", 4, 4),
+        ("Solid brick, as built, insulated (assumed)", 3, 4),
+        ("Cob, as built", 7, 4),
         ("Cavity wall, with internal insulation", 4, 3),
         ("Cavity wall, with external insulation", 4, 1),
         ("Cavity wall, filled cavity", 4, 2),
@@ -87,9 +95,9 @@ def test_overlay_targets_the_extension_building_part() -> None:
     "wall_type_value",
     [
         "Unknown",
-        # material maps, but the "(assumed) insulated" state is deferred (ADR-0032
-        # — its wall_insulation_type code needs Elmhurst validation), so still None.
-        "Solid brick, as built, insulated (assumed)",
+        # Basement wall is not one of the nine RdSAP materials, so it stays
+        # unresolved (handled by the calculator's basement path, not here).
+        "Basement wall, as built",
         "",
     ],
 )

tests/domain/modelling/test_overlay_applicator.py

@@ -55,6 +55,27 @@ def test_apply_writes_targeted_building_part_and_leaves_others_untouched() -> No
     )
 
 
+def test_flat_roof_construction_type_folds_onto_the_part() -> None:
+    # ADR-0033: a flat-roof landlord override sets `roof_construction_type` so the
+    # calculator's flat path (`"flat" in roof_construction_type`) fires the
+    # age-band default. Proves the generic field loop wires the new overlay field.
+    # Arrange
+    baseline: EpcPropertyData = build_epc()
+    simulation = EpcSimulation(
+        building_parts={
+            BuildingPartIdentifier.MAIN: BuildingPartOverlay(
+                roof_construction_type="Flat"
+            )
+        }
+    )
+
+    # Act
+    result: EpcPropertyData = apply_simulations(baseline, [simulation])
+
+    # Assert
+    assert _part(result, BuildingPartIdentifier.MAIN).roof_construction_type == "Flat"
+
+
 def test_empty_simulation_is_a_no_op() -> None:
     # Arrange
     baseline: EpcPropertyData = build_epc()