mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
Merge branch 'main' into audit/bad-lodged-source-data
This commit is contained in:
Daniel Roth
commit
c73846b448
23 changed files with 1775 additions and 40 deletions
|
|
@ -101,6 +101,10 @@ _Avoid_: energy assessment, site survey, field survey, Domna survey, Hestia surv
|
|||
Property data supplied by a landlord that may correct or supplement the public EPC for a single Property; triggers Rebaselining when applied; not applicable when Site Notes are present.
|
||||
_Avoid_: patches (deprecated), corrections, manual EPC, edits
|
||||
|
||||
**Landlord-Description Classification**:
|
||||
Resolving a **Landlord Description** (unbounded free-text a landlord supplies for one component — "CWI" / "Cav filled" / "cavity insulated" all name one thing) onto a **Recognised Internal Description** via an LLM classifier, persisted in the `landlord_*_overrides` table (`source=classifier`) as a reviewed cache. Four vocabularies are kept **distinct** and must not be conflated: a **Landlord Description** (unbounded input); a **Recognised Internal Description** (the closed target taxonomy — e.g. a `MainHeatingSystemType` archetype — each binding to a Simulation Overlay); a **Lodged Description** (the gov-EPC `main_heating[].description` rendering, e.g. "Room heaters, electric" — only an example of which system *types* occur, never a map key); and the **SAP main heating code** (Table 4a/4b, what the calculator consumes). The classifier maps Landlord → Recognised Internal → SAP code. When it cannot confidently place the text it emits **`None`** (no overlay → the lodged EPC stands, surfaced to the user as "no suitable match"), **never the nearest wrong archetype** — the target taxonomy must be complete enough that a real system always has a correct home, so the classifier never overflows into a garbage-drawer archetype (ADR-0041).
|
||||
_Avoid_: "the LLM mapper is unreliable" (the failure mode is a too-small target taxonomy, not LLM language ability); conflating the landlord input vocabulary with the gov-EPC lodged rendering or the RdSAP entry-tool catalogue; treating a deterministic dict as a *replacement* for the LLM rather than a reviewed cache of its output
|
||||
|
||||
### Modelling
|
||||
|
||||
**Effective EPC**:
|
||||
|
|
@ -116,11 +120,11 @@ Deterministically translating an **old / reduced-data EPC schema** into the curr
|
|||
_Avoid_: gap-fill (means the neighbour-ML path), reduced-data expansion (overloaded with the calculator's Table-5 step), remapping (the schema-translation part only)
|
||||
|
||||
**Baseline Performance**:
|
||||
A Property's current performance aggregate, holding both Lodged Performance and Effective Performance plus the energy block: delivered kWh **per end use** (heating, hot water, lighting, appliances, cooking, pumps/fans, cooling) and the **annual bill** composed into per-section costs plus a total, produced by **Bill Derivation** from SAP10 Calculation's per-end-use kWh × current Fuel Rates. Persisted as one row (flat typed columns, per-section kWh + cost + total); surfaced as one block in the UI.
|
||||
A Property's current performance aggregate, holding both Lodged Performance and Effective Performance plus the energy block: delivered kWh **per end use** (heating, hot water, lighting, appliances, cooking, pumps/fans, cooling) and the **annual bill** composed into per-section costs plus a total, produced by **Bill Derivation** from SAP10 Calculation's per-end-use kWh × current Fuel Rates. Persisted as one row (flat typed columns, per-section kWh + cost + total); surfaced as one block in the UI. The **Lodged half is optional**: a predicted Property has no lodged record (see Lodged Performance), so its `lodged_*` are `NULL` and only the Effective half + energy block are populated (ADR-0004 amendment, #1361).
|
||||
_Avoid_: baseline predictions, predicted baseline, rebaselined values
|
||||
|
||||
**Lodged Performance**:
|
||||
The SAP / EPC Band / carbon emissions / Primary Energy Intensity recorded on the public EPC (or the Site Notes' as-surveyed values when Site Notes are the source) — unmodified by modelling. The half of Baseline Performance that says "what the government register says about this Property".
|
||||
The SAP / EPC Band / carbon emissions / Primary Energy Intensity recorded on the public EPC (or the Site Notes' as-surveyed values when Site Notes are the source) — unmodified by modelling. The half of Baseline Performance that says "what the government register says about this Property". **Requires a record _of this Property_** — a lodged cert or a Site Notes survey — so it is **absent (`None`/NULL) for a predicted Property**: **EPC Prediction** synthesises the picture from neighbours, copying a comparable's recorded figures, so there is no government-register record of _this_ Property to lodge. Only the Effective half is persisted then (ADR-0004 amendment, #1361). Reading a predicted EPC's recorded fields as Lodged Performance manufactures a phantom — a neighbour's SAP presented as this Property's lodged figure.
|
||||
_Avoid_: original performance, raw EPC values, recorded baseline
|
||||
|
||||
**Effective Performance**:
|
||||
|
|
|
|||
|
|
@ -108,7 +108,10 @@ from repositories.product.composite_product_repository import (
|
|||
from repositories.property.in_memory_property_overrides_reader import (
|
||||
InMemoryPropertyOverridesReader,
|
||||
)
|
||||
from repositories.property.landlord_override_overlays import overlays_from
|
||||
from repositories.property.landlord_override_overlays import (
|
||||
flag_fuel_mismatch,
|
||||
overlays_from,
|
||||
)
|
||||
from repositories.property.override_backed_prediction_attributes_reader import (
|
||||
OverrideBackedPredictionAttributesReader,
|
||||
)
|
||||
|
|
@ -561,7 +564,9 @@ def handler(
|
|||
epc = (
|
||||
None # no stored lodged EPC; prediction path handles this property
|
||||
)
|
||||
overrides = overlays_from(overrides_reader.overrides_for(pid))
|
||||
resolved_overrides = overrides_reader.overrides_for(pid)
|
||||
flag_fuel_mismatch(resolved_overrides)
|
||||
overrides = overlays_from(resolved_overrides)
|
||||
predicted_epc: Optional[EpcPropertyData] = None
|
||||
predicted_epc_is_new = False
|
||||
|
||||
|
|
|
|||
|
|
@ -39,3 +39,58 @@ than churning the table twice.
|
|||
The SQLModel row is defined in `infrastructure/postgres/` so the ephemeral-Postgres tests build it
|
||||
via `create_all`; the production migration is FE-owned (Drizzle ORM) and tracked in
|
||||
`docs/migrations/`.
|
||||
|
||||
### Amendment (2026-06-30, #1361 Class B): the Lodged half is absent for a predicted Property
|
||||
|
||||
The original invariant — **every** `PropertyBaselinePerformance` populates **both** halves, even when
|
||||
equal — assumed every Property has a record *of its own* performance to lodge: a public EPC cert (the
|
||||
`epc_with_overlay` path) or a Site Notes survey. It does **not** hold for the **EPC Prediction** path
|
||||
(ADR-0029/0031). A predicted Property has no record of itself — its `EpcPropertyData` is deterministic
|
||||
neighbour synthesis that copies a representative comparable's structure wholesale, so the
|
||||
`energy_rating_current` / band / CO2 / Primary Energy Intensity on the synthesised picture are a
|
||||
*different dwelling's* lodged figures. Reading **Lodged Performance** off it manufactures a **phantom**:
|
||||
a borrowed neighbour's SAP presented as this Property's government-register figure. This affected
|
||||
**every** predicted Property (~12,236 rows estate-wide), not the 8 the `effective-lodged-divergence`
|
||||
audit surfaced — that audit only fires when the borrowed figure lands ≥15 SAP from the Effective, so it
|
||||
under-counts the phantom by three orders of magnitude.
|
||||
|
||||
**Decision.** The Lodged half is **optional**. When `source_path == "predicted"` there is no Lodged
|
||||
Performance: `PropertyBaselinePerformance.lodged` is `None` and the four `lodged_*` columns are `NULL`.
|
||||
The **Effective half is unchanged and still persisted** — a predicted Property is a first-class modelled
|
||||
output (it flows through Rebaselining, Bill Derivation, and Modelling like any other; its Effective
|
||||
Performance and bill block are correct and load-bearing for the FE and the plan-vs-effective audit
|
||||
checks). `rebaseline_reason` stays `physical_state_changed` / `both`, which already records that the
|
||||
Effective figure was scored from a changed picture.
|
||||
|
||||
**Principle (the boundary).** Lodged Performance requires a record **of this Property** — a lodged cert
|
||||
*or* a Site Notes survey (the as-surveyed values are a real observation of this dwelling). EPC
|
||||
Prediction borrows a neighbour's, so it has none. The branch is therefore `source_path == "predicted"` —
|
||||
**not** `physical_state_changed` (true for Site Notes and Landlord Overrides too) and **not** "no public
|
||||
EPC" (Site Notes has none yet keeps a legitimate Lodged Performance). Whether the Site Notes as-surveyed
|
||||
values are truly "lodged" is a separate question, deferred.
|
||||
|
||||
**Rejected — a sentinel (`lodged = 0`).** Considered, to make "no record" visibly present rather than an
|
||||
empty cell. Rejected: `0` is a valid-looking SAP score that (a) re-trips `effective-lodged-divergence`
|
||||
for every predicted Property (`|effective − 0| ≥ 15`), (b) poisons every `AVG(lodged_*)` aggregate that
|
||||
`NULL` is correctly excluded from, and (c) has no coherent `lodged_epc_band` enum member. The "no lodged
|
||||
record" signal belongs in `NULL` (honest absence) plus the **structural provenance** already carried by
|
||||
the distinct predicted-EPC slot — which the FE renders as a predicted badge — not overloaded onto the
|
||||
score column.
|
||||
|
||||
**Consequences.**
|
||||
- The four `lodged_*` columns become **nullable**. The production table is **FE-owned (Drizzle)**, so the
|
||||
migration (`ALTER … DROP NOT NULL`) lands in the FE repo and **must precede** any backend write of a
|
||||
`NULL` lodged, or the predicted-Property INSERT violates the constraint and aborts the batch
|
||||
(ADR-0012). The SQLModel mirror in `infrastructure/postgres/` is updated to `Optional` so the
|
||||
ephemeral-Postgres tests build the nullable shape.
|
||||
- The fix lives in `PropertyBaselineOrchestrator.run()` — the single chokepoint both the First Run
|
||||
pipeline and `applications/modelling_e2e/handler.py` call — so one change repairs both entry points.
|
||||
The Rebaseliner port takes `Optional[Performance]`; for a predicted Property `physical_state_changed`
|
||||
is always true, so `CalculatorRebaseliner` adopts the calculator output and never reads the absent
|
||||
lodged half (the divergence-log path is the pristine-cert case only, where lodged is non-null).
|
||||
- A one-time backfill (`scripts/`, dry-run default + `--apply`, idempotent) NULLs the four `lodged_*`
|
||||
columns on existing predicted-source rows (~12,236). It corrects only the Lodged half; Effective, the
|
||||
bill block, and `rebaseline_reason` are left intact.
|
||||
- `effective-lodged-divergence` already short-circuits on `lodged_sap IS NULL`, so it goes green for
|
||||
predicted Properties once backfilled. Class C of #1361 (a floor that ignores implausibly-low lodged
|
||||
scores on *real* certs) is an additive guard, independent of this change.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
# Landlord-heating classification targets a complete, modellable taxonomy; unmapped input is no-override
|
||||
|
||||
## Status
|
||||
|
||||
accepted
|
||||
|
||||
## Context
|
||||
|
||||
Landlord supplementary data describes a dwelling's components in **unbounded
|
||||
free-text** — a wall is "cavity insulated" / "CWI" / "Cav filled"; a heating
|
||||
system is "communal gas boiler" / "Quantum storage" / a boiler make. An LLM
|
||||
classifier maps that free-text onto a **closed internal taxonomy** of recognised
|
||||
descriptions, each of which binds to a Simulation Overlay applied to
|
||||
`EpcPropertyData` (ADR-0032). For heating the taxonomy is the
|
||||
`MainHeatingSystemType` enum, mapped to a representative SAP Table 4a/4b code by
|
||||
`main_heating_system_overlay`, with coherent companions dragged from the code
|
||||
(ADR-0035). The LLM is the right tool for the unbounded-free-text problem; the
|
||||
deterministic dicts in legacy `asset_list` were a *reviewed cache of LLM output*,
|
||||
not a replacement, and the `landlord_*_overrides` table already serves as that
|
||||
verified cache (keyed `portfolio_id + description`).
|
||||
|
||||
The taxonomy was **too small** — 9 heating archetypes against a ~13-family RdSAP
|
||||
main-heating taxonomy. With no correct target the classifier force-picked the
|
||||
nearest wrong archetype, and treated **"Gas CPSU" as a garbage drawer**: oil and
|
||||
solid-fuel room heaters, community heating, and `"boiler: a rated na"` all
|
||||
classified to Gas CPSU; the word "convector" in `"electric (direct acting) room
|
||||
heaters: panel, convector or radiant heaters"` pulled it to `"Electric storage
|
||||
heaters, convector"` (code 403, off-peak storage) instead of `"Electric room
|
||||
heaters"` (691, direct-acting, single-rate).
|
||||
|
||||
Folded into the Effective EPC, a single-rate dwelling modelled as off-peak
|
||||
storage scores **SAP ~10 (band G)**. This is the true cause of **PRD #1361
|
||||
Class A** — 14 properties lodged in band C/D rebaselining to band G — which had
|
||||
been mis-attributed to a fabric "no insulation (assumed)" assumption and the
|
||||
pre-SAP10 rebaseline. The walls are correct (landlord asserted as-built); the
|
||||
heating is the crater. Exemplar **property 718066**: lodged SAP 57, landlord
|
||||
described "electric (direct acting) room heaters: panel, convector or radiant"
|
||||
(correct code 691, single-rate), mis-stored as code 403 → live SAP **9.9**. The
|
||||
stored override was classified `source=classifier` on 2026-06-20, **before** the
|
||||
691 "Electric room heaters" archetype existed (commit ada2bc07, 2026-06-29) — so
|
||||
it is also stale.
|
||||
|
||||
## Decision
|
||||
|
||||
The fix is the **target taxonomy, not the mapper**. The LLM stays.
|
||||
|
||||
1. **The classifier targets a complete, modellable taxonomy.** Expand
|
||||
`MainHeatingSystemType` + `_MAIN_HEATING_CODES` to cover every RdSAP
|
||||
main-heating family the calculator can score: room heaters by fuel
|
||||
(gas / oil / solid), warm air, **heat pumps**, and **community heating**. Each
|
||||
maps to a representative Table 4a/4b code; coherent companions drag from the
|
||||
code per ADR-0035, so adding an archetype is still "just add its code".
|
||||
|
||||
2. **Unmapped free-text → `None`, never a forced archetype.** The overlay already
|
||||
returns `None` (no overlay → the lodged EPC heating stands) for any value not
|
||||
in `_MAIN_HEATING_CODES`; the classifier must *emit* Unknown/`None` when it
|
||||
cannot confidently place the text, rather than the nearest wrong archetype.
|
||||
"Gas CPSU" (or any other archetype) is never a fallback. The `None` mapping is
|
||||
persisted and surfaced to the user as "no suitable match" for later edit. A
|
||||
forced archetype overwrites a correct lodged cert; `None` (keep lodged) is the
|
||||
only safe and honest behaviour.
|
||||
|
||||
3. **Heat pumps are modellable without a PCDB index** — a "model unknown" heat
|
||||
pump maps to a default Table 4a heat-pump code (211–224) with the table's
|
||||
default seasonal efficiency; no `main_heating_index_number` is required.
|
||||
|
||||
4. **Community heating is modelled via its explicit codes** — 301 (boiler
|
||||
community) / 302 (CHP + boilers) / 304 (electric heat-pump community), which
|
||||
the catalogue strings disambiguate ("community boilers only" / "community CHP
|
||||
and boilers" / "community heat pump"). The calculator (Appendix C §C3.2
|
||||
distribution loss, Table 12c DLF, Table 12 source factors) and Bill Derivation
|
||||
(`HEAT_NETWORK` indicative rate) already support these. **Load-bearing
|
||||
assumption:** when the landlord text does not supply them, the **DLF defaults
|
||||
to Table 12c 1.50** and the source to a boiler — community SAP is *sensitive*
|
||||
to the DLF (1.0 vs 1.5 is a large swing), so this default is documented here
|
||||
and surfaced as an assumption. Coarse "communal heating" with **no named
|
||||
source** falls to `None` rather than inventing a heat-network model.
|
||||
|
||||
5. **Re-classify the stale `source=classifier` overrides** once the taxonomy is
|
||||
complete, so the band-G properties pick up their correct code.
|
||||
|
||||
## Considered Options
|
||||
|
||||
- **Replace the LLM with a deterministic catalogue map** — rejected: landlord
|
||||
input is unbounded free-text, not a closed vocabulary. The LLM is the right
|
||||
front door; determinism belongs as a downstream *verified* cache (the override
|
||||
table), built from LLM output + review, exactly as `asset_list` was.
|
||||
- **Keep the 9-member enum, tune the LLM prompt** — rejected: with no correct
|
||||
target the LLM must pick a wrong archetype regardless of prompt. Target
|
||||
completeness is the fix; prompt quality is secondary.
|
||||
- **Force unmapped input to a conservative archetype** (e.g. direct-acting
|
||||
electric) — rejected: any forced archetype overwrites a correct lodged cert.
|
||||
- **Defer community heating to `None`** (grill option b) — rejected in favour of
|
||||
modelling the three explicit community codes, since calculator + bill already
|
||||
support heat networks and the catalogue strings disambiguate the cases; only
|
||||
coarse unnamed "communal heating" falls to `None`.
|
||||
|
||||
This extends [ADR-0032](0032-landlord-override-epc-overlay.md) (the override
|
||||
overlay) and [ADR-0035](0035-coherent-heating-system-synthesis.md) (coherent
|
||||
companions drag from the code). The visible baseline shift it produces is
|
||||
correct Rebaselining per [ADR-0039](0039-override-aware-rebaselining.md). It
|
||||
**corrects the stated cause of PRD #1361 Class A** (classifier taxonomy gap, not
|
||||
fabric/rebaseline).
|
||||
|
||||
### Amendment: natural-fuel coherence
|
||||
|
||||
A room-heater / direct-acting archetype names a *system*, not its fuel — and fuel
|
||||
arrives as its own composable `main_fuel` override. Two rules keep the system
|
||||
self-coherent without coupling the two overlays:
|
||||
|
||||
- **Every archetype drags a natural fuel so the system self-coheres without a
|
||||
`main_fuel` override.** Where the fuel is unambiguous it is exact — electric
|
||||
room heaters / direct-acting / storage → electricity (RdSAP `main_fuel` 29), a
|
||||
gas boiler → mains gas (26), an oil room heater → oil (28). **Solid fuel is
|
||||
ambiguous** (coal / anthracite / smokeless / dual fuel / wood logs / pellets),
|
||||
so it **defaults to house coal (33)** — the most common solid fuel — which a
|
||||
specific `main_fuel` override then refines. (The `main_fuel` override's own
|
||||
vocabulary must grow to carry the full solid-fuel list — a parallel taxonomy
|
||||
gap, same shape as this one.)
|
||||
- **A present `main_fuel` override wins** by the applicator's last-wins
|
||||
composition (`apply_simulations` setattrs non-`None` fields in order), so the
|
||||
`main_fuel` overlay is applied **after** the heating overlay. The agreeing
|
||||
case (electric system + electricity fuel) is order-immaterial; only a
|
||||
*disagreeing* one depends on the ordering.
|
||||
- **A landlord fuel that contradicts the archetype's natural fuel is *logged*,
|
||||
not raised** (e.g. "gas" fuel on a solid-fuel room heater) — an
|
||||
override-plausibility signal (cf. the ADR-0039 scanner); what to do with it is
|
||||
deferred. The override is still honoured (the landlord wins); we only record
|
||||
the implausibility.
|
||||
|
|
@ -18,10 +18,10 @@ straight lift-and-shift of the columns below.
|
|||
|---|---|---|
|
||||
| `id` | serial PK | |
|
||||
| `property_id` | int, FK → `property.id`, **unique** | one Baseline Performance per Property |
|
||||
| `lodged_sap_score` | int | Lodged Performance — gov register, off the Effective EPC |
|
||||
| `lodged_epc_band` | text | the `Epc` enum, stored as its string value (e.g. `"C"`) |
|
||||
| `lodged_co2_emissions_t_per_yr` | float | tonnes CO₂/yr (whole dwelling) |
|
||||
| `lodged_primary_energy_intensity_kwh_per_m2_yr` | int | PEUI (kWh/m²/yr); **not** "heat demand" — see CONTEXT.md |
|
||||
| `lodged_sap_score` | int, **nullable** | Lodged Performance — gov register, off the Effective EPC. **NULL for a predicted Property** (no lodged cert — #1361, see below) |
|
||||
| `lodged_epc_band` | text, **nullable** | the `Epc` enum, stored as its string value (e.g. `"C"`) |
|
||||
| `lodged_co2_emissions_t_per_yr` | float, **nullable** | tonnes CO₂/yr (whole dwelling) |
|
||||
| `lodged_primary_energy_intensity_kwh_per_m2_yr` | int, **nullable** | PEUI (kWh/m²/yr); **not** "heat demand" — see CONTEXT.md |
|
||||
| `effective_sap_score` | int | Effective Performance — what modelling scored against |
|
||||
| `effective_epc_band` | text | |
|
||||
| `effective_co2_emissions_t_per_yr` | float | tonnes CO₂/yr (whole dwelling) |
|
||||
|
|
@ -30,6 +30,21 @@ straight lift-and-shift of the columns below.
|
|||
| `space_heating_kwh` | float | EPC `renewable_heat_incentive` recorded demand. **Superseded** by `heating_kwh` (delivered) when the bill block populates; kept until then to avoid an empty-kWh gap, dropped in the population slice. |
|
||||
| `water_heating_kwh` | float | EPC `renewable_heat_incentive`; **superseded** by `hot_water_kwh`. |
|
||||
|
||||
### Lodged half is nullable (#1361 Class B, 2026-06-30)
|
||||
|
||||
The four `lodged_*` columns are **nullable**. A **predicted** Property (EPC Prediction, ADR-0029/0031:
|
||||
no lodged cert, its `EpcPropertyData` synthesised from neighbours) has **no Lodged Performance** — the
|
||||
backend now writes `lodged_* = NULL` for it and persists only the Effective half (ADR-0004 amendment).
|
||||
Reading the synthesised EPC's recorded fields as a lodged figure had been manufacturing a phantom (a
|
||||
neighbour's SAP), which the migration + a one-time backfill remove.
|
||||
|
||||
**FE-owned Drizzle migration required:** `ALTER TABLE property_baseline_performance ALTER COLUMN
|
||||
lodged_sap_score DROP NOT NULL` (and the same for `lodged_epc_band`, `lodged_co2_emissions_t_per_yr`,
|
||||
`lodged_primary_energy_intensity_kwh_per_m2_yr`). This **must land before** the backend deploys the
|
||||
orchestrator change or runs `scripts/null_predicted_lodged_performance.py` — a `NULL` write against a
|
||||
`NOT NULL` column aborts the batch (ADR-0012). The backfill NULLs the four columns on the ~12,236
|
||||
existing predicted-source rows; Effective, the bill block, and `rebaseline_reason` are left intact.
|
||||
|
||||
### Bill block (ADR-0014) — the energy bill, composed per section
|
||||
|
||||
Produced by **Bill Derivation**: the calculator's **delivered** kWh per end use priced at current
|
||||
|
|
|
|||
|
|
@ -68,6 +68,61 @@ _ASSUMED_DUAL_METER_CODES = OFF_PEAK_IMPLYING_HEATING_CODES | _ROOM_HEATER_CODES
|
|||
_MANUAL_CHARGE_CONTROL = 2401
|
||||
_STORAGE_HEATER_CODES = frozenset(range(401, 410))
|
||||
|
||||
# SAP Table 4a category 10 ("Room heaters") and its conservative Table 4e Group 6
|
||||
# control. A landlord names a room heater, not its control; code 2601 ("no
|
||||
# thermostatic control of room temperature") is the lowest-SAP room-heater
|
||||
# control — the modal one real solid-fuel room-heater certs lodge — so it never
|
||||
# over-credits an unobserved control (the room-heater mirror of the storage
|
||||
# manual-charge default). Scoped per family as archetypes land; solid-fuel room
|
||||
# heaters are Table 4a 631-636.
|
||||
_ROOM_HEATER_CATEGORY = 10
|
||||
_ROOM_HEATER_CONTROL = 2601
|
||||
_SOLID_FUEL_ROOM_HEATER_CODES = frozenset(range(631, 637))
|
||||
# Oil room heaters (SAP Table 4a 621-625) — category 10, conservative room-heater
|
||||
# control, natural fuel heating oil (RdSAP main_fuel 28).
|
||||
_OIL_ROOM_HEATER_CODES = frozenset(range(621, 626))
|
||||
_OIL_FUEL = 28
|
||||
# Gas (incl. LPG) room heaters (SAP Table 4a 601-613) — category 10, conservative
|
||||
# control, natural fuel mains gas (26); an LPG dwelling is refined by a main_fuel
|
||||
# override (the overlay can't see the mains connection).
|
||||
_GAS_ROOM_HEATER_CODES = frozenset(range(601, 614))
|
||||
# Fuel-burning room heaters (solid + oil + gas) share category 10 + the
|
||||
# conservative room-heater control; only the natural fuel differs by family.
|
||||
# (Distinct from `_ROOM_HEATER_CODES` above, the electric-room-heater dual-meter
|
||||
# set.)
|
||||
_CATEGORY_10_ROOM_HEATER_CODES = (
|
||||
_SOLID_FUEL_ROOM_HEATER_CODES | _OIL_ROOM_HEATER_CODES | _GAS_ROOM_HEATER_CODES
|
||||
)
|
||||
|
||||
# Natural-fuel coherence (ADR-0041): where an archetype unambiguously implies a
|
||||
# fuel, the overlay drags it so a system-only override is self-coherent on a cert
|
||||
# that lodged a different fuel. A later `main_fuel` override still wins (last-wins
|
||||
# composition); a contradicting landlord fuel is logged, not silently overridden.
|
||||
# Electric room heaters (Table 4a 691-701) are unambiguously electricity (RdSAP
|
||||
# main_fuel code 29). Solid fuel is ambiguous (coal / anthracite / smokeless /
|
||||
# dual fuel / wood logs / pellets), so it defaults to house coal (33) — the most
|
||||
# common solid fuel — when the landlord gives no `main_fuel` override; a specific
|
||||
# solid-fuel override still wins by last-wins composition.
|
||||
_ELECTRICITY_FUEL = 29
|
||||
_HOUSE_COAL_FUEL = 33
|
||||
_ELECTRIC_ROOM_HEATER_CODES = frozenset(range(691, 702))
|
||||
|
||||
# Heat pumps (SAP Table 4a 211-224 wet, 521-527 warm-air) are category 4 and
|
||||
# unambiguously electric (natural fuel 29). Modellable on the default code's SPF
|
||||
# without a PCDB index (ADR-0041).
|
||||
_HEAT_PUMP_CATEGORY = 4
|
||||
_HEAT_PUMP_CODES = frozenset(range(211, 225)) | frozenset(range(521, 528))
|
||||
|
||||
# Community / heat-network heating (SAP Table 4a 301-304) is category 6; the
|
||||
# calculator's `_is_heat_network` keys off code OR category 6. The boiler-driven
|
||||
# schemes (301/302/303) are dominantly mains gas (community) — RdSAP main_fuel 20
|
||||
# — per real community certs; the heat-pump scheme (304) is electricity
|
||||
# (community), added with its archetype (ADR-0041).
|
||||
_HEAT_NETWORK_CATEGORY = 6
|
||||
_HEAT_NETWORK_CODES = frozenset({301, 302, 303, 304})
|
||||
_COMMUNITY_BOILER_CODES = frozenset({301, 302, 303})
|
||||
_COMMUNITY_GAS_FUEL = 20
|
||||
|
||||
# SAP Table 4c full boiler-control code: programmer + room thermostat + TRVs. The
|
||||
# landlord names the boiler, not its controls — but a gas boiler installed under
|
||||
# modern Building Regs must carry compliant controls, and this overlay already
|
||||
|
|
@ -109,6 +164,29 @@ _MAIN_HEATING_CODES: dict[str, int] = {
|
|||
"Electric storage heaters, fan": 404,
|
||||
"Direct-acting electric": 191,
|
||||
"Electric room heaters": 691,
|
||||
"Solid fuel room heater, closed": 633,
|
||||
# Default air-source heat pump — SAP Table 4a 211 ("flow temp in other
|
||||
# cases", default SPF 2.30). Modellable without a PCDB index; an actual
|
||||
# product index would refine it (ADR-0041).
|
||||
"Air source heat pump": 211,
|
||||
# Boiler-driven community / heat-network heating — SAP Table 4a 301. The
|
||||
# calculator derives the heat-source efficiency (80%) + DLF (1.50 default)
|
||||
# from the code (cert_to_inputs). A generic "Community heating" with no named
|
||||
# source stays unmapped (-> None) — the source can't be assumed (ADR-0041).
|
||||
"Community heating, boilers": 301,
|
||||
# Community CHP + boilers — SAP Table 4a 302 (heat network, category 6).
|
||||
"Community heating, CHP and boilers": 302,
|
||||
# Modern standalone oil room heater — SAP Table 4a 623 ("Oil room heater,
|
||||
# 2000 or later", no boiler). Natural fuel heating oil (ADR-0041).
|
||||
"Oil room heater, 2000 or later": 623,
|
||||
# Gas room heaters — each catalogue variant to its own SAP Table 4a code
|
||||
# (wide efficiency spread, so no single representative). Natural fuel mains
|
||||
# gas; an LPG dwelling is refined by a `main_fuel` override (ADR-0041).
|
||||
"Gas room heater, condensing fire": 611,
|
||||
"Gas room heater, decorative fuel-effect": 612,
|
||||
"Gas room heater, flush live-effect": 605,
|
||||
"Gas room heater, open flue 1980 or later": 603,
|
||||
"Gas room heater, open flue pre-1980": 601,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -129,6 +207,38 @@ def _control_for(code: int) -> Optional[int]:
|
|||
return _MANUAL_CHARGE_CONTROL
|
||||
if code in _GAS_BOILER_CODES:
|
||||
return _FULL_BOILER_CONTROL
|
||||
if code in _CATEGORY_10_ROOM_HEATER_CODES:
|
||||
return _ROOM_HEATER_CONTROL
|
||||
return None
|
||||
|
||||
|
||||
def _category_for(code: int) -> Optional[int]:
|
||||
"""The SAP Table 4a heating category a code implies, where the archetype
|
||||
fixes it. Room heaters are category 10 — set so a system-only override reads
|
||||
as a room-heater system, not whatever category the replaced system carried."""
|
||||
if code in _CATEGORY_10_ROOM_HEATER_CODES:
|
||||
return _ROOM_HEATER_CATEGORY
|
||||
if code in _HEAT_PUMP_CODES:
|
||||
return _HEAT_PUMP_CATEGORY
|
||||
if code in _HEAT_NETWORK_CODES:
|
||||
return _HEAT_NETWORK_CATEGORY
|
||||
return None
|
||||
|
||||
|
||||
def _natural_fuel_for(code: int) -> Optional[int]:
|
||||
"""The fuel an archetype unambiguously implies (a coherent default), or None
|
||||
where the fuel is ambiguous and must come from the `main_fuel` override. A
|
||||
present `main_fuel` override wins by last-wins composition."""
|
||||
if code in _ELECTRIC_ROOM_HEATER_CODES or code in _HEAT_PUMP_CODES:
|
||||
return _ELECTRICITY_FUEL
|
||||
if code in _SOLID_FUEL_ROOM_HEATER_CODES:
|
||||
return _HOUSE_COAL_FUEL
|
||||
if code in _OIL_ROOM_HEATER_CODES:
|
||||
return _OIL_FUEL
|
||||
if code in _GAS_ROOM_HEATER_CODES:
|
||||
return _MAINS_GAS_FUEL
|
||||
if code in _COMMUNITY_BOILER_CODES:
|
||||
return _COMMUNITY_GAS_FUEL
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -151,6 +261,16 @@ def _gas_boiler_overlay(code: int) -> HeatingOverlay:
|
|||
)
|
||||
|
||||
|
||||
def natural_fuel_for(main_heating_value: str) -> Optional[int]:
|
||||
"""The RdSAP `main_fuel` code a heating archetype unambiguously implies — its
|
||||
natural fuel (ADR-0041) — or None for an unmapped archetype. Exposed so a
|
||||
plausibility check can compare it against a landlord `main_fuel` override."""
|
||||
code = _MAIN_HEATING_CODES.get(main_heating_value)
|
||||
if code is None:
|
||||
return None
|
||||
return _natural_fuel_for(code)
|
||||
|
||||
|
||||
def main_heating_overlay_for(
|
||||
main_heating_value: str, building_part: int
|
||||
) -> Optional[EpcSimulation]:
|
||||
|
|
@ -162,6 +282,8 @@ def main_heating_overlay_for(
|
|||
return EpcSimulation(
|
||||
heating=HeatingOverlay(
|
||||
sap_main_heating_code=code,
|
||||
main_heating_category=_category_for(code),
|
||||
main_fuel_type=_natural_fuel_for(code),
|
||||
meter_type=_meter_for(code),
|
||||
main_heating_control=_control_for(code),
|
||||
# A landlord override describes the existing dwelling, so its assumed
|
||||
|
|
|
|||
|
|
@ -25,4 +25,14 @@ class MainHeatingSystemType(Enum):
|
|||
ELECTRIC_STORAGE_FAN = "Electric storage heaters, fan"
|
||||
DIRECT_ELECTRIC = "Direct-acting electric"
|
||||
ELECTRIC_ROOM_HEATERS = "Electric room heaters"
|
||||
SOLID_FUEL_ROOM_HEATER_CLOSED = "Solid fuel room heater, closed"
|
||||
AIR_SOURCE_HEAT_PUMP = "Air source heat pump"
|
||||
COMMUNITY_BOILERS = "Community heating, boilers"
|
||||
COMMUNITY_CHP_AND_BOILERS = "Community heating, CHP and boilers"
|
||||
OIL_ROOM_HEATER_POST_2000 = "Oil room heater, 2000 or later"
|
||||
GAS_FIRE_CONDENSING = "Gas room heater, condensing fire"
|
||||
GAS_FIRE_DECORATIVE = "Gas room heater, decorative fuel-effect"
|
||||
GAS_FIRE_FLUSH_LIVE_EFFECT = "Gas room heater, flush live-effect"
|
||||
GAS_FIRE_OPEN_FLUE_POST_1980 = "Gas room heater, open flue 1980 or later"
|
||||
GAS_FIRE_OPEN_FLUE_PRE_1980 = "Gas room heater, open flue pre-1980"
|
||||
UNKNOWN = "Unknown"
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class CalculatorRebaseliner(Rebaseliner):
|
|||
self,
|
||||
property_id: int,
|
||||
effective_epc: "EpcPropertyData",
|
||||
lodged: Performance,
|
||||
lodged: Optional[Performance],
|
||||
*,
|
||||
physical_state_changed: bool = False,
|
||||
) -> RebaselineResult:
|
||||
|
|
@ -87,6 +87,11 @@ class CalculatorRebaseliner(Rebaseliner):
|
|||
reason=reason,
|
||||
sap_result=result,
|
||||
)
|
||||
# The pristine-lodged path: not pre-SAP10 and not physical-state-changed, so
|
||||
# there is a real lodged cert to validate against. ``lodged is None`` only
|
||||
# happens for a predicted Property, which is physical_state_changed and so
|
||||
# took the branch above — it never reaches here.
|
||||
assert lodged is not None
|
||||
self._log_divergence(
|
||||
property_id=property_id, sap_version=sap_version, result=result, lodged=lodged
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@ class PropertyBaselinePerformance:
|
|||
Holds both halves — ``lodged`` (what the gov register says) and
|
||||
``effective`` (what the modelling pipeline scored against) — plus the
|
||||
``rebaseline_reason`` recording *why* they differ (``"none"`` when equal).
|
||||
Both halves are always populated, even when equal.
|
||||
Both halves are populated for a Property with a real record of itself; the
|
||||
``lodged`` half is ``None`` for a predicted Property, which has no lodged cert
|
||||
to read a Lodged Performance off — only the Effective half is established then
|
||||
(ADR-0004 amendment, #1361).
|
||||
|
||||
Carries the part of the energy block that needs no derivation: annual
|
||||
``space_heating_kwh`` / ``water_heating_kwh`` read off the EPC's RHI.
|
||||
|
|
@ -25,7 +28,7 @@ class PropertyBaselinePerformance:
|
|||
ran (the stub path produced no ``SapResult`` to price).
|
||||
"""
|
||||
|
||||
lodged: Performance
|
||||
lodged: Optional[Performance]
|
||||
effective: Performance
|
||||
rebaseline_reason: RebaselineReason
|
||||
space_heating_kwh: float
|
||||
|
|
|
|||
|
|
@ -62,11 +62,15 @@ class Rebaseliner(ABC):
|
|||
self,
|
||||
property_id: int,
|
||||
effective_epc: EpcPropertyData,
|
||||
lodged: Performance,
|
||||
lodged: Optional[Performance],
|
||||
*,
|
||||
physical_state_changed: bool = False,
|
||||
) -> RebaselineResult:
|
||||
"""Produce Effective Performance. ``physical_state_changed`` is True when
|
||||
"""Produce Effective Performance. ``lodged`` is ``None`` for a predicted
|
||||
Property — it has no lodged cert, so there is no Lodged Performance to
|
||||
compare against (which is also why ``physical_state_changed`` is always
|
||||
True there: the calculator output IS the Effective Performance).
|
||||
``physical_state_changed`` is True when
|
||||
the Effective EPC was assembled from something other than a pristine
|
||||
lodged cert — Landlord Overrides, Site Notes, or EPC Prediction moved the
|
||||
physical picture (Rebaselining trigger (b)/(c)) — so the accredited lodged
|
||||
|
|
@ -91,7 +95,7 @@ class StubRebaseliner(Rebaseliner):
|
|||
self,
|
||||
property_id: int,
|
||||
effective_epc: EpcPropertyData,
|
||||
lodged: Performance,
|
||||
lodged: Optional[Performance],
|
||||
*,
|
||||
physical_state_changed: bool = False,
|
||||
) -> RebaselineResult:
|
||||
|
|
@ -108,4 +112,8 @@ class StubRebaseliner(Rebaseliner):
|
|||
"Property needs rebaselining (physical state changed by overrides "
|
||||
"/ prediction); this stub does not run the calculator"
|
||||
)
|
||||
# Only a pristine SAP10 cert reaches here, and that always has a Lodged
|
||||
# Performance — ``lodged is None`` implies a predicted Property, which is
|
||||
# ``physical_state_changed`` and so raised above.
|
||||
assert lodged is not None
|
||||
return RebaselineResult(effective=lodged, reason="none", sap_result=None)
|
||||
|
|
|
|||
|
|
@ -48,12 +48,16 @@ class PropertyBaselinePerformanceModel(SQLModel, table=True):
|
|||
# Postgres won't coerce to the enum. Bind the native enums explicitly
|
||||
# (``create_type=False``: the types already exist). Values stay plain
|
||||
# strings on the domain side (``Epc(...)`` / the RebaselineReason Literal).
|
||||
lodged_sap_score: int
|
||||
lodged_epc_band: str = Field(
|
||||
sa_column=Column(SAEnum(*_EPC_BANDS, name="epc", create_type=False), nullable=False)
|
||||
# The four lodged_* columns are nullable as a unit: a predicted Property has no
|
||||
# lodged cert, so it has no Lodged Performance (ADR-0004 amendment, #1361). They
|
||||
# are written all-set or all-NULL; lodged_sap_score is the read discriminator.
|
||||
lodged_sap_score: Optional[int] = None
|
||||
lodged_epc_band: Optional[str] = Field(
|
||||
default=None,
|
||||
sa_column=Column(SAEnum(*_EPC_BANDS, name="epc", create_type=False), nullable=True),
|
||||
)
|
||||
lodged_co2_emissions_t_per_yr: float
|
||||
lodged_primary_energy_intensity_kwh_per_m2_yr: int
|
||||
lodged_co2_emissions_t_per_yr: Optional[float] = None
|
||||
lodged_primary_energy_intensity_kwh_per_m2_yr: Optional[int] = None
|
||||
|
||||
effective_sap_score: int
|
||||
effective_epc_band: str = Field(
|
||||
|
|
@ -102,12 +106,17 @@ class PropertyBaselinePerformanceModel(SQLModel, table=True):
|
|||
def from_domain(
|
||||
cls, baseline: PropertyBaselinePerformance, property_id: int
|
||||
) -> "PropertyBaselinePerformanceModel":
|
||||
# A predicted Property has no Lodged Performance — the four lodged_* columns
|
||||
# are all NULL then (ADR-0004 amendment, #1361).
|
||||
lodged = baseline.lodged
|
||||
model = cls(
|
||||
property_id=property_id,
|
||||
lodged_sap_score=baseline.lodged.sap_score,
|
||||
lodged_epc_band=baseline.lodged.epc_band.value,
|
||||
lodged_co2_emissions_t_per_yr=baseline.lodged.co2_emissions,
|
||||
lodged_primary_energy_intensity_kwh_per_m2_yr=baseline.lodged.primary_energy_intensity,
|
||||
lodged_sap_score=lodged.sap_score if lodged is not None else None,
|
||||
lodged_epc_band=lodged.epc_band.value if lodged is not None else None,
|
||||
lodged_co2_emissions_t_per_yr=lodged.co2_emissions if lodged is not None else None,
|
||||
lodged_primary_energy_intensity_kwh_per_m2_yr=(
|
||||
lodged.primary_energy_intensity if lodged is not None else None
|
||||
),
|
||||
effective_sap_score=baseline.effective.sap_score,
|
||||
effective_epc_band=baseline.effective.epc_band.value,
|
||||
effective_co2_emissions_t_per_yr=baseline.effective.co2_emissions,
|
||||
|
|
@ -139,12 +148,7 @@ class PropertyBaselinePerformanceModel(SQLModel, table=True):
|
|||
|
||||
def to_domain(self) -> PropertyBaselinePerformance:
|
||||
return PropertyBaselinePerformance(
|
||||
lodged=Performance(
|
||||
sap_score=self.lodged_sap_score,
|
||||
epc_band=Epc(self.lodged_epc_band),
|
||||
co2_emissions=self.lodged_co2_emissions_t_per_yr,
|
||||
primary_energy_intensity=self.lodged_primary_energy_intensity_kwh_per_m2_yr,
|
||||
),
|
||||
lodged=self._read_lodged(),
|
||||
effective=Performance(
|
||||
sap_score=self.effective_sap_score,
|
||||
epc_band=Epc(self.effective_epc_band),
|
||||
|
|
@ -157,6 +161,23 @@ class PropertyBaselinePerformanceModel(SQLModel, table=True):
|
|||
bill=self._read_bill(),
|
||||
)
|
||||
|
||||
def _read_lodged(self) -> Optional[Performance]:
|
||||
"""The Lodged half, or None for a predicted Property whose four lodged_*
|
||||
columns are NULL (ADR-0004 amendment, #1361). They are written as a unit
|
||||
(`from_domain`), so lodged_sap_score is the discriminator and the other
|
||||
three are non-null alongside it."""
|
||||
if self.lodged_sap_score is None:
|
||||
return None
|
||||
assert self.lodged_epc_band is not None
|
||||
assert self.lodged_co2_emissions_t_per_yr is not None
|
||||
assert self.lodged_primary_energy_intensity_kwh_per_m2_yr is not None
|
||||
return Performance(
|
||||
sap_score=self.lodged_sap_score,
|
||||
epc_band=Epc(self.lodged_epc_band),
|
||||
co2_emissions=self.lodged_co2_emissions_t_per_yr,
|
||||
primary_energy_intensity=self.lodged_primary_energy_intensity_kwh_per_m2_yr,
|
||||
)
|
||||
|
||||
def _read_bill(self) -> Optional[Bill]:
|
||||
"""Reconstruct the Bill from the ``bill_*`` columns. The total is the
|
||||
not-None discriminator: a persisted bill always sets it, so its absence
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from domain.billing.bill import EnergyBreakdown
|
|||
from domain.sap10_calculator.calculator import SapResult
|
||||
from domain.billing.bill_derivation import BillDerivation
|
||||
from domain.property_baseline.property_baseline_performance import PropertyBaselinePerformance
|
||||
from domain.property_baseline.performance import lodged_performance
|
||||
from domain.property_baseline.performance import Performance, lodged_performance
|
||||
from domain.property_baseline.rebaseliner import Rebaseliner
|
||||
from repositories.fuel_rates.fuel_rates_repository import FuelRatesRepository
|
||||
from repositories.unit_of_work import UnitOfWork
|
||||
|
|
@ -50,7 +50,15 @@ class PropertyBaselineOrchestrator:
|
|||
properties = uow.property.get_many(property_ids)
|
||||
for property_id, prop in zip(property_ids, properties, strict=True):
|
||||
effective_epc = prop.effective_epc
|
||||
lodged = lodged_performance(effective_epc)
|
||||
# A predicted Property has no lodged cert — its Effective EPC is a
|
||||
# neighbour-synthesised picture, so its recorded performance fields
|
||||
# are a different dwelling's. There is no Lodged Performance to read
|
||||
# (ADR-0004 amendment, #1361); only the Effective half is established.
|
||||
lodged: Optional[Performance] = (
|
||||
None
|
||||
if prop.source_path == "predicted"
|
||||
else lodged_performance(effective_epc)
|
||||
)
|
||||
rebaselined = self._rebaseliner.rebaseline(
|
||||
property_id,
|
||||
effective_epc,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ values are live vs no-op before any write.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, Optional
|
||||
|
||||
from domain.epc.property_overlays.attribute_overlay import (
|
||||
|
|
@ -41,6 +42,7 @@ from domain.epc.property_overlays.glazing_overlay import glazing_overlay_for
|
|||
from domain.epc.property_overlays.main_fuel_overlay import fuel_overlay_for
|
||||
from domain.epc.property_overlays.main_heating_system_overlay import (
|
||||
main_heating_overlay_for,
|
||||
natural_fuel_for,
|
||||
)
|
||||
from domain.epc.property_overlays.water_heating_overlay import (
|
||||
water_heating_overlay_for,
|
||||
|
|
@ -50,6 +52,8 @@ from domain.epc.property_overlays.wall_type_overlay import wall_overlay_for
|
|||
from domain.modelling.simulation import EpcSimulation
|
||||
from repositories.property.property_overrides_reader import ResolvedPropertyOverrides
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Each override component maps its value (+ building part) to an overlay, or None
|
||||
# when the value isn't resolvable. Fabric (wall/roof) folds onto building parts;
|
||||
# property_type / built_form_type are whole-dwelling categorical corrections
|
||||
|
|
@ -67,9 +71,22 @@ _COMPONENT_OVERLAYS: dict[str, Callable[[str, int], Optional[EpcSimulation]]] =
|
|||
}
|
||||
|
||||
|
||||
# Components whose overlay must be applied LAST so an explicit value wins a
|
||||
# default another overlay dragged. `apply_simulations` is last-wins and override
|
||||
# rows arrive in arbitrary order, so a `main_fuel` override must be applied after
|
||||
# the `main_heating_system` archetype, whose natural-fuel default it overrides
|
||||
# (ADR-0041) — e.g. "smokeless coal" must beat a solid-fuel room heater's coal
|
||||
# default.
|
||||
_APPLY_LAST: frozenset[str] = frozenset({"main_fuel"})
|
||||
|
||||
|
||||
def overlays_from(overrides: ResolvedPropertyOverrides) -> list[EpcSimulation]:
|
||||
overlays: list[EpcSimulation] = []
|
||||
for row in overrides.rows:
|
||||
# Stable sort: non-`_APPLY_LAST` rows keep their order, `main_fuel` goes last.
|
||||
ordered_rows = sorted(
|
||||
overrides.rows, key=lambda row: row.override_component in _APPLY_LAST
|
||||
)
|
||||
for row in ordered_rows:
|
||||
mapper = _COMPONENT_OVERLAYS.get(row.override_component)
|
||||
if mapper is None:
|
||||
continue
|
||||
|
|
@ -77,3 +94,55 @@ def overlays_from(overrides: ResolvedPropertyOverrides) -> list[EpcSimulation]:
|
|||
if overlay is not None:
|
||||
overlays.append(overlay)
|
||||
return overlays
|
||||
|
||||
|
||||
# Coarse fuel family per RdSAP `main_fuel` code (main_fuel_overlay._FUEL_CODES),
|
||||
# for the plausibility check. The natural fuel a solid-fuel archetype drags
|
||||
# (house coal) is a *default* across the ambiguous solid family, so a same-family
|
||||
# override (smokeless / dual fuel / biomass) is a refinement, not a contradiction
|
||||
# — only a different family (gas/electric on a solid heater) is flagged.
|
||||
_FUEL_FAMILY: dict[int, str] = {
|
||||
26: "gas", 20: "gas",
|
||||
27: "lpg", 3: "lpg", 17: "lpg",
|
||||
28: "oil",
|
||||
29: "electricity", 25: "electricity",
|
||||
33: "solid", 15: "solid", 10: "solid", 31: "solid",
|
||||
}
|
||||
|
||||
|
||||
def _override_value(overrides: ResolvedPropertyOverrides, component: str) -> Optional[str]:
|
||||
for row in overrides.rows:
|
||||
if row.override_component == component:
|
||||
return row.override_value
|
||||
return None
|
||||
|
||||
|
||||
def flag_fuel_mismatch(overrides: ResolvedPropertyOverrides) -> None:
|
||||
"""Log (not raise) when a landlord `main_fuel` override contradicts the
|
||||
`main_heating_system` archetype's natural fuel — a plausibility signal (e.g.
|
||||
a gas fuel on an electric room heater). The override is still honoured (it
|
||||
wins, ADR-0041); we only record the implausibility. Deferred handling."""
|
||||
heating_value = _override_value(overrides, "main_heating_system")
|
||||
fuel_value = _override_value(overrides, "main_fuel")
|
||||
if heating_value is None or fuel_value is None:
|
||||
return
|
||||
natural = natural_fuel_for(heating_value)
|
||||
fuel_overlay = fuel_overlay_for(fuel_value, 0)
|
||||
override_fuel = (
|
||||
fuel_overlay.heating.main_fuel_type
|
||||
if fuel_overlay is not None and fuel_overlay.heating is not None
|
||||
else None
|
||||
)
|
||||
if natural is None or not isinstance(override_fuel, int):
|
||||
return
|
||||
natural_family = _FUEL_FAMILY.get(natural)
|
||||
override_family = _FUEL_FAMILY.get(override_fuel)
|
||||
if natural_family is None or override_family is None:
|
||||
return
|
||||
if natural_family != override_family:
|
||||
logger.warning(
|
||||
"Landlord main_fuel %r contradicts the natural fuel of heating "
|
||||
"system %r (override is honoured; flagged for review)",
|
||||
fuel_value,
|
||||
heating_value,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,10 @@ from domain.property.properties import Properties
|
|||
from domain.property.property import Property, PropertyIdentity
|
||||
from infrastructure.postgres.property_table import PropertyRow
|
||||
from repositories.epc.epc_repository import EpcRepository
|
||||
from repositories.property.landlord_override_overlays import overlays_from
|
||||
from repositories.property.landlord_override_overlays import (
|
||||
flag_fuel_mismatch,
|
||||
overlays_from,
|
||||
)
|
||||
from repositories.property.property_overrides_reader import PropertyOverridesReader
|
||||
from repositories.property.property_repository import (
|
||||
PropertyIdentityInsert,
|
||||
|
|
@ -63,7 +66,9 @@ class PropertyPostgresRepository(PropertyRepository):
|
|||
no reader is wired (the overlay stays off) or the Property has none."""
|
||||
if self._overrides_reader is None:
|
||||
return []
|
||||
return overlays_from(self._overrides_reader.overrides_for(property_id))
|
||||
overrides = self._overrides_reader.overrides_for(property_id)
|
||||
flag_fuel_mismatch(overrides)
|
||||
return overlays_from(overrides)
|
||||
|
||||
def get(self, property_id: int) -> Property:
|
||||
row = self._session.get(PropertyRow, property_id)
|
||||
|
|
|
|||
414
scripts/data_exports.py
Normal file
414
scripts/data_exports.py
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
"""Principal-pitch data export — new DDD model edition.
|
||||
|
||||
Replaces sfr/principal_pitch/2_export_data.py, which read the retired
|
||||
``plan_recommendations`` m2m and ``recommendation_materials`` table. In the
|
||||
current model:
|
||||
* a Recommendation links to its Plan directly (``recommendation.plan_id``),
|
||||
* materials are inline on the Recommendation (``material_id`` etc.),
|
||||
* the chosen Plan per (scenario, property) is the one with ``is_default``,
|
||||
* post-works SAP/EPC + savings live on the Plan row (the new SAP calculator's
|
||||
output), so we read them directly rather than summing recommendation uplifts.
|
||||
|
||||
Give it a portfolio id; it resolves every *modelled* scenario for that portfolio
|
||||
(scenarios that have plans) and writes ONE workbook with a ``properties`` sheet
|
||||
per scenario. EPC descriptive fields (walls/roof/heating/windows/floor area/
|
||||
lodgement) come live from the EPC service, because ``property_details_epc`` is
|
||||
dead under the new backend.
|
||||
|
||||
python scripts/data_exports.py --portfolio 814
|
||||
python scripts/data_exports.py --portfolio 814 --out "sfr/principal_pitch/Durkan.xlsx"
|
||||
|
||||
Reads DB_* + OPEN_EPC_API_TOKEN from backend/.env. Run from the worktree root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
import sys
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(_REPO_ROOT))
|
||||
|
||||
from backend.app.utils import sap_to_epc # noqa: E402
|
||||
from infrastructure.epc_client.epc_client_service import EpcClientService # noqa: E402
|
||||
from scripts.e2e_common import ENV_PATH, build_engine, load_env # noqa: E402
|
||||
from backend.app.config import get_settings # noqa: E402
|
||||
|
||||
# Measure columns always present in the wide sheet (stable column set across runs).
|
||||
EXPECTED_MEASURE_COLUMNS: tuple[str, ...] = (
|
||||
"suspended_floor_insulation",
|
||||
"solid_floor_insulation",
|
||||
"external_wall_insulation",
|
||||
"internal_wall_insulation",
|
||||
"cavity_wall_insulation",
|
||||
"loft_insulation",
|
||||
"flat_roof_insulation",
|
||||
"room_roof_insulation",
|
||||
"secondary_glazing",
|
||||
"double_glazing",
|
||||
"solar_pv",
|
||||
"high_heat_retention_storage_heaters",
|
||||
"air_source_heat_pump",
|
||||
"boiler_upgrade",
|
||||
"gas_boiler_upgrade",
|
||||
"roomstat_programmer_trvs",
|
||||
"time_temperature_zone_control",
|
||||
"low_energy_lighting",
|
||||
"mechanical_ventilation",
|
||||
"system_tune_up",
|
||||
"system_tune_up_zoned",
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# EPC descriptive fields (live from the EPC service)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _description_text(item: Any) -> str:
|
||||
if not isinstance(item, dict):
|
||||
return ""
|
||||
desc = item.get("description")
|
||||
if isinstance(desc, dict):
|
||||
desc = desc.get("value")
|
||||
return str(desc or "")
|
||||
|
||||
|
||||
def _join_descriptions(value: Any) -> str:
|
||||
if isinstance(value, list):
|
||||
return "; ".join(t for t in (_description_text(d) for d in value) if t)
|
||||
return _description_text(value)
|
||||
|
||||
|
||||
# Gov RdSAP property-type codes (the raw cert stores a code, not a word).
|
||||
_PROPERTY_TYPE_CODES: dict[str, str] = {
|
||||
"0": "House", "1": "Bungalow", "2": "Flat", "3": "Maisonette", "4": "Park home",
|
||||
}
|
||||
|
||||
|
||||
def _decode_property_type(value: Any) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
s = str(value).strip()
|
||||
if s in _PROPERTY_TYPE_CODES:
|
||||
return _PROPERTY_TYPE_CODES[s]
|
||||
return s or None
|
||||
|
||||
|
||||
def _is_expired(registration_date: Optional[str]) -> Optional[bool]:
|
||||
if not registration_date:
|
||||
return None
|
||||
try:
|
||||
lodged = datetime.fromisoformat(registration_date[:10]).date()
|
||||
except ValueError:
|
||||
return None
|
||||
return (date.today() - lodged).days > 365 * 10
|
||||
|
||||
|
||||
def epc_details_from_service(svc: EpcClientService, uprn: Optional[int]) -> dict[str, Any]:
|
||||
"""Flatten the UPRN's latest raw certificate into the descriptive fields the
|
||||
export needs. Returns ``{}`` when the UPRN has no EPC (blank columns)."""
|
||||
if uprn is None:
|
||||
return {}
|
||||
results = svc._search(uprn=uprn) # pyright: ignore[reportPrivateUsage]
|
||||
if not results:
|
||||
return {}
|
||||
latest = max(results, key=lambda r: r.registration_date)
|
||||
raw = svc._fetch_certificate(latest.certificate_number) # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
def _to_int(value: Any) -> Optional[int]:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
current_sap = _to_int(raw.get("energy_rating_current"))
|
||||
return {
|
||||
"property_type": _decode_property_type(raw.get("property_type")),
|
||||
"walls": _join_descriptions(raw.get("walls")),
|
||||
"roof": _join_descriptions(raw.get("roofs")),
|
||||
"floor": _join_descriptions(raw.get("floors")),
|
||||
"windows": _join_descriptions(raw.get("window")),
|
||||
"heating": _join_descriptions(raw.get("main_heating")),
|
||||
"hot_water": _join_descriptions(raw.get("hot_water")),
|
||||
"lighting": _join_descriptions(raw.get("lighting")),
|
||||
"total_floor_area": raw.get("total_floor_area"),
|
||||
"lodgement_date": raw.get("registration_date"),
|
||||
"is_expired": _is_expired(raw.get("registration_date")),
|
||||
"current_epc_rating": raw.get("current_energy_efficiency_band"),
|
||||
"current_sap_points": current_sap,
|
||||
"original_sap_points": current_sap,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DB reads (new model: scenario -> plan(is_default) -> recommendation)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def modelled_scenarios(engine: Engine, portfolio_id: int) -> list[dict[str, Any]]:
|
||||
"""Scenarios for the portfolio that actually have plans, newest first."""
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT s.id, s.name
|
||||
FROM scenario s
|
||||
WHERE s.portfolio_id = :p
|
||||
AND EXISTS (SELECT 1 FROM plan pl WHERE pl.scenario_id = s.id)
|
||||
ORDER BY s.id
|
||||
"""
|
||||
),
|
||||
{"p": portfolio_id},
|
||||
).mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def load_properties(engine: Engine, portfolio_id: int, svc: EpcClientService) -> pd.DataFrame:
|
||||
"""Base property identity (property_type falls back to the landlord override)
|
||||
plus live EPC descriptive fields."""
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT p.id AS property_id, p.id AS id, p.uprn, p.address, p.postcode,
|
||||
p.landlord_property_id, p.number_of_rooms,
|
||||
COALESCE(p.property_type, po.override_value) AS property_type
|
||||
FROM property p
|
||||
LEFT JOIN property_overrides po
|
||||
ON po.property_id = p.id
|
||||
AND po.override_component = 'property_type'
|
||||
AND po.building_part = 0
|
||||
WHERE p.portfolio_id = :p
|
||||
ORDER BY p.id
|
||||
"""
|
||||
),
|
||||
{"p": portfolio_id},
|
||||
).mappings().all()
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
for i, r in enumerate(rows, 1):
|
||||
base: dict[str, Any] = dict(r)
|
||||
uprn = int(base["uprn"]) if base.get("uprn") is not None else None
|
||||
for key, value in epc_details_from_service(svc, uprn).items():
|
||||
if base.get(key) is None:
|
||||
base[key] = value
|
||||
records.append(base)
|
||||
if i % 50 == 0:
|
||||
print(f" EPC fetched {i}/{len(rows)}")
|
||||
df = pd.DataFrame(records)
|
||||
df["uprn"] = df["uprn"].astype("string")
|
||||
return df
|
||||
|
||||
|
||||
def load_recommendations(engine: Engine, scenario_id: int) -> pd.DataFrame:
|
||||
"""Default, not-already-installed recommendations on each property's default
|
||||
plan for the scenario, with the material type/battery flag joined."""
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT pl.property_id,
|
||||
r.measure_type,
|
||||
r.description,
|
||||
r.estimated_cost,
|
||||
r.sap_points,
|
||||
r.co2_equivalent_savings,
|
||||
r.kwh_savings,
|
||||
r.energy_cost_savings,
|
||||
m.type AS material_type,
|
||||
COALESCE(m.includes_battery, FALSE) AS includes_battery
|
||||
FROM recommendation r
|
||||
JOIN plan pl ON pl.id = r.plan_id
|
||||
LEFT JOIN material m ON m.id = r.material_id
|
||||
WHERE pl.scenario_id = :s
|
||||
AND pl.is_default = TRUE
|
||||
AND r.default = TRUE
|
||||
AND r.already_installed = FALSE
|
||||
"""
|
||||
),
|
||||
{"s": scenario_id},
|
||||
).mappings().all()
|
||||
return pd.DataFrame([dict(r) for r in rows])
|
||||
|
||||
|
||||
def load_default_plans(engine: Engine, scenario_id: int) -> pd.DataFrame:
|
||||
"""The chosen (is_default) plan per property — the new SAP calculator's
|
||||
post-works results."""
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT property_id, post_sap_points, post_epc_rating,
|
||||
cost_of_works, contingency_cost, co2_savings,
|
||||
energy_bill_savings, energy_consumption_savings,
|
||||
valuation_increase
|
||||
FROM plan
|
||||
WHERE scenario_id = :s AND is_default = TRUE
|
||||
"""
|
||||
),
|
||||
{"s": scenario_id},
|
||||
).mappings().all()
|
||||
return pd.DataFrame([dict(r) for r in rows])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Sheet building
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _apply_battery_suffix(recs: pd.DataFrame) -> pd.DataFrame:
|
||||
"""solar_pv recommendations that carry a battery material become
|
||||
solar_pv_with_battery (mirrors the old export)."""
|
||||
if recs.empty:
|
||||
return recs
|
||||
is_solar_battery = (recs["material_type"] == "solar_pv") & (recs["includes_battery"])
|
||||
recs = recs.copy()
|
||||
recs["measure_type"] = np.where(
|
||||
is_solar_battery,
|
||||
recs["measure_type"].astype(str) + "_with_battery",
|
||||
recs["measure_type"],
|
||||
)
|
||||
return recs
|
||||
|
||||
|
||||
def build_scenario_sheet(
|
||||
properties_df: pd.DataFrame, recs: pd.DataFrame, plans: pd.DataFrame
|
||||
) -> pd.DataFrame:
|
||||
recs = _apply_battery_suffix(recs)
|
||||
|
||||
# Pivot: one column per measure_type holding its estimated_cost.
|
||||
if not recs.empty:
|
||||
deduped = recs.drop_duplicates(subset=["property_id", "measure_type"], keep="first")
|
||||
cost_pivot = deduped.pivot(
|
||||
index="property_id", columns="measure_type", values="estimated_cost"
|
||||
).reset_index()
|
||||
sap_uplift = (
|
||||
recs.groupby("property_id")["sap_points"].sum().reset_index(name="sap_points")
|
||||
)
|
||||
savings = (
|
||||
recs.groupby("property_id")[
|
||||
["co2_equivalent_savings", "kwh_savings", "energy_cost_savings"]
|
||||
]
|
||||
.sum()
|
||||
.reset_index()
|
||||
)
|
||||
else:
|
||||
cost_pivot = pd.DataFrame({"property_id": []})
|
||||
sap_uplift = pd.DataFrame({"property_id": [], "sap_points": []})
|
||||
savings = pd.DataFrame(
|
||||
{"property_id": [], "co2_equivalent_savings": [], "kwh_savings": [], "energy_cost_savings": []}
|
||||
)
|
||||
|
||||
id_cols = [
|
||||
c
|
||||
for c in [
|
||||
"landlord_property_id", "property_id", "uprn", "address", "postcode",
|
||||
"property_type", "walls", "roof", "heating", "windows",
|
||||
"current_epc_rating", "current_sap_points", "original_sap_points",
|
||||
"total_floor_area", "number_of_rooms", "lodgement_date", "is_expired", "id",
|
||||
]
|
||||
if c in properties_df.columns
|
||||
]
|
||||
|
||||
df = (
|
||||
properties_df[id_cols]
|
||||
.merge(cost_pivot, how="left", on="property_id")
|
||||
.merge(sap_uplift, how="left", on="property_id")
|
||||
.merge(savings, how="left", on="property_id")
|
||||
.merge(plans, how="left", on="property_id")
|
||||
)
|
||||
|
||||
# total retrofit cost = sum of the per-measure cost columns
|
||||
measure_cols_present = [c for c in df.columns if c in set(EXPECTED_MEASURE_COLUMNS)
|
||||
or c.endswith("_with_battery")]
|
||||
df["total_retrofit_cost"] = df[measure_cols_present].sum(axis=1) if measure_cols_present else 0.0
|
||||
|
||||
df["sap_points"] = df["sap_points"].fillna(0)
|
||||
# Post-works SAP/EPC straight from the new SAP calculator's plan row;
|
||||
# fall back to current + uplift / sap_to_epc only when the plan lacks them.
|
||||
df["predicted_post_works_sap"] = df["post_sap_points"].where(
|
||||
df["post_sap_points"].notna(), df.get("current_sap_points", 0) + df["sap_points"]
|
||||
)
|
||||
df["predicted_post_works_epc"] = df["post_epc_rating"].where(
|
||||
df["post_epc_rating"].notna(),
|
||||
df["predicted_post_works_sap"].apply(lambda x: sap_to_epc(x) if pd.notna(x) else None),
|
||||
)
|
||||
|
||||
# ensure the stable measure column set exists
|
||||
for col in EXPECTED_MEASURE_COLUMNS:
|
||||
if col not in df.columns:
|
||||
df[col] = ""
|
||||
return df
|
||||
|
||||
|
||||
def _safe_sheet_name(name: str, used: set[str]) -> str:
|
||||
clean = re.sub(r"[:\\/?*\[\]]", "", name or "scenario").strip() or "scenario"
|
||||
clean = clean[:31]
|
||||
base, i = clean, 1
|
||||
while clean in used:
|
||||
suffix = f" ({i})"
|
||||
clean = base[: 31 - len(suffix)] + suffix
|
||||
i += 1
|
||||
used.add(clean)
|
||||
return clean
|
||||
|
||||
|
||||
def export_portfolio(portfolio_id: int, out_path: Path) -> None:
|
||||
load_env(ENV_PATH)
|
||||
settings = get_settings()
|
||||
engine = build_engine()
|
||||
svc = EpcClientService(auth_token=settings.OPEN_EPC_API_TOKEN)
|
||||
|
||||
with engine.connect() as conn:
|
||||
pname = conn.execute(
|
||||
text("SELECT name FROM portfolio WHERE id = :p"), {"p": portfolio_id}
|
||||
).scalar()
|
||||
|
||||
scenarios = modelled_scenarios(engine, portfolio_id)
|
||||
if not scenarios:
|
||||
raise SystemExit(f"No modelled scenarios (with plans) for portfolio {portfolio_id}.")
|
||||
print(f"Portfolio {portfolio_id} ({pname}) — {len(scenarios)} modelled scenario(s): "
|
||||
f"{[s['name'] for s in scenarios]}")
|
||||
|
||||
print("Loading properties + EPC descriptive fields…")
|
||||
properties_df = load_properties(engine, portfolio_id, svc)
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
used_names: set[str] = set()
|
||||
with pd.ExcelWriter(out_path) as writer:
|
||||
for s in scenarios:
|
||||
recs = load_recommendations(engine, s["id"])
|
||||
plans = load_default_plans(engine, s["id"])
|
||||
sheet_df = build_scenario_sheet(properties_df, recs, plans)
|
||||
sheet = _safe_sheet_name(s["name"] or f"scenario_{s['id']}", used_names)
|
||||
sheet_df.to_excel(writer, sheet_name=sheet, index=False)
|
||||
print(f" sheet {sheet!r}: {len(sheet_df)} properties, "
|
||||
f"{0 if recs.empty else len(recs)} recommendations")
|
||||
print(f"Wrote {out_path}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--portfolio", type=int, required=True)
|
||||
ap.add_argument("--out", type=Path, default=None,
|
||||
help="output xlsx path (default: sfr/principal_pitch/<portfolio name>.xlsx)")
|
||||
args = ap.parse_args()
|
||||
|
||||
out = args.out
|
||||
if out is None:
|
||||
load_env(ENV_PATH)
|
||||
with build_engine().connect() as conn:
|
||||
nm = conn.execute(text("SELECT name FROM portfolio WHERE id=:p"), {"p": args.portfolio}).scalar()
|
||||
safe = re.sub(r"[\\/:*?\"<>|]", "_", str(nm or f"portfolio_{args.portfolio}"))
|
||||
out = _REPO_ROOT / "sfr" / "principal_pitch" / f"{safe}.xlsx"
|
||||
export_portfolio(args.portfolio, out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
111
scripts/null_predicted_lodged_performance.py
Normal file
111
scripts/null_predicted_lodged_performance.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
"""One-time backfill: NULL the phantom Lodged Performance on predicted Properties.
|
||||
|
||||
#1361 Class B: ``PropertyBaselineOrchestrator`` used to read Lodged Performance
|
||||
off a predicted Property's neighbour-synthesised EPC, persisting a phantom set of
|
||||
``lodged_*`` figures on ``property_baseline_performance`` — a *different
|
||||
dwelling's* SAP / band / carbon / Primary Energy Intensity presented as this
|
||||
Property's government-register record. The orchestrator no longer does this
|
||||
(``lodged`` is ``None`` when ``source_path == "predicted"``, ADR-0004 amendment),
|
||||
but the ~12k rows written before the fix still carry the phantom.
|
||||
|
||||
This NULLs the four ``lodged_*`` columns on every predicted-source baseline row,
|
||||
leaving the **Effective** half, the **bill** block, and ``rebaseline_reason``
|
||||
untouched (a predicted Property's Effective Performance is correct — it is a
|
||||
first-class modelled output).
|
||||
|
||||
A predicted-source Property is identified exactly as the orchestrator's
|
||||
``source_path == "predicted"``: it has a **predicted** EPC and **no lodged** EPC.
|
||||
Site Notes keep their as-surveyed Lodged Performance and so are excluded — though
|
||||
no Site-Notes-sourced EPC exists in ``epc_property`` today, the predicate (a
|
||||
predicted EPC, no lodged EPC) holds regardless.
|
||||
|
||||
DRY-RUN BY DEFAULT: prints the count it would change and writes nothing. Pass
|
||||
``--apply`` to execute inside a transaction. **Idempotent** — only rows whose
|
||||
``lodged_sap_score`` is still non-NULL are touched, so a second run is a no-op.
|
||||
|
||||
Requires the FE-owned Drizzle ``ALTER ... DROP NOT NULL`` on the four ``lodged_*``
|
||||
columns to have landed first; without it the UPDATE to NULL violates the
|
||||
constraint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import Connection, text
|
||||
|
||||
from scripts.e2e_common import build_engine, load_env
|
||||
|
||||
# A predicted-source baseline row: a predicted EPC exists for the property and no
|
||||
# lodged one does (``source_path == "predicted"``). ``lodged_sap_score IS NOT
|
||||
# NULL`` makes it idempotent — a row already nulled is skipped on a re-run.
|
||||
_PREDICTED_PHANTOM_PREDICATE = """
|
||||
pbp.lodged_sap_score IS NOT NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM epc_property e
|
||||
WHERE e.property_id = pbp.property_id AND e.source = 'predicted'
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM epc_property e
|
||||
WHERE e.property_id = pbp.property_id AND e.source = 'lodged'
|
||||
)
|
||||
"""
|
||||
|
||||
_COUNT = text(
|
||||
f"""
|
||||
SELECT count(*) FROM property_baseline_performance pbp
|
||||
WHERE {_PREDICTED_PHANTOM_PREDICATE}
|
||||
"""
|
||||
)
|
||||
|
||||
_NULL_LODGED = text(
|
||||
f"""
|
||||
UPDATE property_baseline_performance AS pbp
|
||||
SET lodged_sap_score = NULL,
|
||||
lodged_epc_band = NULL,
|
||||
lodged_co2_emissions_t_per_yr = NULL,
|
||||
lodged_primary_energy_intensity_kwh_per_m2_yr = NULL
|
||||
WHERE {_PREDICTED_PHANTOM_PREDICATE}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def backfill(conn: Connection, *, apply: bool) -> int:
|
||||
"""NULL the four ``lodged_*`` columns on predicted-source baseline rows.
|
||||
|
||||
Returns the number of phantom rows found (those that ``--apply`` would / did
|
||||
null). Reads the count first so the dry-run reports it without writing.
|
||||
"""
|
||||
found = conn.execute(_COUNT).scalar() or 0
|
||||
if apply:
|
||||
conn.execute(_NULL_LODGED)
|
||||
return found
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env()
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="execute the update (default: dry-run, writes nothing)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = build_engine()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("SET statement_timeout = 120000"))
|
||||
found = backfill(conn, apply=args.apply)
|
||||
|
||||
verb = "NULLed" if args.apply else "would NULL"
|
||||
print(
|
||||
f"{verb} the Lodged Performance (lodged_* → NULL) on {found} "
|
||||
"predicted-source baseline row(s); Effective / bill / rebaseline_reason "
|
||||
"left intact."
|
||||
)
|
||||
if not args.apply:
|
||||
print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
176
scripts/reclassify_heating_overrides.py
Normal file
176
scripts/reclassify_heating_overrides.py
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
"""One-time re-classification of stale landlord main-heating-system overrides.
|
||||
|
||||
PRD #1361 Class A: the LLM landlord-description classifier ran against a
|
||||
too-small `MainHeatingSystemType` taxonomy (ADR-0041), so canonical RdSAP
|
||||
heating descriptions were stored against the wrong archetype — community heating
|
||||
and oil/solid room heaters dumped into "Gas CPSU", and direct-acting
|
||||
"panel, convector or radiant" room heaters mis-read as off-peak storage (the
|
||||
band-G crater). The taxonomy is now complete, so these *canonical* descriptions
|
||||
re-resolve deterministically — no LLM, no nondeterminism.
|
||||
|
||||
This rewrites the resolved archetype on every affected row in BOTH the
|
||||
per-Property fact layer (`property_overrides.override_value`) and the
|
||||
per-portfolio classifier cache (`landlord_main_heating_system_overrides.value`),
|
||||
keyed on the raw description (`original_spreadsheet_description` / `description`).
|
||||
|
||||
DRY-RUN BY DEFAULT: prints the row counts it would change and writes nothing.
|
||||
Pass `--apply` to execute inside a transaction. Idempotent — only rows whose
|
||||
stored value differs from the corrected archetype are touched, so a second run
|
||||
is a no-op. Descriptions with no confident archetype yet (solid-fuel open-fire /
|
||||
with-boiler, warm air, electric underfloor) are deliberately NOT remapped here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from domain.epc.property_overrides.main_heating_system_type import (
|
||||
MainHeatingSystemType,
|
||||
)
|
||||
from scripts.e2e_common import build_engine, load_env
|
||||
|
||||
# Canonical RdSAP main-heating description (lowercased, as stored in
|
||||
# `original_spreadsheet_description` / `description`) → the corrected archetype.
|
||||
# Only descriptions whose correct archetype now exists in the taxonomy appear
|
||||
# here; the value is a `MainHeatingSystemType` so a typo can't slip a non-archetype
|
||||
# string into the DB (the enum is the single source of truth — ADR-0041).
|
||||
REMAP: dict[str, MainHeatingSystemType] = {
|
||||
"community heating systems: community boilers only (rdsap)": (
|
||||
MainHeatingSystemType.COMMUNITY_BOILERS
|
||||
),
|
||||
"community heating systems: community chp and boilers (rdsap)": (
|
||||
MainHeatingSystemType.COMMUNITY_CHP_AND_BOILERS
|
||||
),
|
||||
"electric (direct acting) room heaters: panel, convector or radiant heaters": (
|
||||
MainHeatingSystemType.ELECTRIC_ROOM_HEATERS
|
||||
),
|
||||
"oil room heaters: room heater, 2000 or later": (
|
||||
MainHeatingSystemType.OIL_ROOM_HEATER_POST_2000
|
||||
),
|
||||
"solid fuel room heaters: closed room heater": (
|
||||
MainHeatingSystemType.SOLID_FUEL_ROOM_HEATER_CLOSED
|
||||
),
|
||||
"gas (including lpg) room heaters: condensing gas fire": (
|
||||
MainHeatingSystemType.GAS_FIRE_CONDENSING
|
||||
),
|
||||
"gas (including lpg) room heaters: decorative fuel effect gas fire, "
|
||||
"open to chimney": MainHeatingSystemType.GAS_FIRE_DECORATIVE,
|
||||
"gas (including lpg) room heaters: flush fitting live fuel effect gas fire "
|
||||
"(open fronted), sealed to fireplace opening": (
|
||||
MainHeatingSystemType.GAS_FIRE_FLUSH_LIVE_EFFECT
|
||||
),
|
||||
"gas (including lpg) room heaters: gas fire, open flue, 1980 or later "
|
||||
"(open fronted), sitting proud of, and sealed to, fireplace opening": (
|
||||
MainHeatingSystemType.GAS_FIRE_OPEN_FLUE_POST_1980
|
||||
),
|
||||
"gas (including lpg) room heaters: gas fire, open flue, pre-1980 "
|
||||
"(open fronted)": MainHeatingSystemType.GAS_FIRE_OPEN_FLUE_PRE_1980,
|
||||
}
|
||||
|
||||
# property_overrides keys the raw text on `original_spreadsheet_description`; the
|
||||
# classifier cache keys it on `description`. Both compared case-insensitively.
|
||||
_PROPERTY_OVERRIDES_UPDATE = text(
|
||||
"""
|
||||
UPDATE property_overrides
|
||||
SET override_value = :new_value
|
||||
WHERE override_component = 'main_heating_system'
|
||||
AND lower(original_spreadsheet_description) = :description
|
||||
AND override_value <> :new_value
|
||||
"""
|
||||
)
|
||||
# The cache `value` column is the Drizzle-owned `main_heating_system` pgEnum;
|
||||
# compare/update it as text (no CAST) so a target value the DB enum does not yet
|
||||
# carry doesn't error the read. The UPDATE is only issued for values the live
|
||||
# enum already has — `_db_enum_values` gates it (see main()).
|
||||
_CACHE_UPDATE = text(
|
||||
"""
|
||||
UPDATE landlord_main_heating_system_overrides
|
||||
SET value = :new_value, updated_at = now()
|
||||
WHERE lower(description) = :description
|
||||
AND value::text <> :new_value
|
||||
"""
|
||||
)
|
||||
_PROPERTY_OVERRIDES_COUNT = text(
|
||||
"""
|
||||
SELECT count(*) FROM property_overrides
|
||||
WHERE override_component = 'main_heating_system'
|
||||
AND lower(original_spreadsheet_description) = :description
|
||||
AND override_value <> :new_value
|
||||
"""
|
||||
)
|
||||
_CACHE_COUNT = text(
|
||||
"""
|
||||
SELECT count(*) FROM landlord_main_heating_system_overrides
|
||||
WHERE lower(description) = :description
|
||||
AND value::text <> :new_value
|
||||
"""
|
||||
)
|
||||
_ENUM_VALUES = text(
|
||||
"SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid "
|
||||
"WHERE t.typname = 'main_heating_system'"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env()
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="execute the updates (default: dry-run, writes nothing)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = build_engine()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("SET statement_timeout = 120000"))
|
||||
# The cache `value` is the Drizzle-owned pgEnum; only update it for
|
||||
# archetypes the live enum already carries. The rest need the Drizzle
|
||||
# migration (FE repo) before the cache — and the classifier — can store
|
||||
# them; property_overrides (TEXT, what the modelling reads) is updated
|
||||
# regardless, which is the actual band-G fix.
|
||||
enum_values = {r[0] for r in conn.execute(_ENUM_VALUES)}
|
||||
prop_total = 0
|
||||
cache_total = 0
|
||||
deferred: list[str] = []
|
||||
for description, archetype in REMAP.items():
|
||||
params = {"description": description, "new_value": archetype.value}
|
||||
prop_n = conn.execute(_PROPERTY_OVERRIDES_COUNT, params).scalar() or 0
|
||||
prop_total += prop_n
|
||||
in_enum = archetype.value in enum_values
|
||||
cache_n = conn.execute(_CACHE_COUNT, params).scalar() or 0 if in_enum else 0
|
||||
cache_total += cache_n
|
||||
if not in_enum and prop_n:
|
||||
deferred.append(archetype.value)
|
||||
if prop_n or cache_n:
|
||||
tag = "" if in_enum else " [cache deferred: enum value missing]"
|
||||
print(
|
||||
f" {prop_n:>6} property_overrides + {cache_n} cache "
|
||||
f"-> {archetype.value!r} <= {description!r}{tag}"
|
||||
)
|
||||
if args.apply:
|
||||
conn.execute(_PROPERTY_OVERRIDES_UPDATE, params)
|
||||
if in_enum:
|
||||
conn.execute(_CACHE_UPDATE, params)
|
||||
verb = "updated" if args.apply else "would update"
|
||||
print(
|
||||
f"\n{verb} {prop_total} property_overrides rows + {cache_total} "
|
||||
f"classifier-cache rows across {len(REMAP)} descriptions."
|
||||
)
|
||||
if deferred:
|
||||
print(
|
||||
f"\n{len(set(deferred))} archetype(s) NOT in the Drizzle "
|
||||
"`main_heating_system` pgEnum — their cache rows are deferred "
|
||||
"until the FE-repo enum migration lands (property_overrides was "
|
||||
"still updated, which is what the modelling reads):"
|
||||
)
|
||||
for v in sorted(set(deferred)):
|
||||
print(f" {v!r}")
|
||||
if not args.apply:
|
||||
print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -245,13 +245,14 @@ def test_off_peak_archetypes_drag_dual_others_drag_single() -> None:
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"main_heating_value",
|
||||
["Unknown", "", "Air source heat pump", "Community heating"],
|
||||
["Unknown", "", "Community heating"],
|
||||
)
|
||||
def test_unresolvable_or_unmodelled_heating_produces_no_overlay(
|
||||
main_heating_value: str,
|
||||
) -> None:
|
||||
# Heat pumps (main_heating_index_number) and community heating (community
|
||||
# codes) don't map to a Table 4b sap_main_heating_code yet — no overlay.
|
||||
# Genuinely unrecognised values (Unknown / empty) and not-yet-modelled
|
||||
# community heating produce no overlay — the lodged EPC heating stands
|
||||
# (ADR-0041). Air source heat pumps ARE now modelled (Table 4a 211).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for(main_heating_value, 0)
|
||||
|
|
@ -383,3 +384,238 @@ def test_every_resolvable_main_heating_value_decodes(
|
|||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
|
||||
|
||||
def test_solid_fuel_room_heater_decodes_to_the_closed_room_heater_code() -> None:
|
||||
# A landlord-named solid-fuel room heater (e.g. a closed stove) is a
|
||||
# recognised archetype, not a gas wet system — it must decode to its SAP
|
||||
# Table 4a code (633, closed solid-fuel room heater), not overflow into the
|
||||
# nearest wrong archetype the way the under-populated taxonomy did, sending
|
||||
# "solid fuel room heaters: closed room heater" to Gas CPSU (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Solid fuel room heater, closed", 0)
|
||||
|
||||
# Assert — SAP Table 4a code 633 (closed solid-fuel room heater).
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.sap_main_heating_code == 633
|
||||
|
||||
|
||||
def test_solid_fuel_room_heater_drags_its_coherent_room_heater_companions() -> None:
|
||||
# The landlord names the system, not its category/control/meter. A solid-fuel
|
||||
# room heater is a room-heater system (Table 4a category 10) on a single-rate
|
||||
# meter (not off-peak storage), under the conservative room-heater control
|
||||
# real certs lodge (Table 4e Group 6 code 2601 — "no thermostatic control",
|
||||
# the lowest-SAP room-heater control, so it never over-credits an unobserved
|
||||
# control, mirroring the storage manual-charge default). ADR-0035.
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Solid fuel room heater, closed", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.main_heating_category == 10
|
||||
assert heating.main_heating_control == 2601
|
||||
assert heating.meter_type == "Single"
|
||||
|
||||
|
||||
def test_electric_room_heaters_drag_their_natural_electricity_fuel() -> None:
|
||||
# A landlord names the system; an electric room heater's natural fuel is
|
||||
# unambiguously electricity (RdSAP main_fuel code 29). The overlay drags it so
|
||||
# a system-only override is self-coherent even on a cert that lodged a
|
||||
# different fuel — a later `main_fuel` override still wins (last-wins
|
||||
# composition), and a contradicting fuel is logged, not silently kept
|
||||
# (ADR-0041 natural-fuel coherence).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Electric room heaters", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.main_fuel_type == 29
|
||||
|
||||
|
||||
def test_solid_fuel_room_heater_defaults_to_house_coal_as_its_natural_fuel() -> None:
|
||||
# Solid fuel is fuel-ambiguous (coal / anthracite / smokeless / dual fuel /
|
||||
# wood logs / pellets), but the system must still self-cohere when no
|
||||
# main_fuel override is given. Default to house coal (RdSAP main_fuel code
|
||||
# 33), the most common solid fuel; a main_fuel override naming a specific
|
||||
# solid fuel still wins by last-wins composition (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Solid fuel room heater, closed", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.main_fuel_type == 33
|
||||
|
||||
|
||||
def test_air_source_heat_pump_decodes_to_the_default_ashp_code() -> None:
|
||||
# A landlord names "air source heat pump" without a PCDB model. It is
|
||||
# modellable via the SAP Table 4a default ASHP code (211, "flow temp in other
|
||||
# cases", default SPF 2.30) — no main_heating_index_number needed — so it must
|
||||
# decode to 211, not produce no overlay / fall to Unknown (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Air source heat pump", 0)
|
||||
|
||||
# Assert — SAP Table 4a code 211 (default air-source heat pump).
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.sap_main_heating_code == 211
|
||||
|
||||
|
||||
def test_air_source_heat_pump_drags_its_coherent_companions() -> None:
|
||||
# A heat pump is unambiguously electric (natural fuel 29) and SAP Table 4a
|
||||
# category 4. Unlike storage/CPSU it does NOT imply an off-peak tariff (SAP
|
||||
# §12 Rule 3 is conditional, not forcing), so the coherent default meter is
|
||||
# single-rate — forcing Dual would assume an off-peak split it may not have
|
||||
# and mis-bill it. The overlay drags these code-derived companions so a
|
||||
# system-only override is self-coherent (ADR-0035 / ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Air source heat pump", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.main_heating_category == 4
|
||||
assert heating.main_fuel_type == 29
|
||||
assert heating.meter_type == "Single"
|
||||
|
||||
|
||||
def test_community_boilers_decode_to_the_heat_network_boiler_code() -> None:
|
||||
# A community/heat-network scheme driven by boilers is SAP Table 4a code 301
|
||||
# (the calculator derives the heat-source efficiency + DLF from it). It must
|
||||
# decode to 301, not the Gas CPSU (120) the under-populated taxonomy forced —
|
||||
# a single-dwelling gas wet boiler is the wrong picture for a heat network
|
||||
# (ADR-0041). A generic "Community heating" with no named source stays None.
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Community heating, boilers", 0)
|
||||
|
||||
# Assert — SAP Table 4a code 301 (boiler-driven community heating).
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.sap_main_heating_code == 301
|
||||
|
||||
|
||||
def test_community_boilers_drag_heat_network_category_and_a_community_gas_fuel() -> None:
|
||||
# A community boiler scheme is SAP main_heating_category 6 (heat network), so
|
||||
# the calculator treats it as a heat network (cert_to_inputs `_is_heat_network`
|
||||
# checks code OR category 6). Its natural fuel is mains gas (community) — RdSAP
|
||||
# main_fuel code 20, the dominant fuel real community-boiler certs lodge. A
|
||||
# specific main_fuel override (e.g. biomass community) still wins (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Community heating, boilers", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.main_heating_category == 6
|
||||
assert heating.main_fuel_type == 20
|
||||
|
||||
|
||||
def test_community_chp_and_boilers_decode_to_their_heat_network_code() -> None:
|
||||
# A community scheme with CHP + boilers is SAP Table 4a code 302, still a heat
|
||||
# network (category 6) on community mains gas (20). Real audit data: 10
|
||||
# properties carry this, today mis-bucketed to Gas CPSU (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Community heating, CHP and boilers", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.sap_main_heating_code == 302
|
||||
assert heating.main_heating_category == 6
|
||||
assert heating.main_fuel_type == 20
|
||||
|
||||
|
||||
def test_oil_room_heater_decodes_to_the_post_2000_code() -> None:
|
||||
# A modern standalone oil room heater is SAP Table 4a code 623 ("Oil room
|
||||
# heater, 2000 or later", no boiler). It must decode to 623, not the Gas CPSU
|
||||
# the under-populated taxonomy forced (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Oil room heater, 2000 or later", 0)
|
||||
|
||||
# Assert — SAP Table 4a code 623.
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.sap_main_heating_code == 623
|
||||
|
||||
|
||||
def test_oil_room_heater_drags_its_coherent_room_heater_companions() -> None:
|
||||
# An oil room heater is a room-heater system (category 10) on a single-rate
|
||||
# meter, under the conservative room-heater control (2601), with its natural
|
||||
# fuel heating oil (RdSAP main_fuel 28). The overlay drags these so a
|
||||
# system-only override is self-coherent (ADR-0035 / ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Oil room heater, 2000 or later", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.main_heating_category == 10
|
||||
assert heating.main_heating_control == 2601
|
||||
assert heating.main_fuel_type == 28
|
||||
assert heating.meter_type == "Single"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "code"),
|
||||
[
|
||||
("Gas room heater, condensing fire", 611),
|
||||
("Gas room heater, decorative fuel-effect", 612),
|
||||
("Gas room heater, flush live-effect", 605),
|
||||
("Gas room heater, open flue 1980 or later", 603),
|
||||
("Gas room heater, open flue pre-1980", 601),
|
||||
],
|
||||
)
|
||||
def test_gas_room_heater_variants_decode_to_their_sap_codes(
|
||||
value: str, code: int
|
||||
) -> None:
|
||||
# Gas room heaters span a wide efficiency range (decorative 0.20 -> condensing
|
||||
# 0.85), so each catalogue variant maps to its own SAP Table 4a code rather
|
||||
# than collapsing to one representative — which would over-credit a decorative
|
||||
# fire and under-credit a condensing one. Today they mis-bucket to "Gas
|
||||
# boiler, regular" (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for(value, 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
assert simulation.heating.sap_main_heating_code == code
|
||||
|
||||
|
||||
def test_gas_room_heater_drags_its_coherent_companions() -> None:
|
||||
# A gas room heater is a room-heater system (category 10) on a single-rate
|
||||
# meter, under the conservative room-heater control (2601). Its natural fuel
|
||||
# is mains gas (26) — an LPG dwelling is refined by a main_fuel override
|
||||
# (the overlay can't see the mains connection) (ADR-0041).
|
||||
|
||||
# Act
|
||||
simulation = main_heating_overlay_for("Gas room heater, condensing fire", 0)
|
||||
|
||||
# Assert
|
||||
assert simulation is not None
|
||||
assert simulation.heating is not None
|
||||
heating = simulation.heating
|
||||
assert heating.main_heating_category == 10
|
||||
assert heating.main_heating_control == 2601
|
||||
assert heating.main_fuel_type == 26
|
||||
assert heating.meter_type == "Single"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Optional, TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -161,10 +161,11 @@ class _ScoringRebaseliner(Rebaseliner):
|
|||
self,
|
||||
property_id: int,
|
||||
effective_epc: EpcPropertyData,
|
||||
lodged: Performance,
|
||||
lodged: Optional[Performance],
|
||||
*,
|
||||
physical_state_changed: bool = False,
|
||||
) -> RebaselineResult:
|
||||
assert lodged is not None # this stub is only used on the real-cert path
|
||||
return RebaselineResult(
|
||||
effective=lodged, reason="none", sap_result=self._result
|
||||
)
|
||||
|
|
@ -244,6 +245,80 @@ def test_run_raises_when_no_rhi_and_no_scored_result() -> None:
|
|||
assert uow.commits == 0
|
||||
|
||||
|
||||
class _PredictedRebaseliner(Rebaseliner):
|
||||
"""Scores a predicted picture into a fixed Effective Performance, ignoring the
|
||||
(absent) lodged half — a predicted Property always has ``physical_state_changed``,
|
||||
so the calculator output IS the Effective Performance (ADR-0039)."""
|
||||
|
||||
def __init__(self, effective: Performance, result: SapResult) -> None:
|
||||
self._effective = effective
|
||||
self._result = result
|
||||
|
||||
def rebaseline(
|
||||
self,
|
||||
property_id: int,
|
||||
effective_epc: EpcPropertyData,
|
||||
lodged: Optional[Performance],
|
||||
*,
|
||||
physical_state_changed: bool = False,
|
||||
) -> RebaselineResult:
|
||||
return RebaselineResult(
|
||||
effective=self._effective,
|
||||
reason="physical_state_changed",
|
||||
sap_result=self._result,
|
||||
)
|
||||
|
||||
|
||||
def _predicted_property() -> Property:
|
||||
"""A predicted Property (ADR-0031): no lodged EPC, only a neighbour-synthesised
|
||||
predicted EPC whose recorded performance fields are a *different dwelling's* —
|
||||
the phantom Lodged Performance this fix must not propagate. Mirrors the #1361
|
||||
exemplar (uprn 34003067 / property 742072), predicted-only."""
|
||||
predicted = object.__new__(EpcPropertyData)
|
||||
predicted.energy_rating_current = 14 # phantom: copied from a neighbour template
|
||||
predicted.current_energy_efficiency_band = Epc.G
|
||||
predicted.co2_emissions_current = 5.0
|
||||
predicted.energy_consumption_current = 400
|
||||
predicted.sap_version = 10.2
|
||||
predicted.renewable_heat_incentive = None
|
||||
return Property(
|
||||
identity=PropertyIdentity(
|
||||
portfolio_id=1, postcode="A0 0AA", address="1 Some Street", uprn=34003067
|
||||
),
|
||||
epc=None,
|
||||
predicted_epc=predicted,
|
||||
)
|
||||
|
||||
|
||||
def test_run_persists_no_lodged_performance_for_a_predicted_property() -> None:
|
||||
# Arrange — a predicted Property (no lodged cert); the rebaseliner scores its
|
||||
# predicted picture into a known Effective Performance.
|
||||
property_baseline_repo = FakePropertyBaselineRepo()
|
||||
uow = FakeUnitOfWork(
|
||||
property=FakePropertyRepo({742072: _predicted_property()}),
|
||||
property_baseline=property_baseline_repo,
|
||||
)
|
||||
effective = Performance(
|
||||
sap_score=57, epc_band=Epc.D, co2_emissions=1.5, primary_energy_intensity=160
|
||||
)
|
||||
orchestrator = PropertyBaselineOrchestrator(
|
||||
unit_of_work=lambda: uow,
|
||||
rebaseliner=_PredictedRebaseliner(
|
||||
effective, _sap_result_with_heating(space_kwh=4200.0, water_kwh=1600.0)
|
||||
),
|
||||
fuel_rates=FuelRatesStaticFileRepository(),
|
||||
)
|
||||
|
||||
# Act
|
||||
orchestrator.run([742072])
|
||||
|
||||
# Assert — no lodged comparator was manufactured from the neighbour-synthesised
|
||||
# EPC's recorded fields; the Effective half is still persisted.
|
||||
(baseline, _) = property_baseline_repo.saved[0]
|
||||
assert baseline.lodged is None
|
||||
assert baseline.effective == effective
|
||||
|
||||
|
||||
def test_run_derives_and_persists_a_bill_when_the_rebaseliner_scores() -> None:
|
||||
# Arrange — a rebaseliner that hands back a SapResult with lighting energy,
|
||||
# so the orchestrator prices it into a Bill at the committed snapshot.
|
||||
|
|
|
|||
|
|
@ -6,12 +6,23 @@ that fold onto the lodged EPC — per component, partial, skipping unmapped rows
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from datatypes.epc.domain.epc_property_data import BuildingPartIdentifier
|
||||
from repositories.property.landlord_override_overlays import overlays_from
|
||||
from domain.modelling.scoring.overlay_applicator import apply_simulations
|
||||
from repositories.property.landlord_override_overlays import (
|
||||
flag_fuel_mismatch,
|
||||
overlays_from,
|
||||
)
|
||||
from repositories.property.property_overrides_reader import (
|
||||
ResolvedPropertyOverride,
|
||||
ResolvedPropertyOverrides,
|
||||
)
|
||||
from tests.domain.sap10_calculator.worksheet._elmhurst_worksheet_000490 import (
|
||||
build_epc,
|
||||
)
|
||||
|
||||
|
||||
def test_roof_type_row_produces_a_roof_overlay() -> None:
|
||||
|
|
@ -143,3 +154,74 @@ def test_unresolvable_rows_are_skipped() -> None:
|
|||
|
||||
# Assert
|
||||
assert overlays == []
|
||||
|
||||
|
||||
def test_main_fuel_override_wins_over_a_heating_archetype_natural_fuel() -> None:
|
||||
# A heating-system override drags a natural fuel (a solid-fuel room heater
|
||||
# defaults to house coal, RdSAP main_fuel 33), but an explicit main_fuel
|
||||
# override must win. apply_simulations is last-wins and the override rows
|
||||
# arrive in arbitrary order, so overlays_from must apply main_fuel AFTER
|
||||
# main_heating_system — otherwise the natural-fuel default clobbers the
|
||||
# explicit fuel (ADR-0041). The rows here are in the order that exposes it.
|
||||
baseline = build_epc()
|
||||
overrides = ResolvedPropertyOverrides(
|
||||
rows=(
|
||||
ResolvedPropertyOverride("main_fuel", 0, "smokeless coal"),
|
||||
ResolvedPropertyOverride(
|
||||
"main_heating_system", 0, "Solid fuel room heater, closed"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Act
|
||||
result = apply_simulations(baseline, overlays_from(overrides))
|
||||
|
||||
# Assert — the explicit fuel (smokeless coal 15) wins over the coal default.
|
||||
assert result.sap_heating.main_heating_details[0].main_fuel_type == 15
|
||||
|
||||
|
||||
def test_a_fuel_override_contradicting_the_heating_archetype_is_logged(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
# An electric room heater is unambiguously electricity; a "mains gas" main_fuel
|
||||
# override contradicts it (a different fuel family). The override still wins
|
||||
# (ADR-0041), but the implausibility is logged for later review — not raised.
|
||||
overrides = ResolvedPropertyOverrides(
|
||||
rows=(
|
||||
ResolvedPropertyOverride(
|
||||
"main_heating_system", 0, "Electric room heaters"
|
||||
),
|
||||
ResolvedPropertyOverride("main_fuel", 0, "mains gas"),
|
||||
)
|
||||
)
|
||||
|
||||
# Act
|
||||
with caplog.at_level(logging.WARNING):
|
||||
flag_fuel_mismatch(overrides)
|
||||
|
||||
# Assert
|
||||
assert any("fuel" in r.message.lower() for r in caplog.records)
|
||||
|
||||
|
||||
def test_a_same_family_fuel_refinement_is_not_logged(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
# A solid-fuel room heater's natural fuel (house coal 33) is a *default* for
|
||||
# the ambiguous solid family. A "smokeless coal" override refines it within
|
||||
# the same family — that is expected, not a contradiction, so it must NOT log.
|
||||
# Only a different fuel family (gas/electric on a solid heater) is flagged.
|
||||
overrides = ResolvedPropertyOverrides(
|
||||
rows=(
|
||||
ResolvedPropertyOverride(
|
||||
"main_heating_system", 0, "Solid fuel room heater, closed"
|
||||
),
|
||||
ResolvedPropertyOverride("main_fuel", 0, "smokeless coal"),
|
||||
)
|
||||
)
|
||||
|
||||
# Act
|
||||
with caplog.at_level(logging.WARNING):
|
||||
flag_fuel_mismatch(overrides)
|
||||
|
||||
# Assert
|
||||
assert caplog.records == []
|
||||
|
|
|
|||
|
|
@ -83,6 +83,38 @@ def test_resaving_baseline_for_a_property_replaces_rather_than_duplicating(
|
|||
assert loaded == rerun
|
||||
|
||||
|
||||
def _predicted_baseline() -> PropertyBaselinePerformance:
|
||||
"""A predicted Property's baseline: no Lodged Performance (no cert to read one
|
||||
off), only the Effective half established (ADR-0004 amendment, #1361)."""
|
||||
return PropertyBaselinePerformance(
|
||||
lodged=None,
|
||||
effective=Performance(
|
||||
sap_score=57, epc_band=Epc.D, co2_emissions=1.5, primary_energy_intensity=160
|
||||
),
|
||||
rebaseline_reason="physical_state_changed",
|
||||
space_heating_kwh=4200.0,
|
||||
water_heating_kwh=1600.0,
|
||||
)
|
||||
|
||||
|
||||
def test_predicted_baseline_round_trips_with_no_lodged_half(db_engine: Engine) -> None:
|
||||
# Arrange — a predicted Property has no Lodged Performance; the four lodged_*
|
||||
# columns persist as NULL and reconstruct as a None lodged half.
|
||||
baseline = _predicted_baseline()
|
||||
with Session(db_engine) as session:
|
||||
PropertyBaselinePostgresRepository(session).save(baseline, property_id=13)
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
loaded = PropertyBaselinePostgresRepository(session).get_for_property(13)
|
||||
|
||||
# Assert — the lodged half round-trips as None; the Effective half is intact.
|
||||
assert loaded == baseline
|
||||
assert loaded is not None
|
||||
assert loaded.lodged is None
|
||||
|
||||
|
||||
def test_get_for_property_returns_none_when_absent(db_engine: Engine) -> None:
|
||||
# Arrange / Act
|
||||
with Session(db_engine) as session:
|
||||
|
|
|
|||
120
tests/scripts/test_null_predicted_lodged_performance.py
Normal file
120
tests/scripts/test_null_predicted_lodged_performance.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""The one-time backfill nulls a predicted Property's phantom Lodged Performance,
|
||||
leaves a lodged Property's intact, and is idempotent (#1361 Class B)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Engine
|
||||
from sqlmodel import Session
|
||||
|
||||
from datatypes.epc.domain.epc import Epc
|
||||
from datatypes.epc.domain.epc_property_data import EpcPropertyData
|
||||
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
|
||||
from domain.property_baseline.performance import Performance
|
||||
from domain.property_baseline.property_baseline_performance import (
|
||||
PropertyBaselinePerformance,
|
||||
)
|
||||
from repositories.epc.epc_postgres_repository import EpcPostgresRepository
|
||||
from repositories.property_baseline.property_baseline_postgres_repository import (
|
||||
PropertyBaselinePostgresRepository,
|
||||
)
|
||||
from scripts.null_predicted_lodged_performance import backfill
|
||||
|
||||
_JSON_SAMPLES = Path(__file__).resolve().parents[2] / "backend/epc_api/json_samples"
|
||||
|
||||
_PHANTOM_LODGED = Performance(
|
||||
sap_score=14, epc_band=Epc.G, co2_emissions=5.0, primary_energy_intensity=400
|
||||
)
|
||||
_EFFECTIVE = Performance(
|
||||
sap_score=57, epc_band=Epc.D, co2_emissions=1.5, primary_energy_intensity=160
|
||||
)
|
||||
|
||||
|
||||
def _epc() -> EpcPropertyData:
|
||||
raw: dict[str, Any] = json.loads(
|
||||
(_JSON_SAMPLES / "RdSAP-Schema-21.0.0" / "epc.json").read_text()
|
||||
)
|
||||
return EpcPropertyDataMapper.from_api_response(raw)
|
||||
|
||||
|
||||
def _baseline(lodged: Performance) -> PropertyBaselinePerformance:
|
||||
return PropertyBaselinePerformance(
|
||||
lodged=lodged,
|
||||
effective=_EFFECTIVE,
|
||||
rebaseline_reason="physical_state_changed",
|
||||
space_heating_kwh=4200.0,
|
||||
water_heating_kwh=1600.0,
|
||||
)
|
||||
|
||||
|
||||
def _seed(db_engine: Engine) -> None:
|
||||
"""A predicted Property (id 1, predicted EPC only, phantom lodged) and a
|
||||
lodged Property (id 2, lodged EPC, real lodged) — both with a baseline row
|
||||
carrying a populated lodged half (the pre-fix state)."""
|
||||
epc = _epc()
|
||||
with Session(db_engine) as session:
|
||||
epc_repo = EpcPostgresRepository(session)
|
||||
epc_repo.save(epc, property_id=1, source="predicted")
|
||||
epc_repo.save(epc, property_id=2, source="lodged")
|
||||
baseline_repo = PropertyBaselinePostgresRepository(session)
|
||||
baseline_repo.save(_baseline(_PHANTOM_LODGED), property_id=1)
|
||||
baseline_repo.save(_baseline(_PHANTOM_LODGED), property_id=2)
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_apply_nulls_only_the_predicted_propertys_lodged_half(
|
||||
db_engine: Engine,
|
||||
) -> None:
|
||||
# Arrange
|
||||
_seed(db_engine)
|
||||
|
||||
# Act
|
||||
with db_engine.begin() as conn:
|
||||
found = backfill(conn, apply=True)
|
||||
|
||||
# Assert — one phantom found and nulled; the predicted Property loses its
|
||||
# lodged half but keeps its Effective half, while the lodged Property is
|
||||
# untouched.
|
||||
assert found == 1
|
||||
with Session(db_engine) as session:
|
||||
repo = PropertyBaselinePostgresRepository(session)
|
||||
predicted = repo.get_for_property(1)
|
||||
lodged = repo.get_for_property(2)
|
||||
assert predicted is not None
|
||||
assert predicted.lodged is None
|
||||
assert predicted.effective == _EFFECTIVE
|
||||
assert lodged is not None
|
||||
assert lodged.lodged == _PHANTOM_LODGED
|
||||
|
||||
|
||||
def test_dry_run_reports_the_phantom_but_writes_nothing(db_engine: Engine) -> None:
|
||||
# Arrange
|
||||
_seed(db_engine)
|
||||
|
||||
# Act
|
||||
with db_engine.begin() as conn:
|
||||
found = backfill(conn, apply=False)
|
||||
|
||||
# Assert — the phantom is counted, but the row is left untouched.
|
||||
assert found == 1
|
||||
with Session(db_engine) as session:
|
||||
predicted = PropertyBaselinePostgresRepository(session).get_for_property(1)
|
||||
assert predicted is not None
|
||||
assert predicted.lodged == _PHANTOM_LODGED
|
||||
|
||||
|
||||
def test_re_running_apply_is_a_no_op(db_engine: Engine) -> None:
|
||||
# Arrange — the first apply nulls the phantom.
|
||||
_seed(db_engine)
|
||||
with db_engine.begin() as conn:
|
||||
backfill(conn, apply=True)
|
||||
|
||||
# Act — a second apply finds nothing left to null.
|
||||
with db_engine.begin() as conn:
|
||||
found = backfill(conn, apply=True)
|
||||
|
||||
# Assert
|
||||
assert found == 0
|
||||
30
tests/scripts/test_reclassify_heating_overrides.py
Normal file
30
tests/scripts/test_reclassify_heating_overrides.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""The one-time heating-override re-classification map is internally valid."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from domain.epc.property_overlays.main_heating_system_overlay import (
|
||||
main_heating_overlay_for,
|
||||
)
|
||||
from domain.epc.property_overrides.main_heating_system_type import (
|
||||
MainHeatingSystemType,
|
||||
)
|
||||
from scripts.reclassify_heating_overrides import REMAP
|
||||
|
||||
|
||||
def test_every_remap_target_is_a_resolvable_archetype() -> None:
|
||||
# The remap must never point a stored description at an archetype the overlay
|
||||
# can't model — that would replace one broken value with another. Every
|
||||
# target must decode to a real SAP heating code (ADR-0041).
|
||||
for description, archetype in REMAP.items():
|
||||
simulation = main_heating_overlay_for(archetype.value, 0)
|
||||
|
||||
assert simulation is not None, description
|
||||
assert simulation.heating is not None, description
|
||||
assert simulation.heating.sap_main_heating_code is not None, description
|
||||
|
||||
|
||||
def test_remap_targets_are_main_heating_system_members() -> None:
|
||||
# Belt and braces: the values are enum members (a typo can't smuggle a
|
||||
# non-archetype string into the DB).
|
||||
for archetype in REMAP.values():
|
||||
assert archetype in MainHeatingSystemType
|
||||
Loading…
Add table
Reference in a new issue