Merge remote-tracking branch 'origin/main' into feature/historic-epc-repository

.claude/settings.json

@@ -1,168 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(python -c ' *)",
-      "Bash(python -m pytest tests/ -v --no-cov)",
-      "Bash(git -C /workspaces/model diff --stat)",
-      "Bash(python -c \"import moto; print\\('moto installed:', moto.__version__\\)\")",
-      "Bash(grep -E \"\\\\.\\(py|sql\\)$\")",
-      "Bash(xargs basename -a)",
-      "Bash(ls -la /workspaces/home 2>/dev/null | head -20)",
-      "Read(//workspaces/home/**)",
-      "Bash(command -v uv)",
-      "Bash(uv --version)",
-      "Bash(echo \"uv: $\\(uv --version\\)\")",
-      "Bash(python -m pyright --version)",
-      "Bash(npx --no-install pyright --version)",
-      "Bash(node /home/vscode/.npm/_npx/110e52990071af13/node_modules/pyright/dist/pyright.js --outputjson etl/hubspot/hubspotClient.py etl/hubspot/hubspotDataTodB.py etl/hubspot/project_data.py etl/hubspot/scripts/scraper/main.py backend/app/db/models/hubspot_project_data.py backend/app/db/models/hubspot_deal_data.py etl/hubspot/tests/test_scraper_handler.py etl/hubspot/tests/test_hubspot_data_to_db.py etl/hubspot/tests/test_hubspot_client_integration.py)",
-      "Bash(python -c \"import sys,json; d=json.load\\(sys.stdin\\); s=d['summary']; print\\('errors',s['errorCount'],'warnings',s['warningCount']\\); [print\\(f\\\\\"{x['severity']}: {x['file'].split\\('/'\\)[-1]}:{x['range']['start']['line']+1} {x['rule'] if 'rule' in x else ''} -- {x['message'].splitlines\\(\\)[0]}\\\\\"\\) for x in d['generalDiagnostics'] if x['severity']=='error']\")",
-      "Bash(node /home/vscode/.npm/_npx/110e52990071af13/node_modules/pyright/dist/pyright.js etl/hubspot/project_data.py backend/app/db/models/hubspot_project_data.py)",
-      "Bash(python -c \"import sqlmodel, sqlalchemy, hubspot; print\\('deps importable'\\)\")",
-      "Bash(python -m pytest --version)",
-      "Bash(python -m pytest etl/hubspot/tests/ -m \"not integration\" -p no:cacheprovider -o addopts=\"\" -q)",
-      "Bash(node /home/vscode/.npm/_npx/110e52990071af13/node_modules/pyright/dist/pyright.js --pythonpath /usr/local/bin/python etl/hubspot/project_data.py backend/app/db/models/hubspot_project_data.py)",
-      "Bash(node /home/vscode/.npm/_npx/110e52990071af13/node_modules/pyright/dist/pyright.js --pythonpath /usr/local/bin/python etl/hubspot/company_data.py)",
-      "Bash(python /tmp/inspect_project.py)",
-      "Read(//home/vscode/github/**)",
-      "Bash(find / -maxdepth 6 -type d -name assessment-model)",
-      "Bash(terraform fmt *)",
-      "Bash(terraform init *)",
-      "Bash(terraform validate *)",
-      "Bash(python -m pytest tests/orchestration/test_landlord_description_overrides_orchestrator.py -q)",
-      "Bash(python -m pytest tests/orchestration/test_landlord_description_overrides_orchestrator.py tests/repositories/landlord_overrides -q --no-cov)",
-      "Bash(python -m py_compile tests/infrastructure/chatgpt/test_chatgpt_column_classifier.py)",
-      "Bash(git add *)",
-      "Bash(git status *)",
-      "Bash(cp pyrightconfig.json /tmp/pyrightconfig.bak.json)",
-      "Bash(python3 -)",
-      "Bash(npx --yes pyright infrastructure/postgres/landlord_overrides_postgres_repository.py applications/landlord_description_overrides/handler.py tests/repositories/landlord_overrides/postgres/test_landlord_overrides_postgres_repository.py orchestration/classifiable_column.py)",
-      "Bash(git stash *)",
-      "Bash(npx --yes pyright applications/landlord_description_overrides/handler.py)",
-      "Bash(python3 -m pytest tests/repositories/landlord_overrides/postgres/test_landlord_overrides_postgres_repository.py -q)",
-      "Bash(python3 -m pytest tests/orchestration/test_landlord_description_overrides_orchestrator.py tests/repositories/landlord_overrides/ -q -p no:cov)",
-      "Bash(python3 -m pytest tests/orchestration/test_landlord_description_overrides_orchestrator.py tests/repositories/landlord_overrides/ -q)",
-      "Bash(cp pyrightconfig.json /tmp/pyrightconfig.bak2.json)",
-      "Bash(npx --yes pyright infrastructure/landlord_overrides/landlord_overrides_postgres_repository.py tests/repositories/landlord_overrides/postgres/test_landlord_overrides_postgres_repository.py)",
-      "Bash(GIT_EDITOR=true git rebase --continue)",
-      "Bash(git worktree *)",
-      "Bash(git branch *)",
-      "Bash(echo \"exit: $?\")",
-      "Bash(git reset *)",
-      "Bash(echo \"fetch-exit: $?\")",
-      "Bash(sed -n *)",
-      "Bash(set -e)",
-      "Bash(git rm *)",
-      "Bash(git ls-tree *)",
-      "Bash(command -v pyright)",
-      "Bash(git merge *)",
-      "Bash(git rev-list *)",
-      "Bash(git remote *)",
-      "Bash(git ls-remote *)",
-      "Bash(grep -v '\\\\.sample$')",
-      "Bash([ -f \".git/hooks/$h\" ])",
-      "Bash(python3 -m pytest tests/test_lambda_packaging.py -p no:cacheprovider --no-header -q)",
-      "Bash(python3 -m pytest tests/test_lambda_packaging.py -p no:cov --no-header -q)",
-      "Bash(python3 -m pytest tests/test_lambda_packaging.py --no-cov -p no:cacheprovider -q)",
-      "Bash(python3 -m pytest tests/test_lambda_packaging.py --no-cov -q)",
-      "Bash(python3 -m pyright tests/test_lambda_packaging.py)",
-      "Bash(python3 -m pytest tests/ --collect-only --no-cov -q)",
-      "Bash(python3 -c \"import yaml; [print\\(f, 'OK'\\) for f in ['.github/workflows/ddd_tests.yml','.github/workflows/unit_tests.yml'] if yaml.safe_load\\(open\\(f\\)\\)]\")",
-      "Bash(python3 -m pytest tests/ -q --no-cov)",
-      "Bash(python3 -c ' *)",
-      "Bash(python3 -m pyright --outputjson scripts/hyde/main.py)",
-      "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); s=d['summary']; print\\('errors:',s['errorCount'],'warnings:',s['warningCount']\\); [print\\(f\\\\\"  L{e['range']['start']['line']+1}: {e['message'].splitlines\\(\\)[0]}\\\\\"\\) for e in d['generalDiagnostics'] if e['severity']=='error']\")",
-      "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); s=d['summary']; print\\('pyright errors:',s['errorCount'],'warnings:',s['warningCount']\\); [print\\(f\\\\\"  L{e['range']['start']['line']+1}: {e['message'].splitlines\\(\\)[0]}\\\\\"\\) for e in d['generalDiagnostics'] if e['severity']=='error']\")",
-      "Bash(python3 main.py)",
-      "Bash(netstat -ltnp)",
-      "Bash(fuser 8000/tcp)",
-      "Bash(kill 351610 685390 351625)",
-      "Read(//proc/531213/net/**)",
-      "Bash(ps -p 351610 -o pid=)",
-      "Bash(python -)",
-      "Bash(python -m pytest \"datatypes/epc/domain/tests/test_from_rdsap_schema.py::TestRdSap20_0_0ReducedFieldSynthesis::test_cert_omitting_sap_windows_maps_without_missing_required_field\")",
-      "Bash(python -m pytest datatypes/epc/domain/tests/test_from_rdsap_schema.py infrastructure/epc_client/tests/test_mapper_corpus.py)",
-      "Bash(git -C /workspaces/model stash push -u -- datatypes/epc/schema/rdsap_schema_20_0_0.py datatypes/epc/domain/mapper.py datatypes/epc/domain/tests/test_from_rdsap_schema.py)",
-      "Bash(python -m pytest \"datatypes/epc/domain/tests/test_from_rdsap_schema.py::TestFromRdSapSchema21_0_1::test_total_floor_area\")",
-      "Bash(git -C /workspaces/model stash pop)",
-      "Bash(python -m pytest \"datatypes/epc/domain/tests/test_from_rdsap_schema.py::TestRdSap20_0_0ReducedFieldSynthesis::test_band_more_than_typical_scales_glazing_by_1_25\")",
-      "Bash(python -m pytest \"datatypes/epc/domain/tests/test_from_rdsap_schema.py::TestRdSap20_0_0ReducedFieldSynthesis\")",
-      "Bash(python -m pytest \"datatypes/epc/domain/tests/test_from_rdsap_schema.py::TestRdSap20_0_0ReducedFieldSynthesis::test_synthesised_glazing_type_routed_through_cascade\")",
-      "Bash(python -m pytest infrastructure/epc_client/tests/test_mapper_corpus.py -k \"wip_schema_20\" -q -p no:cov --no-header -rN)",
-      "Bash(python -m pytest infrastructure/epc_client/tests/test_mapper_corpus.py -k \"wip_schema_20\" -q --no-header)",
-      "Bash(timeout 150 python scripts/eon/find_epc_data.py)",
-      "Bash(python -m scripts.run_modelling_e2e --scenario-id 1266 --portfolio-id 785 --measures high_heat_retention_storage_heaters,solar_pv 709634 709635 709636 709638 709639 709640 709641 709642 709644)",
-      "Read(//home/vscode/.claude/skills/grill-me/**)",
-      "Bash(awk '{print $9, $5}')",
-      "Bash(grep -E \"\\\\.py$\")",
-      "Bash(python3 scripts/eon/harvest_certs.py)",
-      "Bash(python scripts/eon/harvest_certs.py)",
-      "Bash(git commit -q -m ' *)",
-      "Bash(timeout 250 npx -y pyright@1.1.410 datatypes/epc/domain/mapper.py)",
-      "Bash(git check-ignore *)",
-      "Bash(timeout 250 npx -y pyright@1.1.410 datatypes/epc/schema/rdsap_schema_18_0.py datatypes/epc/domain/mapper.py)",
-      "Bash(timeout 250 npx -y pyright@1.1.410 datatypes/epc/schema/rdsap_schema_17_1.py datatypes/epc/domain/mapper.py)",
-      "Bash(python -m scripts.run_modelling_e2e --scenario-id 1266 --portfolio-id 785 --measures high_heat_retention_storage_heaters,solar_pv 709634 709635 709636 709638 709639 709640 709641 709642 709643 709644 709645 709637)",
-      "Read(//workspaces/**)",
-      "Bash(python -m scripts.run_modelling_e2e --scenario-id 1266 --portfolio-id 785 --measures high_heat_retention_storage_heaters,solar_pv 709634)",
-      "Bash(python 2_export_data.py)",
-      "Bash(git commit -q -m 'Map RdSAP-Schema-17.0 certs to EpcPropertyData 🟥 *)",
-      "Bash(python -m py_compile tests/utilities/floats.py backend/app/db/functions/tests/test_portfolio_functions.py backend/documents_parser/tests/test_summary_pdf_mapper_chain.py backend/documents_parser/tests/test_heating_systems_corpus.py)",
-      "Bash(PYTHONPATH=/workspaces/model python -)",
-      "Bash(npx --no-install pyright datatypes/epc/schema/sap_schema_17_1.py datatypes/epc/domain/mapper.py datatypes/epc/domain/tests/test_from_sap_schema.py)",
-      "Bash(git commit -q -m 'Map full-SAP cert identity and scalar fields to EpcPropertyData 🟩 *)",
-      "Bash(npx --no-install pyright --outputjson datatypes/epc/domain/mapper.py)",
-      "Bash(git commit -q -m 'Carry full-SAP measured fabric U-value descriptions into the domain model 🟩 *)",
-      "Bash(python scripts/hyde/elmhurst_session.py --help)",
-      "Bash(python scripts/hyde/elmhurst_session.py status)",
-      "Bash(DISPLAY=:99 import -window root /tmp/hyde-viewer/elm_now2.png)",
-      "Bash(DISPLAY=:99 timeout 90 python scripts/hyde/elmhurst_explore.py \"https://rdsap10online.elmhurstenergy.co.uk/Processing/WebFormAddress.aspx?Guid=B44A0DB4-4C08-4241-B818-86F060172105&Referrer=https%3a%2f%2fmembers.elmhurstenergy.co.uk%2fRdsapZone%2fHome.aspx\")",
-      "Bash(DISPLAY=:99 timeout 90 python scripts/hyde/elmhurst_explore.py \"https://rdsap10online.elmhurstenergy.co.uk/Processing/WebFormPropertyDescription.aspx?Guid=B44A0DB4-4C08-4241-B818-86F060172105\")",
-      "Bash(DISPLAY=:99 timeout 90 python scripts/hyde/elmhurst_explore.py \"https://rdsap10online.elmhurstenergy.co.uk/Processing/WebFormAddress.aspx?Guid=B44A0DB4-4C08-4241-B818-86F060172105\")",
-      "Bash(DISPLAY=:99 timeout 150 python scripts/hyde/elmhurst_fill.py)",
-      "Bash(pkill -f \"rdsap10\\\\|for-testing\")",
-      "Bash(rm -f scripts/hyde/.elmhurst-session/Singleton*)",
-      "Bash(DISPLAY=:99 timeout 280 python -u scripts/hyde/elmhurst_fill.py --page property_description --commit)",
-      "Bash(npx --no-install pyright scripts/hyde/elmhurst_download.py)",
-      "Edit(/.claude/skills/expand-sap-accuracy-corpus/**)",
-      "Bash(PYTHONPATH=/workspaces/model python scripts/fetch_real_life_epc_sample.py 10093116528)",
-      "Bash(PYTHONPATH=/workspaces/model python -c ' *)",
-      "Bash(bash scripts/hyde/start_viewer.sh)",
-      "Bash(DISPLAY=:99 ELMHURST_URL=\"https://rdsap10online.elmhurstenergy.co.uk/Processing/WebFormAddress.aspx?Guid=B44A0DB4-4C08-4241-B818-86F060172105\" python scripts/hyde/elmhurst_session.py login)",
-      "Bash(break)",
-      "Bash(DISPLAY=:99 timeout 90 python /tmp/check_session.py)",
-      "Bash(pkill -9 -f \"elm_login_hold.py\")",
-      "Bash(pkill -9 -f \"elmhurst-session\")",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 150 python /tmp/elm_build_dims.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_dump_full.py Walls WebFormWalls.aspx)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_walls_disc.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_roof_disc.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_dump_full.py Openings WebFormOpenings.aspx)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_win_probe.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 220 python /tmp/elm_build_openings.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_tab_probe.py)",
-      "Bash(ELM_ACCESS=\"P960-0001\" ELM_PWD=\"y22bseGUzr\" DISPLAY=:99 timeout 120 python /tmp/elm_del_probe.py)",
-      "Bash(npx --no-install pyright datatypes/epc/domain/epc_property_data.py datatypes/epc/domain/mapper.py domain/sap10_calculator/rdsap/cert_to_inputs.py)",
-      "Bash(PYTHONPATH=/workspaces/model python -m pytest domain/sap10_calculator/tests/test_sap_accuracy_corpus.py -q --no-cov)",
-      "Bash(PYTHONPATH=/workspaces/model python -m pytest tests/domain/sap10_calculator/test_real_cert_sap_accuracy.py -q --no-cov)",
-      "Edit(/.claude/skills/epc-to-elmhurst-rdsap-inputs/**)",
-      "Bash(python3 -c \"import sys; sys.path.insert\\(0,'scripts/hyde'\\); import elmhurst_lib; print\\('elmhurst_lib imports OK:', [f for f in dir\\(elmhurst_lib\\) if not f.startswith\\('_'\\)][:12]\\)\")",
-      "Bash(PYTHONPATH=/workspaces/model python -m pytest tests/domain/sap10_calculator/test_real_cert_sap_accuracy.py tests/infrastructure/epc_client/test_sap_accuracy_corpus.py -q --no-cov)",
-      "Bash(PYTHONPATH=/workspaces/model python scripts/fetch_real_life_epc_sample.py 10093116543)",
-      "Bash(echo \"---EXIT $?---\")",
-      "Bash(sudo apt-get update -o Dir::Etc::sourceparts=-)",
-      "Bash(apt-cache policy *)",
-      "Bash(sudo apt-get update)",
-      "Bash(bash scripts/hyde/start_viewer.sh restart)",
-      "Bash(echo \"=== listen sockets \\(port 6080 / 5900\\) ===\"; ss -ltnp 2>/dev/null | grep -E ':6080|:5900' || netstat -ltnp 2>/dev/null | grep -E ':6080|:5900' || echo \"ss/netstat unavailable\" *)",
-      "Read(//proc/5476/**)",
-      "Bash(sudo python -m playwright install-deps chromium)",
-      "Bash(python -m playwright install chromium)",
-      "Bash(python -m playwright --version)",
-      "Bash(pip show *)"
-    ],
-    "additionalDirectories": [
-      "/tmp",
-      "/workspaces/model/scripts/hyde/.elmhurst-session"
-    ]
-  }
-}

.claude/skills/audit-ara-portfolio/SKILL.md

@@ -0,0 +1,122 @@
+---
+name: audit-ara-portfolio
+description: Audit a modelled Ara portfolio for odd results — run the deterministic anomaly checks over the DB, review the groups, deep-dive a sample of each, characterise sub-classes, and cross-reference open PRs/ADRs. Use when the user wants to audit or review a portfolio's modelling output, hunt for dodgy plans / baselines / SAP scores / recommendations, or triage modelling anomalies. Asks for portfolio id and scenario id.
+---
+
+# Audit Ara portfolio
+
+Turn the deterministic anomaly scan into a triaged, root-caused review. The
+deterministic checks (`scripts/audit/anomalies.py`) find KNOWN anti-patterns
+exhaustively and cheaply; this skill adds the judgement — confirm, characterise,
+cross-reference existing work, and feed novel patterns back as new checks.
+
+## Input
+
+Ask for **portfolio_id** and **scenario_id** if the user didn't give them.
+`scenario_id` is optional — without it, each Property's *default* plan (the one
+shown in the FE) is audited.
+
+## Phase 1 — Build the dataset
+
+```
+python -m scripts.audit.anomalies --portfolio <portfolio_id> --scenario <scenario_id>
+```
+
+Writes `modelling_audit.md` (grouped, ranked by severity) and
+`modelling_audit.csv` (every flagged row: property_id, uprn, severity, check,
+detail). Read the printed summary and `modelling_audit.md`.
+
+## Phase 2 — Review the high-level results
+
+For each check group, HIGH severity first: note the count, read a few example
+rows. State, in one line each, a **root-cause hypothesis** and whether it is a
+real bug, a known/expected effect, or a threshold that needs tuning.
+
+## Phase 3 — Deep-dive a sample per group
+
+For 2–3 representative properties in each meaningful group, reproduce
+end-to-end (NO DB writes — never pass `--persist`):
+
+```
+python -m scripts.run_modelling_e2e <property_id> --scenario-id <scenario_id>
+```
+
+Read the printed overrides + effective SAP + measure-by-measure plan table.
+Confirm the anomaly and isolate **where** it goes wrong: the baseline, a specific
+measure, the SAP calc, or the bill. For SAP-value bugs, the `SapResult`
+worksheet / `intermediate` trail and the `/diagnose` loop are the tools.
+
+## Phase 4 — Characterise sub-classes
+
+A group of thousands is usually several distinct causes. Within each meaningful
+group, cluster the flagged properties by a distinguishing trait — EPC source
+(lodged / predicted), `rebaseline_reason`, property type, the dominant measure,
+fuel — using SQL against the DB. Report the sub-classes and their sizes.
+
+## Phase 5 — Cross-reference open work
+
+Before proposing fixes, check whether one is already in flight:
+
+```
+gh pr list --repo Hestia-Homes/Model --state open
+gh pr list --repo Hestia-Homes/Model --state all --search "<keyword>"
+```
+
+Map each sub-class to an existing PR / ADR where one applies (e.g.
+baseline-vs-plan divergence → the override-aware-rebaseline + persistence-fidelity
+work; oversized solar → ADR-0038 dwelling-roof cap). Flag sub-classes with no
+coverage as new work.
+
+## Output
+
+A triage report. Per check group:
+- count + root-cause hypothesis,
+- sub-classes, each sized,
+- example property ids (to reproduce),
+- existing-PR/ADR coverage,
+- recommended action: **fix** / **tune threshold** / **accept (expected)** / **new ticket**.
+
+Then run Phase 6 to make the audit permanently better.
+
+## Phase 6 — Self-improve (the compounding loop)
+
+When the review confirms a **novel, systematic** problem, codify it so every
+future run catches it automatically. This is what makes the audit get better each
+time it runs. Apply the gates below — they keep the registry sharp, not noisy.
+
+**Gates (all must hold before adding a check):**
+1. **Systematic** — reproduced on **≥ 5** properties and root-caused, not a
+   one-off. (A single weird property is a ticket, not a check.)
+2. **Not already covered** — no existing check fires on it, and no open/merged PR
+   or ADR already addresses the cause (you checked in Phase 5).
+3. **Pressure-tested** — for any non-trivial check (a threshold, a heuristic),
+   run `/grill-me` on the proposed check first: what's the false-positive rate on
+   this portfolio? is the threshold defensible against the real distribution? does
+   it overlap an existing check? Tune from the answers before committing.
+
+**What to change, smallest first:**
+- **A check** — add one decorated `(PropertyAudit) -> Optional[str]` function to
+  `scripts/audit/anomalies.py`. Its docstring MUST record **provenance**: the
+  motivating property ids and the one-line root cause, so the check is traceable
+  and re-verifiable later. If it needs a field not on `PropertyAudit`, extend the
+  bundle + query.
+- **The skill** — if the review revealed a new *expectation* (a pattern that is
+  expected-not-a-bug, or a new deep-dive technique), add it to this file's Notes
+  / phases so the next reviewer starts ahead.
+- **Docs** — if the cause is a load-bearing modelling decision, an ADR may be
+  warranted (rare; only when hard-to-reverse + surprising + a real trade-off).
+
+Commit each codified check on its own with the motivating run referenced, then
+**re-run Phase 1** to confirm the new check fires on the cases that motivated it
+and nothing else surprising. The check registry — with provenance — is the
+durable, compounding output of every audit.
+
+## Notes
+
+- Read-only on the DB. `run_modelling_e2e` is a dry run.
+- **Expected, not bugs** (until the override-aware-rebaseline + persistence-fidelity
+  PR deploys and the portfolio is re-modelled): much of `zero-works-post-differs`
+  and `plan-score-below-baseline` is the pre-fix baseline-vs-plan divergence and
+  should shrink after re-model — note it, don't re-debug it.
+- Adding a check is one decorated `(PropertyAudit) -> Optional[str]` function in
+  `scripts/audit/anomalies.py`; see its module docstring.

.claude/skills/epc-to-elmhurst-rdsap-inputs/reference/mapping.md

@@ -180,12 +180,23 @@ Table 32 unit costs, p/kWh (`domain/sap10_calculator/tables/table_32.py`):
 **`main_fuel_type` / `water_heating_fuel` 29 = off-peak (7-hour) electricity** →
 Elmhurst Electricity meter type = **Dual-rate / Economy 7 (7-hour)**, NOT Single.
 
-⚠️ **Known over-rating bug:** the engine prices **100% of off-peak space heating
-AND hot water at the 5.50p low rate** (`inputs.space_heating_fuel_cost_gbp_per_kwh`
-= 0.055), instead of the SAP **Table 12a high/low split** (a portion at the 15.29p
-high rate). This under-costs all-electric Economy-7 dwellings and inflates the SAP
-score. Always surface this in the output's "Known divergences". Canonical case:
-UPRN 10002468137 — lodged 55, engine 62.
+✅ **Economy-7 high/low split — FIXED (PR #1217).** The engine now applies the SAP
+**Table 12a Grid 1** (space) + **Table 13** (immersion DHW) high/low split rather
+than pricing 100% at the 5.50p low rate. Electric STORAGE heaters legitimately get
+a 0.00 SH high-rate fraction (100% low — spec value, not a bug); immersion HW takes
+the cylinder-volume/occupancy/single-dual Table 13 blend (applied in
+`cert_to_inputs._hot_water_fuel_cost_gbp_per_kwh` when volume + occupancy +
+single/dual are all resolved; absent any of them it still falls back to 100% low —
+a rarer edge). Verified: canonical UPRN 10002468137 engine 60.92 = Elmhurst 61 to
+the penny (its lodged 55 is the OLD SAP-2012 schema, not comparable); UPRN
+10022893721 engine 79 = lodged 79, Elmhurst (Dual meter) 81.
+
+⚠️ **When building in Elmhurst you MUST set the Economy-7 meter** (`main_fuel_type`
+/ `water_heating_fuel` 29 = off-peak 7-hour → Electricity meter type **Dual**, NOT
+Single). Elmhurst silently defaults to Single/Standard and prices at the 13.19p
+standard rate, collapsing the worksheet SAP ~13 points — which can masquerade as an
+engine "over-rating". The control is a hidden Meters sub-tab on the SpaceHeating
+page (`TabPanelMeters_RadioButtonListElectricityType`).
 
 ## Water Heating
 

.claude/skills/expand-sap-accuracy-corpus/SKILL.md

@@ -19,6 +19,74 @@ score the property would get unchanged. Elmhurst is the accredited ground truth;
 its Input Summary (parsed back to `EpcPropertyData`) exposes mapper holes, and its
 worksheet exposes calculator holes.
 
+## ⏩ Resume in a fresh context (autonomous run)
+
+If the user says "continue" / "next" / "keep going through the worklist", run this
+loop **continuously without asking between certs** — report only `eng X / elm Y`
+per cert and tick the worklist line as each is pinned. The build automation is
+fully working (see `scripts/hyde/elmhurst_lib.py` helpers + the `build_<uprn>.py`
+templates); most certs build unattended end-to-end.
+
+**State (2026-06-19):** pinned this campaign — SAP-17.1 cohort + RdSAP-17.1/18.0
+(older) plus: **16.1** `100021943298` (76/75), **19.1.0** `10096028301` (82/82),
+**16.3** `44012843` (79/78), **17.0** `10023444324` (80/80) + `10023444320`
+(81/82), **RdSAP-20.0.0** `10090844932` (78/77), **16.2** `100090182288` (69/71,
+semi house). Latest run (2026-06-19): **16.2** `100021985993` (74/72, end-terrace
+bungalow), **17.1** `10091568921` (82/80, full-SAP end-terrace house combi 17615)
++ `10093718424` (81/80, semi sibling), **RdSAP-18.0** `10022893721` (79/81, first
+NON-BOILER cert — electric storage heaters + immersion; storage-heater automation
+now SOLVED incl. the Economy-7 Dual-meter step, see banked findings; engine 79 =
+lodged, NO bug), **RdSAP-21.0.1** `10023443426` (76/79, native schema, combi house;
+engine 76 = lodged EXACTLY; Elmhurst +3 = omitted-secondary build gap). Next `[ ]`:
+`10093412452` (SAP-17.1), then `10090343335` (17.0) / `10093115480` (17.1) /
+`68151071` (RdSAP-17.0). Skip `100020933699` (user said skip), `[⛔]` (NOT
+MAPPABLE), `[⚠]` (flagged engine bugs: MVHR / heat-pump fuel-39).
+
+**Per-UPRN recipe** (all commands `DISPLAY=:99`, cwd `scripts/hyde`; run
+`bash scripts/hyde/start_viewer.sh` once; creds in `.elmhurst-creds.json`; shared
+assessment GUID `B44A0DB4-4C08-4241-B818-86F060172105`):
+1. `PYTHONPATH=/workspaces/model python scripts/fetch_real_life_epc_sample.py <uprn>`
+   then scope: dwelling_type/built_form, age band, walls/roof/floor descriptions,
+   heating `main_heating_index_number`/category/`has_hot_water_cylinder`, window
+   total area (`sum(sap_windows w*h)`), party_wall_length, lighting %, MEV/AP50.
+2. **Copy the closest `build_<uprn>.py` template** and adjust values:
+   - combi flat → `build_100021943298.py`; regular-boiler+cylinder flat →
+     `build_44012843.py`; full-SAP combi flat (MEV+AP50) → `build_10096028301.py`
+     / `build_10023444324.py` (+party wall) / `build_10023444320.py` (mid-floor);
+   - combi house → `build_10090844932.py` (end-terrace, party wall) /
+     `build_100090182288.py` (semi, no party wall).
+   Adjust: property type/built-form, band (`_pick` by year, e.g. "1950"/"2012"/
+   "2023"), two-floor dims + party wall, wall insulation, roof, floor, window m²,
+   doors, lighting, boiler PCDB ref + search query, MEV/AP50 if present.
+3. Run pages: `for p in property_description [flats] dimensions walls roofs floors
+   openings ventilation; do … build_<uprn>.py $p; done` (one Save&Close each,
+   ~1 min/page; flats only for Flat property type). Then a window-verify/fix
+   snippet (re-add the combined window if the grid shows 0.00), then
+   `build_<uprn>.py space_heating` and `water_heating`.
+4. Heating uses the `elmhurst_lib` helpers: `E.select_boiler(page, "<model>",
+   "<pcdb_id>")` (look up id/type in `domain/elmhurst/pcdb_gas_oil_boiler_codes.csv`;
+   the lodged `main_heating_index_number` IS the id); control
+   `E.set_heating_dialog(page, "…ButtonMainHeatingControls", "^Boilers",
+   "^Standard", "CBE Programmer, room thermostat and TRVs")` (=2106); water
+   `E.set_heating_dialog(page, "…ButtonWaterHeatingCode", "From Space Heating",
+   "From the primary heating system")`; combi → `E.clear_hot_water_cylinder(page)`.
+5. Download: edit `elmhurst_download.py` `SAMPLE_DIR` to the cert's
+   `<schema>/uprn_<uprn>` dir; first confirm Recommendations is clean (parse
+   `[id*=ContentPlaceHolder1] a` link text — Summary silently redirects to Address
+   until zero errors); `python scripts/hyde/elmhurst_download.py` (retry once; the
+   nav goes Address→Recommendations→Summary).
+6. `PYTHONPATH=/workspaces/model python scripts/compare_epc_paths.py <uprn>` →
+   read **"gov-API inputs → SAP"** (engine) and **"Elmhurst's OWN engine
+   (worksheet …)"** (Elmhurst ground truth). Target ≤0.5–1.
+7. **Pin** the engine value: add a `RealCertExpectation(schema, sample=uprn_<uprn>,
+   cert_num, sap_score=<engine>)` in `tests/.../test_real_cert_sap_accuracy.py`;
+   run `…::test_real_cert_sap_score`.
+8. Tick the worklist line `[x] … · eng X / elm Y (lodged Z) · PINNED …`. Next cert.
+
+See **Banked findings** below for the modal-dialog mechanics (all already encoded
+in the helpers). New schema not mappable → add a dedicated `from_*_schema_*`
+mapper first (per-schema convention) + guard with the RdSAP-21.0.1 corpus gauge.
+
 ## The loop (one UPRN)
 
 1. **Pick** the first `[ ]` UPRN in [worklist.md](worklist.md).
@@ -83,6 +151,14 @@ Pattern: `with E.session() as (ctx,page): E.goto(...); E.set_text/set_select(...
   2022" ≈ U1.4/g0.72) — the "...known data" option demands per-row Frame-Factor/
   Data-Source/Location/Building-Part and fails validation. Trust the grid only
   after Save & Close + reload (the live grid mis-shows older rows as 0.00).
+- **Window-width `.fill()` can DOUBLE the value** → "Openings: Width has to be less
+  or equal to 50" validation error. `_add_window`'s `page.locator(TextBoxExtWidth)
+  .fill(str(area))` went through Elmhurst's keystroke/numeric formatter and rendered
+  `11.1` as `111.10` (uprn_10010215568) — the Recommendations gate then blocks the
+  download. After building Openings, VERIFY the grid width (reload + read the row);
+  if wrong, JS-set it: `el.value='11.1'; dispatch input+change+blur`, then Save&Close
+  (the entry boxes mirror the single row = edit-in-place). JS-set bypasses the
+  formatter and sticks. Belt-and-braces: set the width by JS in `_add_window` too.
 - **U-value-known overrides** (walls/roof/floor) don't toggle via automation →
   accept the natural RdSAP age-band U-value; note the divergence for reconciliation.
 - **Recommendations page = validation gate.** It must show ZERO errors before the
@@ -93,6 +169,112 @@ Pattern: `with E.session() as (ctx,page): E.goto(...); E.set_text/set_select(...
   lines 17/18). This drove the first campaign mapper fix — see Banked findings.
 
 ## Banked findings (fold new ones in here as the corpus grows)
+- **MAPPER GAP — cylinder insulation thickness dropped (RdSAP-17.0+):** the mapper
+  carries `cylinder_size` + `cylinder_insulation_type` but NOT
+  `cylinder_insulation_thickness` → `EpcPropertyData.sap_heating.cylinder_
+  insulation_thickness_mm` stays None even when the cert lodges it (e.g. 50 mm).
+  The engine then assumes a poorly-insulated cylinder → over-counts HW cylinder
+  loss → under-rates. Confirmed uprn_68151071 (raw 50 mm → mapped None; engine HW
+  3446 vs Elmhurst 2911 kWh; engine 68 vs lodged 70 / Elmhurst 71). FIX: map the
+  thickness in the RdSAP per-schema mapper; check blast radius (any pinned cylinder
+  cert may shift) + regress the RdSAP-21.0.1 corpus gauge. Leverage point — likely
+  improves every cylinder-with-lodged-thickness cert. Flagged, not yet fixed.
+- **Party-wall type `_pick` gotcha:** matching `"filled"` ALSO matches "Cavity
+  masonry **UN**filled" (CU, U≈0.5) — the wrong type for a cert whose party wall is
+  U≈0. Match `"masonry filled"` to hit CF (Cavity masonry filled, U≈0). Affects
+  cavity builds (10090844932 / 10091568921 / 10093718424 / 10093412452 used the
+  loose `"filled"` and may have got CU in Elmhurst — only the Elmhurst cross-check,
+  not the pinned engine value which is validated against lodged). Fixed for
+  uprn_10093115480.
+- **Shared-assessment reset: storage/electric → boiler cert.** The shared
+  assessment carries the PRIOR cert's heating system. Going storage→combi, the
+  boiler search dialog won't open while a SAP-table `MainHeatingCode` (e.g. SEB) is
+  set. Fix: JS-clear `MH1.TextBoxMainHeatingCode` to `''` + dispatch change, Save &
+  Close, then `select_boiler` works. Also RESET the electricity meter on the Meters
+  sub-tab (a prior off-peak cert leaves it Dual; gas certs want Single) and the
+  SECONDARY heating (a prior cert's secondary persists even when the calc shows
+  presence=No — set it explicitly or it pollutes the worksheet).
+- **Secondary heating must be built in Elmhurst when lodged.** Certs lodge a
+  secondary (`sap_heating.secondary_heating_type`, e.g. 612 = mains-gas room heater
+  @ Table 4a seasonal efficiency 0.20 — a low-eff decorative/old gas fire). The
+  engine models it (fraction 0.1 from Table 11 ÷ the secondary efficiency → e.g.
+  3065 kWh for code 612); omitting it in Elmhurst inflates the worksheet (uprn_
+  10023443426: omitted → 79 vs engine/lodged 76). Build via Secondary present=Yes →
+  `ButtonSecondaryHeatingCode` cascade (title "Select secondary heating"): fuel →
+  sub-fuel → appliance → type, e.g. Gas → Mains gas → Room Heaters → RGx (pick the
+  RGx whose efficiency matches the lodged code, ~0.20 = decorative/old gas fire).
+  ⚠ For a NATIVE RdSAP-21.0.1 cert, engine = lodged (exact, all components) is the
+  authoritative validation — if the Elmhurst rebuild diverges, suspect an omitted
+  lodged feature (secondary / meter), confirm via engine-on-Elmhurst-inputs ≈
+  worksheet, and pin the engine = lodged value.
+  ⚠ **`present=No` does NOT clear the shared assessment's leftover secondary** — the
+  prior cert's secondary SYSTEM (fuel + SAP code) persists server-side even when the
+  UI dropdown reads "No", and it silently re-enters the worksheet. On a storage-heater
+  cert (uprn_100020665611, RdSAP-20.0.0 end-terrace house) a leftover House-Coal
+  closed-room-heater (SAP 633, cheap solid fuel) inflated the Elmhurst worksheet to
+  44 vs engine 36 / lodged 37 until OVERWRITTEN. Always set the secondary EXPLICITLY
+  to the lodged appliance: storage-heater certs lodge "Portable electric heaters
+  (assumed)" → present=Yes → `ButtonSecondaryHeatingCode` cascade Electric → Electric
+  → Room Heaters → "REA Panel, convector or radiant heaters" (= SAP 691, eff 100%).
+  That dropped the worksheet 44 → 35 ≡ engine-on-Elmhurst-inputs 35 (faithful).
+- **Openings: standard double/triple-glazing bands REQUIRE Frame Type + Glazing Gap.**
+  The window grid's `DropDownListExtFrameType` (PVC/Wood/Metal) and
+  `DropDownListExtGlazingGap` (6 mm / 12 mm / 16 mm or more) are required for any
+  band other than the new-build "Double post or during 2022" default — leaving them
+  empty fails the Recommendations gate ("Openings: Frame Type / Glazing Gap must be
+  entered"). Set both in `_add_window` BEFORE clicking Add (cert `pvc_window_frames`
+  → PVC; `glazing_gap` mm → the band). Glazing bands available: Single / Double|Triple
+  {pre 2002, between 2002 and 2021, post or during 2022, with unknown install date} /
+  Secondary / *…known data*. For RdSAP double-glazing with no lodged install year
+  (engine U 2.8) pick "Double pre 2002" (~U3.1, sub-1-SAP diff) — the "…known data"
+  options demand per-row U/g values.
+- **Non-boiler (storage-heater) main heating — SOLVED** (uprn_10022893721, RdSAP-
+  18.0 GF flat, electric storage heaters + immersion). The SpaceHeating page has NO
+  inline system-type selector and a `ButtonMainHeatingCode` button only APPEARS
+  once the bound PCDF boiler is cleared. Two-pass recipe (one Save & Close each):
+  1. **Clear the leftover PCDB boiler**: set `MH1.TextBoxPCDFBoilerReference` to
+     `"0"` via JS + dispatch `change`, then `save_close`. (It doesn't AutoPostBack;
+     the Save commits it. After reload, boilerRef="0", the boiler fuel/flue fields
+     vanish, and `MH1.ButtonMainHeatingCode` is now present.)
+  2. **Select the SAP-table system** via `set_heating_dialog(..ButtonMainHeating
+     Code, "^Electric","^Electric","Storage","SEB Modern slimline")` (title "Select
+     heating code"; L1 Gas/Oil/Solid Fuel/Electric/Community/No heating; storage L4
+     = SEA old large-volume / SEB modern slimline / SED fan / SEJ integrated / SEK
+     high-heat-retention). **Match the cert's `sap_main_heating_code`**: 402 = SEB.
+     Then the CONTROL via `set_heating_dialog(..ButtonMainHeatingControls,
+     "Storage Radiator","CSA Manual charge control")` (= SAP 2401). Secondary No.
+  Water: `set_heating_dialog(..ButtonWaterHeatingCode, "Water Heater","^Electric",
+  "Immersion")` (→ code HEI) — then the immersion code REQUIRES a cylinder: CHECK
+  `WH.CheckBoxHotWaterCylinder` (JS click+change, AutoPostBacks) or Recommendations
+  errors "must have a Hot Water Cylinder"; set `WH.DropDownListCylinderSize`
+  (size 2→Normal/110L, 3→Medium/160L, 4→Large/210L), `WH.DropDownListInsulated`
+  (Foam/Jacket), `WH.DropDownListInsulationThickness`, and `WH.RadioButton
+  ListImmersionHeater` (off-peak meter → Dual). See `build_10022893721.py` as the
+  template. ⚠ **CRITICAL — set the electricity meter type.** All-electric off-peak
+  certs MUST set the Economy-7 meter or Elmhurst silently defaults to Single /
+  Standard and prices everything at the 13.19p standard rate (worksheet collapses
+  ~13 SAP). The control is a HIDDEN sub-tab on the SpaceHeating page:
+  `E.click_tab(page,"TabContainer_TabPanelMeters")` then `E.set_select(page,
+  "TabContainer_TabPanelMeters_RadioButtonListElectricityType","Dual")` (options
+  Single/Dual/18 Hour/24 Hour/Unknown; cert `meter_type` 1→Dual=7-hour off-peak).
+  Verify in Elmhurst summary §14.2 "Electricity meter type" / worksheet
+  "Electricity Tariff" before trusting the worksheet SAP.
+- **Economy-7 off-peak pricing is CORRECT — do NOT "fix" it** (was a real bug, FIXED
+  in PR #1217 via the Table 13 off-peak water-heating split + window-U fix). The
+  engine applies SAP Table 12a Grid 1 (space) + Table 13 (immersion DHW) high/low
+  splits properly: storage heaters' SH high-rate fraction is legitimately 0.00
+  (100% low rate), immersion HW takes the volume/occupancy/single-dual blend. Proof:
+  uprn_10002468137 (canonical) engine 60.92 = Elmhurst 61 to the penny;
+  uprn_10022893721 engine 79 = lodged 79, Elmhurst (Dual meter) 81. ⚠ If you see a
+  big engine-over-Elmhurst gap on an all-electric off-peak cert, SUSPECT THE BUILD
+  (Elmhurst meter left on Single — see meter step above), not the calculator. The
+  `reference/mapping.md` "known over-rating bug (engine 62)" note is STALE (pre-PR
+  #1217). `cert_to_inputs.py` `_hot_water_fuel_cost_gbp_per_kwh` applies the immersion
+  Table 13 blend when cylinder volume + occupancy resolve; when `immersion_heating_
+  type` is UNLODGED on an off-peak meter it now defaults to DUAL per RdSAP §10.5
+  (was a 100%-low-rate fallback that under-costs — fixed, +regression test
+  `test_off_peak_immersion_unlodged_type_defaults_to_dual_table13_blend`). Only an
+  unresolvable cylinder volume / occupancy still falls back to 100% low.
 - **Air-permeability AP50 fix** (uprn_10093116528, the first campaign cert): the
   full-SAP mapper was routing the lodged q50 to the engine's AP4/Pulse formula
   (`0.263×AP4^0.924`) instead of the AP50 `/20` path — a big infiltration
@@ -105,13 +287,78 @@ Pattern: `with E.session() as (ctx,page): E.goto(...); E.set_text/set_select(...
   default. Pin the engine's observed value; document the Elmhurst delta (don't tune).
 - **FGHRS** (full-SAP `has_fghrs`/`fghrs_index_number`) is dropped by the mapper
   and not yet modelled — omit it on BOTH sides so the comparison stays clean.
+- **Validation errors live on the Recommendations page as LINKS** (red ✗ anchors),
+  not coloured spans — parse `[id*=ContentPlaceHolder1] a` text, not CSS colour.
+  Until Recommendations shows ZERO errors the **Energy Report Summary nav silently
+  redirects to the Address page** and the worksheet/Results PDFs won't generate
+  (Results renders empty). Two errors that bit the 16.1/19.1.0 builds: *"Walls
+  (Main): Insulation Thickness must be entered"* (set the insulation-thickness
+  dropdown — "Unknown" is fine for a reduced cert) and *"Incorrect Controls (NNNN
+  ctrl-group-X) for Heating System (idx ctrl-group-Y)"* (the heating control's
+  system-type must match the boiler's — see the controls dialog below).
+- **All Elmhurst modal dialogs sit above a `modalBackground` that defeats element
+  clicks** (Playwright sees it "intercepting pointer events", even with force).
+  The cracked pattern is now in `elmhurst_lib.py`: open via JS click, set cascade
+  `<select>`s by JS (`.value` + dispatch `change` → drives the AutoPostBack), and
+  COMMIT by `page.mouse.click(x,y)` at the OK/Select control's centre (coordinate
+  clicks hit the topmost element = the dialog). Helpers: `E.select_boiler(page,
+  query, ref)`, `E.set_heating_dialog(page, button_suffix, *level_regexes)` (water
+  method + main-heating controls), `E.dialog_commit(page, label)`,
+  `E.clear_hot_water_cylinder(page)`.
+- **Boiler by exact PCDB code** (eliminates the boiler-efficiency divergence):
+  `E.select_boiler(page, "<brand/model search>", "<pcdb_id>")` drives the search
+  dialog two-step (click the result ROW to highlight → click the top **Select**
+  span). The lodged `main_heating_index_number` IS the `pcdb_id`. Look up the Ref/
+  type/efficiency in **`domain/elmhurst/pcdb_gas_oil_boiler_codes.csv`**. NB:
+  typing the number straight into the *PCDF boiler Reference* box is COSMETIC — it
+  doesn't re-resolve the boiler; only the search dialog sets type/efficiency. A
+  combi keeps the cylinder checked from a prior regular boiler → `clear_hot_water_
+  cylinder()`, then set water method "From Space Heating → primary" via
+  `set_heating_dialog(..ButtonWaterHeatingCode, 'From Space Heating', 'From the
+  primary heating system')`.
+- **Reduced-field (16.x) semi/terraced party wall.** 16.x lodges no
+  `party_wall_length`, so the engine models NO party wall (defaults 0). But
+  Elmhurst REJECTS a semi/terraced with a zero party wall ("At least one building
+  part must have a party wall length of non zero"). Enter the geometry-derived
+  length (solve `2w+d=heat_loss_perimeter`, `w*d=floor_area`; party wall = the
+  joined side `d`) on every floor + a filled-cavity party-wall type (CF), and
+  EXPECT engine to over-read Elmhurst by ~1–2 SAP (engine's no-party-wall vs
+  Elmhurst's forced one). That delta is the documented reduced-field gap, not a
+  bug — confirm via engine-on-Elmhurst-inputs ≈ worksheet. Pin the engine value.
+- **Main-heating control must match the boiler's system type.** Heat-pump leftover
+  control (e.g. CHJ/2210, ctrl-group-2) is invalid for a gas boiler (ctrl-group-1)
+  and blocks the report. For a boiler programmer+room-stat+TRVs (SAP 2106):
+  `set_heating_dialog(..ButtonMainHeatingControls, '^Boilers', '^Standard', 'CBE
+  Programmer, room thermostat and TRVs')`.
+  ⚠ **Zone-control codes are under L2 "Zone control", NOT "Standard".** SAP 2110
+  "Time and temperature zone control" = `set_heating_dialog(.., '^Boilers', 'Zone
+  control', 'CBI Time and temperature zone control')` (Boilers→Standard L3 only has
+  CBA…CBH — no zone option, so a wrong L2 leaves the cascade incomplete and the OK
+  click is swallowed by the modalBackground, hanging the run). CBL is the PCDF
+  variant. Match the lodged `main_heating_control`: 2110 → CBI, 2106 → CBE (Standard).
+- **Removing an inherited secondary when the cert lodges NONE** — `present=No` ALONE
+  does NOT remove it: a persisted secondary CODE (e.g. an electric REA/691 from a
+  prior cert) survives the dropdown=No and the worksheet still applies it at Table-11
+  fraction 0.1 (worth ~1.5 SAP on a gas-combi cert — the electric secondary is priced
+  at the 13.19p peak rate vs gas 3.48p). The flag and the code are independent; the
+  worksheet keys off the CODE. ✅ FIX: `present=Yes` (exposes `TextBoxSecondaryHeating
+  Code`) → JS-clear that textbox to `''` + dispatch change → `present=No` → Save&Close.
+  An EMPTY code drops the worksheet's "Secondary Heating" to None / fraction 0.0.
+  Verify by re-downloading: worksheet line (201) "Fraction of space heat from
+  secondary" must read 0.0000. Seen on uprn_10090944225 (worksheet 79→80; closed the
+  cross-check gap to the residual +1 floor-U difference).
 
 ## Limitations / next improvements (make the campaign scale)
 - **Per-assessment GUID** — `elmhurst_lib.py` reuses one `ASSESSMENT_GUID`
-  (Khalim-test), overwritten per UPRN. For parallelism, create one assessment per
-  UPRN and parameterize the GUID.
-- **Data-driven fill** — `elmhurst_lib.py` gives the primitives; a per-cert driver
-  that reads field values straight from `elmhurst_inputs.md` (or a JSON sidecar)
-  would remove the remaining hand-keying. `elmhurst_fill.py` is the older template.
-- **More dialogs** — boiler cascade is automatable; the control-code (2110) and
-  FGHRS/MV-unit search dialogs still need the open→search→select→OK pattern.
+  (Khalim-test), overwritten per UPRN. ⚠ This is a SHARED assessment: a concurrent
+  user (Khalim) overwrote a build mid-session once — coordinate before long runs,
+  or (better) create one assessment per UPRN and parameterize the GUID.
+- **Data-driven fill** — `build_<uprn>.py` per-cert scripts now drive every page
+  incl. all dialogs via the `elmhurst_lib` helpers (see build_100021943298.py /
+  build_10096028301.py as templates). A generic driver reading `elmhurst_inputs.md`
+  is the remaining step. `elmhurst_fill.py` is the older single-cert template.
+- **Per-schema mappers** — every SAP/RdSAP schema gets its own dedicated
+  `from_*_schema_*` mapper in `datatypes/epc/domain/mapper.py` (the 16.x family
+  shares `_normalize_sap_schema_16_x`; full-SAP revisions delegate to
+  `from_sap_schema_17_1`). Add new schema coverage as a dedicated method + dispatch
+  branch, never by tuple-stuffing. Guard with the RdSAP-21.0.1 corpus gauge.

.claude/skills/expand-sap-accuracy-corpus/worklist.md

@@ -4,12 +4,260 @@ Process **one at a time**, top to bottom (see [SKILL.md](SKILL.md)). After each
 UPRN, tick it and annotate: `— <schema> · eng <X> / elm <Y> · <note>`.
 
 **Legend:** `[ ]` todo · `[x]` pinned (≤0.5) · `🔧` mapper extended · `⚠` xfail
-(engine bug) · `⛔` blocked.
+(engine bug) · `⛔` blocked · `[🔍]` flagged — engine vs Elmhurst-worksheet gap too
+large, or a new/complex build pattern needed; NEEDS INVESTIGATION.
 
 **Worked reference (not in the 100):** `uprn_10092973954` (SAP-Schema-17.1,
 2020 new-build flat) — full loop proven: eng 77 / elm 78, engine-on-Elmhurst-
 inputs 79 (calculator faithful within ~1). Use it to sanity-check the pipeline.
 
+# Files saving instructions
+I've noticed we arn't saving the epc.json (from epc api), input summary and worksheets from elmhurst. We shuold be doing that so we can reproduce behaviours quicker and verify. This is important as other memebers of team can help improve the verification. Files should be saved on backend/epc_api/json_samples/real_life_examples/<schema>/uprn_<uprn>/<files>
+
+
+## Do next — new schemas (need a mapper)
+
+These two were flagged NOT MAPPABLE in the UI (red ✗). Mapper coverage now ADDED
+(dedicated per-schema methods in `EpcPropertyDataMapper`; corpus gauge green, 0
+new pyright errors). Elmhurst build + pin still pending — run the normal loop.
+
+### 🔧🔍 FOR KHALIM — 100061905751 (cert 2628-3046-6233-4584-2970, SAP-16.3) — mapper fix landed, Elmhurst +5 to review
+Surfaced as a PRODUCTION crash (NOT in The 100): `modelling_e2e` property 730259
+`predict_epc` → `EpcComparablePropertiesRepository.candidates_for(postcode)` maps
+EVERY cohort cert in one list comprehension, and THIS cert raised
+`ValueError: RdSapSchema17_1: missing required field 'insulated_door_count'`,
+aborting the whole prediction. 🔧 FIXED generically in `_normalize_sap_schema_16_x`
+(`mapper.py`): `setdefault("insulated_door_count", 0)` — door refinement, absent →
+"no insulated doors recorded" → 0 (conservative RdSAP). +regression test
+`test_16_x_missing_insulated_door_count_defaults_to_zero` (test_from_sap_schema.py);
+0 new pyright errors; real-cert accuracy + 16.x suites green.
+Cert: END-TERRACE HOUSE 2-storey, band A pre-1900, cavity wall AS-BUILT **no
+insulation** (U0.70), pitched 100mm loft, suspended uninsulated floor, mains-gas
+COMBI (PCDB 16366 Glow-worm Ultracom 2 35cxi), control 2106 CBE, double glazed,
+**2 OPEN FIREPLACES** (chimneys), lodged **secondary** open-fire smokeless fuel
+(SAP 631 = Elmhurst RKJ), 3.84m² band-B extension, natural vent, 17% LED, TFA 139.
+**eng 57 = lodged 57 EXACTLY** / elm worksheet 52 (+5). Built in Elmhurst end-to-end
+(boiler 16366, CBE/2106, 2 open chimneys, secondary RKJ open-fire-in-grate, window
+20.57m², party wall geometry GF 6.31/1F 5.14 CF). engine-on-Elmhurst-inputs 53 ≈
+worksheet 52 → **calculator faithful**. The +5 decomposes as: ~2 SAP = the documented
+16.x reduced-field **party-wall gap** (lodges no party_wall_length → engine models
+NONE; Elmhurst forces `Party walls Main 31.02m² × U0.250 = 7.76 W/K`), SCALED UP by
+2-storey geometry (party wall area ~31m² vs ~16m² on the single-storey 16.2 pins
+100021985993/100090182288 which showed only +2); remainder ~2-3 = reduced-field
+build choices (3.84m² extension folded into GF, wall thickness 250 vs lodged 300,
+draughtproofing default). build_100061905751.py complete. **NOT yet pinned** — to
+review with Khalim (his call per CONTEXT.md): pin engine=lodged 57 like other 16.x,
+and/or whether 16.x end-terrace should model a party wall to close the reduced-field
+gap. Sample saved under SAP-Schema-16.3/uprn_100061905751 (epc.json + both PDFs).
+
+### 🔧🔍 modelling_e2e production failure sweep (2026-06-23) — cohort resilience + mapper gaps
+Investigated the 45 FAILED `modelling_e2e` subtasks (DevAssessmentModelDB, portfolio
+796) — 36 property rows, ~25 unique UPRNs. Root cause for most: `predict_epc` →
+`EpcComparablePropertiesRepository.candidates_for(postcode)` mapped EVERY cohort cert
+in one comprehension, so ONE unmappable cohort cert aborted the whole prediction.
+
+**✅ DONE — cohort resilience (skip + report)** so a single bad cert can't sink a
+prediction, and the skipped certs surface for follow-up:
+- `candidates_for` now skips certs that raise `ValueError` (mapper failures:
+  missing-field + `UnmappedApiCode`/`UnmappedElmhurstLabel`, both `ValueError`),
+  recording `SkippedCohortCert(certificate_number, error)` on `repo.skipped` + a
+  `logger.warning`. Transient API errors (not `ValueError`) still propagate.
+- handler surfaces them in the subtask **outputs** (success → `outputs.result.
+  skipped_unmappable_cohort_certs`; failure → appended to the error message), with
+  cert numbers, so the gaps can be closed deliberately. +3 tests; 0 new pyright.
+  Live-verified on BN11 4EP: cohort now builds 35 + records the 1 skip.
+
+**✅ DONE — generic mapper fixes (+regression tests, 0 new pyright):**
+- `_normalize_sap_schema_16_x`: `setdefault("insulated_door_count", 0)` (the
+  original prod crash).
+- ⚠️ REVERTED `setdefault("multiple_glazed_proportion", 100)` — it made an
+  otherwise-unmappable 16.3 cert (0418-3986-7250-2884-7970) join the EPC-prediction
+  **frozen-fixture donor pool** and tipped near-tie similarity matches, regressing
+  the component-accuracy gate (`has_hot_water_cylinder` 30→29/36; `door_count`
+  residual 23→25/36 — both ≫ the gate's 1e-3 float tolerance). The field is ML-only
+  (SAP calc never reads it) and the cohort skip-and-report path handles the cert,
+  so we leave it fail-loud instead of synthesising a value. Re-baselining the gate
+  was rejected (loosens a tighten-only gate for a real regression). For Khalim: if
+  16.x certs missing `multiple_glazed_proportion` should map, derive it (single→0 /
+  double→100) rather than a flat default, and re-measure the gate.
+- `_with_recorded_performance` co2/consumption/rating: `float(co2)` → crashed on
+  certs lodging `co2_emissions_current` as a Measurement dict `{'value':3.5,
+  'quantity':'tonnes per year'}` (16.x cert 2308-4997-7262-0137-9930, surfaced as
+  `TypeError: float() argument must be ... not 'dict'`). Now coerced via
+  `_measurement_value` (handles Measurement/dict/number). Verified maps → 3.5.
+
+**⚖️ FOR KHALIM — uncovered mapper gaps (mapper is your domain per CONTEXT.md):**
+- **`built_form` missing (SAP-16.0)** — cert 8742-6624-9300-2780-4926 (uprn
+  10070004512); has `property_type` but no `built_form`. Defaulting it drives
+  party-wall/exposed-perimeter modelling → a real modelling decision, not a neutral
+  default. Blocks props 710339, 723589.
+- **`window` missing + genuinely sparse (SAP-16.2)** — cert 8257-7539-1649-0633-4992
+  (uprn 10070009758); `windows` is a DICT not list, AND it omits `door_count`,
+  `habitable_room_count`, `glazed_area` — fail-loud is likely correct (data
+  insufficient). Blocks props 733315, 730259.
+- **🔍 RdSAP-21.0.0 systematically lacks the ADR-0028 Optional widenings 21.0.1 got.**
+  Cert 3135-3223-5500-0919-2206 (uprn 100021919725) omits 13 top-level fields
+  (`wet_rooms_count`, `open_chimneys_count`, `mechanical_ventilation_index_number`,
+  `led/cfl_fixed_lighting_bulbs_count`, the `mechanical_vent_duct_*`,
+  `insulated_door_u_value`, `pressure_test_certificate_number`,
+  `windows_transmission_details`) + `SapWindow` (`pvc_frame`/`glazing_gap`/
+  `frame_factor`/`window_transmission_details`) — ALL required in 21.0.0 but
+  `Optional[...] = None` in 21.0.1. NOT applied: widening risks pushing `None` into
+  `from_rdsap_schema_21_0_0` which may assume non-None (needs mapper-coalescing
+  review). Recommend aligning RdSapSchema21_0_0 to 21.0.1 (incl. the same Optionals
+  on insulated_door_count etc.) + reviewing the 21.0.0 mapper. Blocks prop 719897.
+  (The resilience change already stops all of these from crashing prod.)
+
+Caveat: ~13 other failed props showed no mapper gap on replay — genuinely
+not-predictable (empty/insufficient cohort) or already fixed by the
+`insulated_door_count` default; cohort replay only checked the first 60 certs/postcode
+and skipped rate-limited fetches, so a deeper gap could be missed.
+
+**Also done:** `deployment/terraform/lambda/modelling_e2e/variables.tf`
+`maximum_concurrency` default 2 → 4 (per request). Cohort skip-guard widened to
+`(ValueError, AttributeError, KeyError, TypeError)` (mapper-shape errors) so a
+malformed cert (e.g. PV-as-list AttributeError) is skipped+reported too, not just
+missing-field ValueErrors; transient `EpcApiError` (subclasses `Exception`) still
+propagates. +regression test.
+
+### 🔧🔍 modelling_e2e failure sweep — later run (2026-06-23 14:48 BST, portfolio 796, scenario 1268)
+A second `modelling_e2e` run (filter `task_source='modelling_e2e' AND job_started >
+'2026-06-23 14:48:01.806 +0100'`): **200 tasks, 30 FAILED (15%) → 22 unique props**
+(8 retried once). The DB only stores bare `property_id` in subtask `outputs`
+(`cloud_logs_url` empty) — root causes recovered by replaying each through
+`scripts.run_modelling_e2e --scenario-id 1268`. Four root-cause clusters:
+
+**① ✅ FIXED — `KeyError: BuildingPartIdentifier.EXTENSION_1` (14 props, the dominant
+failure).** A **Landlord Override** targeted `extension_1` but the lodged/predicted EPC
+had no such part; `overlay_applicator.apply_simulations` indexed `parts_by_id[identifier]`
+unguarded → crashed the whole property. Root cause is a **numbering mismatch**: the
+override's `building_part` is a positional index (0=main, 1=extension 1…, ADR-0004), but
+the gov-API EPC can label that slot differently (720142's 2nd part lodged as `other`, no
+`extension_1`). Fix (per product decision — *honour the override, don't drop it*):
+`_resolve_part` falls back to the EPC's part **at that position** when the semantic label
+is absent, so the landlord's correction lands; only a position the EPC models no part at
+is skipped (no geometry to model a wholly-absent part). +regression tests
+`test_override_for_an_absent_semantic_part_lands_on_the_part_at_that_position` /
+`test_override_with_no_part_at_that_position_is_skipped`. Not a mapper gap — a
+modelling-overlay bug. The 14 (pid / uprn):
+710295/100020458237, 710482/100020450179, 713040/100020404702, 715575/100020397529,
+715894/100020604961, 717435/22010468, 720142/100020383544, 720560/100021921443,
+721241/100020453651, 724945/100021915421, 725415/100020404036, 726517/100020631307,
+726592/100021918195, 730800/100021920273. (Observed post-fix eng: 720142 69.2,
+710295 63.2→69.8, 720560 61.8→70.4 predicted.)
+
+**② 🔍 mapper/cascade gap — `UnmappedSapCode: fuel_code 10`** (1 prop, 730259 / uprn
+100061905741). EPC-less → prediction synthesises a cohort EPC carrying `fuel_code 10`,
+absent from the calculator's cascade dispatch dict → FATAL (it's the target prop's own
+fuel, not a skippable cohort cert). FOR KHALIM: add fuel-code-10 to the cascade.
+
+**③ ⛔ not-predictable — empty/unresolved postcode cohort** (3 props): 714585/100020612517
+(`CR0 0DD`), 718580/10013149015 (`BR6 6BS`), 723881/22005280 (`BN41 2TP`). EPC-less AND
+cohort empty after filtering → Prediction Path 3 can't fire. Coverage gap, not a crash bug.
+
+**④ ⚠ transient — pass clean on replay** (4 props): 712401/100020394694 (eng 75.3),
+718138/100020397707 (eng 63.5→69.2), 720844/100020472603 (eng 71.8), 723648/100020480302
+(eng 70.8). Likely a flaky EPC/Solar API call during the batch — no code defect.
+
+**E2E candidates to pin (best = the lodged-EPC ones, clean to build in Elmhurst):**
+`[ ]` 100020383544 (pid 720142, eng 69.2) · `[ ]` 100020458237 (710295) · `[ ]`
+100020404702 (713040) · `[ ]` 100020450179 (710482) · `[ ]` 100021915421 (724945) ·
+`[ ]` 100021920273 (730800) · `[ ]` 100020394694 (712401, eng 75.3) · `[ ]`
+100020397707 (718138) · `[ ]` 100020472603 (720844, eng 71.8) · `[ ]` 100020480302
+(723648, eng 70.8). (Schema not populated on the gov-API objects — confirm when keying
+each into Elmhurst.)
+
+### 📋 PLAN — close the 8 modelling_e2e mapping gaps (2026-06-23 run, portfolio 796)
+The 8 failed prediction targets reduce to **5 distinct mapper-gap classes** (the fix
+targets). Per class: fix the mapper GENERICALLY, guard with BOTH the RdSAP-21.0.1
+corpus gauge AND the **component-accuracy gate** (`test_component_accuracy_gate.py`
+— every newly-mappable cert joins the prediction donor pool and can tip near-tie
+similarity, as `multiple_glazed_proportion=100` did: regressed has_hot_water_cylinder
+30→29/36 + door_count 23→25/36 ⇒ REVERTED). Then validate the now-mappable cert via
+`/expand-sap-accuracy-corpus` on **the cohort cert's UPRN** (below), saving
+epc.json + elmhurst_summary.pdf + elmhurst_worksheet.pdf under
+`real_life_examples/<schema>/uprn_<uprn>/` and committing them.
+
+- **P1 — `built_form` missing (SAP-16.0)** — HIGHEST impact (5 certs / 3 targets:
+  10013151061, 100020397529, 100020407755). Cert→UPRN: 8742-…-4926→10070004512,
+  0004-…-7395→100020407745, 2338-…-5950→100020407732, 9328-…-0924→100020407771
+  (8904-…-4623 has NO uprn — validate via one of the others). ⚠ built_form IS a
+  strong prediction similarity feature → highest gate-regression risk: DERIVE it
+  (from `dwelling_type` text / `property_type`), don't flat-default; re-measure the
+  gate. Corpus-validate uprn 10070004512.
+- **P2 — `photovoltaic_supply` lodged as a LIST → AttributeError (`'list' has no
+  attribute none_or_no_details`)** — target 22086690 (BN2 9ZN). NEW real shape bug
+  (not missing-field); escaped the skip net until widened above. Clean fix: coerce
+  the list shape in the PV mapper (like the Measurement-dict co2 fix). Low gate risk.
+  ⚠ FIRST identify the offending cert in the BN2 9ZN cohort (lookup didn't surface it
+  in the first 50 — search the full cohort for `photovoltaic_supply` as list).
+- **P3 — `window` missing, `windows` is a DICT not list (SAP-16.1)** — cert
+  9768-…-7974 → uprn 100062614005, target 100061905741. Clean normaliser fix: handle
+  `windows` as a dict (take it directly) in `_normalize_sap_schema_16_x`. Corpus-
+  validate uprn 100062614005.
+- **P4 — `multiple_glazed_proportion` missing (SAP-16.3)** — cert 0418-…-7970 → uprn
+  22144943, target 22082258. The flat-100 default REGRESSED the gate (reverted).
+  DERIVE it (single→0 / double→100 from `multiple_glazing_type` / window desc) so it
+  doesn't perturb similarity, then re-measure the gate. Corpus-validate uprn 22144943.
+- **P5 — RdSAP-21.0.0 systematic ADR-0028 alignment** (`SapWindow.glazing_gap` +
+  13 top-level fields) — cert 3135-…-2206 → uprn 100021919725, target 100021919718.
+  Align RdSapSchema21_0_0 Optionals to 21.0.1 + review `from_rdsap_schema_21_0_0`
+  None-coalescing. Larger/riskier. Corpus-validate uprn 100021919725.
+- **P6 — `window` + genuinely sparse (SAP-16.2)** — cert 8257-…-4992 → uprn
+  10070009758, target 100020401711. Also omits door_count/habitable/glazed_area →
+  fail-loud is likely CORRECT (insufficient data). DOCUMENT decision; don't force-map.
+
+After all: re-run the modelling_e2e failure sweep to confirm the cohort skips drop
+to zero (or only P6-class), and that no new gate regression slipped in.
+
+### ✅ OUTCOMES (2026-06-23, "do all") — 553 tests pass, gate 26/26, 0 new pyright
+- **P1 ✅ FIXED** — `_normalize_sap_schema_16_x` derives `built_form` from the
+  dwelling_type text (`_derive_built_form_16x`: Mid-terrace→4 / End-terrace→3 /
+  Semi-detached→2 / Detached→1; flats → modal 4). built_form is ML-only (SAP calc
+  never reads it) so SAP-neutral; gate verified insensitive (26/26). 5 SAP-16.0
+  flat certs now map (8742→SAP 66=lodged, etc.). +2 tests.
+- **P2 ✅ FIXED** — `photovoltaic_supply` as a measured-array LIST crashed
+  `from_rdsap_schema_{17_0,17_1,18_0,19_0,20_0_0}` (`'list' has no attribute
+  none_or_no_details`). All 5 now route through `_map_schema_21_pv`, whose list
+  branch is made dict-tolerant (`_pv_array_field` reads dict OR dataclass) — they
+  now CAPTURE the PV arrays like 21.0.x instead of crashing/dropping. cert
+  6102-…-2292 (uprn 22086693, RdSAP-20.0.0) maps with 2 arrays @1.14kW. +test.
+- **P3 ✅ generalised / cert fail-loud** — `_normalize_sap_schema_16_x` now handles
+  `windows` as a dict (not just list). cert 9768 (uprn 100062614005, 16.1) still
+  fails loud — it ALSO omits door_count/habitable_room_count/glazed_area
+  (genuinely sparse "High performance glazing" cert) → CORRECT to fail; resilience
+  skips+reports it.
+- **P6 ✅ same as P3** — cert 8257 (uprn 10070009758, 16.2) — identical sparse
+  "High performance glazing" pattern, fails loud on door_count. Correct.
+- **P4 ⚖️ FOR KHALIM (gate decision)** — left `multiple_glazed_proportion` fail-loud.
+  Re-confirmed: defaulting it (ANY value) still regresses the gate (29/36 +
+  25/36) because it makes a frozen-fixture cert a prediction donor — the field is
+  ML-only (not carried to EpcPropertyData), so the regression is donor-pool
+  composition, NOT the value. Closing this gap needs a GATE RE-BASELINE decision
+  (whether the donor-pool tip is benign per the #1245 precedent) — your call; I did
+  not unilaterally loosen the tighten-only gate.
+- **P5 🔍 FOR KHALIM (large recursive alignment)** — RdSAP-21.0.0 lacks the
+  ADR-0028 Optional widenings 21.0.1 has, and the gap is RECURSIVE (not just the
+  ~13 top-level + SapWindow): `SapBuildingPart` (13 fields incl.
+  construction_age_band/wall_construction/identifier — CORE FABRIC),
+  `SapEnergySource` (pv_battery_count, wind_turbine_details), MainHeatingDetail,
+  ShowerOutlets, PvBatteries, Addendum, + top-level low_energy_fixed_lighting_
+  bulbs_count/multiple_glazed_proportion/suggested_improvements. Aligning to
+  21.0.1's "everything Optional, tolerate sparse" means the 21.0.0 mapper must
+  None-coalesce CORE fabric — a modelling decision in your domain. Partial widening
+  was REVERTED (half-aligned schema is worse). cert 3135 (uprn 100021919725) stays
+  fail-loud → resilience skips+reports. Recommend: align RdSapSchema21_0_0 fully to
+  21.0.1 (recursive) + review mapper coalescing, in a dedicated change.
+- **Corpus-validation pending** for the 2 now-mappable certs (P1 uprn 10070004512,
+  P2 uprn 22086693) — Elmhurst build + summary/worksheet + commit.
+
+- [x] 🔧 10096028301 — SAP-Schema-19.1.0 (full-SAP g/f FLAT, band M, combi PCDB 17929, MEV, AP50 3.5) · eng 82 / elm 82 (lodged 85) · PINNED engine 82. 🔧 mapper added: `from_sap_schema_19_1_0`. Built in Elmhurst (boiler 17929 via search, control CBE/2106, water from primary, MEV on, AP50 Blower Door 3.5+cert). Engine EXACTLY matches Elmhurst worksheet (82.11 vs 82); engine-on-Elmhurst-inputs 82.16 ≈ 82 → calculator faithful. −3 vs lodged = measured-U-vs-RdSAP-default + MEV extract-not-recovery (documented). No mapper change beyond coverage.
+- [x] 🔧 100021943298 — SAP-Schema-16.1 (g/f FLAT, band B, solid-brick internal, combi PCDB 10328) · eng 76 / elm 75 (lodged 72) · PINNED engine 76. 🔧 mapper added: `from_sap_schema_16_1`. Built in Elmhurst (boiler 10328 via search, control CBE/2106, water from primary, wall insulation thickness Unknown); worksheet 75 → engine within ~1 (tightest agreement, reduced-field). Boiler-select + water-heating + control dialogs all driven via automation (two-step row→Select / cascade + coordinate-OK). No mapper change beyond coverage.
+
+(Refactor: split the full-SAP + 16.x dispatch tuples into dedicated per-schema
+mappers — from_sap_schema_17_0/18_0_0/19_1_0 + from_sap_schema_16_0/1/2/3 — so
+each schema has one mapper home for future divergence; all delegate to shared logic.)
+
 ## E2E testing set
 
 UPRNs needed for end-to-end testing (also tracked in The 100 below).
@@ -41,30 +289,30 @@ Skip the 🚩 MVHR / 🚩 heat-pump-fuel and ⛔ sparse certs.
 - [x] 10093116543 — SAP-17.1 (2017 gas-combi semi) · eng 81 / elm 77 (lodged 82) · PINNED engine 81. +4 = documented full-SAP→RdSAP residual, NOT a mapper bug: ~1.5 floor-U (cert lodges measured 0.11 vs Elmhurst RdSAP solid default 0.23; U-known override disabled) + ~1 boiler-eff (cert PCDB 17644 88.5% vs Elmhurst generic BGW combi 84%; PCDB search disabled, 89% cascade option is a regular boiler needing a cylinder) + ~0.5 roof band-L/infil. Conservatory leftover from prior cert cleared (worksheet 73→77). No mapper change.
 - [x] 🔧 10093116529 — SAP-17.1 (2017 gas-combi ground-floor FLAT, TFA 49) · eng 81 / elm 78 (lodged 81) · PINNED engine 81. 🔧 FIXED a real calc bug: full-SAP flats took the 0.25 house party-wall default instead of the RdSAP Table-15 flat 0.0 (flatness is in dwelling_type, not property_type) — heat_transmission._is_flat_or_maisonette_dwelling; +regression test. Cert lodges party u_value 0; Elmhurst worksheet 0.0; fix 80→81. Residual +3 vs Elmhurst = documented full-SAP→RdSAP gap (measured wall 0.184/floor 0.12 + PCDB 88.5% vs generic 84%). Calculator faithful: fed Elmhurst's Us, HTC 93.4 vs ~94. House→Flat Elmhurst switch (storeys→1, roof→another-dwelling-above). No mapper change.
 - [ ] 🔧 100020933699 — SAP-16.2 SCHEMA COVERAGE ADDED (end-terrace house, band G). 16.2 is structurally RdSAP-17.1 (reduced fields, glazed_area band, construction-code building parts) under a different name; mapped via `_normalize_sap_schema_16_2` (renames windows→window, main_gas→mains_gas, boiler_index_number→main_heating_index_number, wwhrs→instantaneous_wwhrs + defaults) → reuses from_rdsap_schema_17_1. 🔧 Also fixed: "Single glazed" description honoured when multiple_glazing_type="ND" (was defaulting to double; RdSAP-21 code 5) → eng 72→71. +4 regression tests, sap_16_2.json fixture, 0 new pyright errors. eng 71 / lodged 70. ⚠ Known gap: 16.2 lodges no party_wall_length → end-terrace party wall unmodelled (likely the residual +1). ⏳ Elmhurst build (partial: PropDesc/Dims/Walls/Roofs done) + pin still pending.
-- [ ] 🔧 44012843 — SAP-16.3 schema coverage (same _normalize_sap_schema_16_x reduced-field path as 16.2) · eng 79 / lodged 81 · g/f flat band K · Elmhurst pin pending
-- [ ] 🔧 10023444324 — SAP-17.0 schema coverage (full-SAP shape ≡ 17.1; dispatched to from_sap_schema_17_1, no normalization) · eng 80 / lodged 82 · Elmhurst pin pending
+- [x] 🔧 44012843 — SAP-16.3 (g/f FLAT, band K, cavity insulated, REGULAR boiler PCDB 9895 + cylinder, double glazed) · eng 79 / elm 78 (lodged 81) · PINNED engine 79. Built in Elmhurst (boiler 9895 via search, control CBE/2106, cylinder Large/foam/50mm, water from primary). Engine within ~1 (78.82 vs 78); engine-on-Elmhurst-inputs 78.48 ≈ 78 → calculator faithful. −2 vs lodged = reduced-field defaults. No mapper change beyond coverage.
+- [x] 🔧 10023444324 — SAP-17.0 (full-SAP g/f FLAT, band M, combi PCDB 16211, MEV, AP50 3.2, party wall 6.43) · eng 80 / elm 80 (lodged 82) · PINNED engine 80. Built in Elmhurst (boiler 16211 via search, control CBE/2106, water from primary, MEV on, AP50 Blower Door 3.2, party wall 6.43). Engine EXACTLY matches Elmhurst worksheet (80.13 vs 80); engine-on-Elmhurst-inputs 81.03 ≈ 80 → calculator faithful. −2 vs lodged = full-SAP→RdSAP residual (measured U + MEV). No mapper change beyond coverage.
 - [⚠] 10092970673 — SAP-17.0 · eng 77 / lodged 86 · 🚩 MVHR idx 500418 not credited (flagged)
 - [⚠] 10094601287 — SAP-18.0.0 · eng 80 / lodged 84 · 🚩 MVHR idx 500230 not credited (flagged)
-- [ ] 10090844932 — RdSAP-20.0.0 · eng 78 / lodged 78
+- [x] 10090844932 — RdSAP-20.0.0 (end-terrace HOUSE 2-storey, band L, combi PCDB 10327, party wall 4.93, 250mm loft) · eng 78 / elm 77 (lodged 78) · PINNED engine 78. Built in Elmhurst (House, boiler 10327 via search, control CBE/2106, water from primary, party wall 4.93 filled). Within ~1 (78.13 vs 77); engine-on-Elmhurst-inputs 77.24 ≈ 77 → faithful.
 - [⛔] 10090844948 — SAP-16.3 · NOT MAPPABLE (ValueError: RdSapSchema17_1: missing required field )
-- [ ] 100090182288 — SAP-16.2 · eng 71 / lodged 71
+- [x] 100090182288 — SAP-16.2 (semi-detached HOUSE 2-storey, band D, filled cavity, combi PCDB 10327, double glazed) · eng 71 / elm 69 (lodged 71) · PINNED engine 71. Built in Elmhurst (semi, boiler 10327, control CBE/2106, water from primary, party wall 4.28 filled — Elmhurst requires non-zero for a semi). +2 vs Elmhurst = documented 16.2 reduced-field party-wall gap (gov-API lodges no party_wall_length → engine models none); engine-on-Elmhurst-inputs 69.35 ≈ worksheet 69 → calculator faithful.
 - [⚠] 10093114053 — SAP-17.0 · eng 93 / lodged 79 · 🚩 heat-pump fuel-39 (flagged)
-- [ ] 10091568921 — SAP-17.1 · eng 82 / lodged 85
-- [ ] 10093718424 — SAP-17.1 · eng 81 / lodged 84
-- [ ] 10022893721 — RdSAP-18.0 · eng 79 / lodged 79
-- [ ] 10023443426 — RdSAP-21.0.1 · eng 76 / lodged 76
-- [ ] 10093412452 — SAP-17.1 · eng 81 / lodged 84
+- [x] 10091568921 — SAP-17.1 (full-SAP END-TERRACE HOUSE 2-storey, band L 2018, combi PCDB 17615 Potterton Promax Ultra, party wall 40.56m², natural vent + 3 extract fans, AP50 4.45, 20 LED, measured U 0.18/0.10/0.15) · eng 82 / elm 80 (lodged 85) · PINNED engine 82. Built in Elmhurst (end-terrace house, boiler 17615, CBE/2106, water from primary, party wall 8.45 filled, AP50 Blower Door 4.45+cert). +2 = documented full-SAP→RdSAP residual (measured U beats band-L defaults); engine-on-Elmhurst-inputs 80.16 ≈ worksheet 80 → faithful. has_hot_water_cylinder lodged true but combi PCDB + no cylinder detail → built combi. No mapper change.
+- [x] 10093718424 — SAP-17.1 (full-SAP SEMI-DETACHED HOUSE 2-storey, band L, combi PCDB 17615, party wall 40.89m², natural vent + 3 extract fans, AP50 4.18, 20 LED, measured U 0.25/0.13/0.16; sibling of 10091568921) · eng 81 / elm 80 (lodged 84) · PINNED engine 81. +1 = documented full-SAP→RdSAP residual; engine-on-Elmhurst-inputs 80.12 ≈ worksheet 80 → faithful. Built combi (cylinder lodged true but combi PCDB). No mapper change.
+- [x] 10022893721 — RdSAP-18.0 (GF FLAT, band I, cavity insulated, ELECTRIC STORAGE HEATERS SAP 402 SEB Modern slimline + manual charge CSA/2401, immersion off-peak dual + cylinder Normal/110L foam 50mm, party wall 21.48) · eng 79 / elm 81 (lodged 79) · PINNED engine 79 = lodged. Built in Elmhurst (storage SEB, CSA control, immersion dual + cylinder, **Dual electricity meter / Economy 7**). engine -2 vs Elmhurst, within tolerance; engine-on-Elmhurst-inputs 78.76 ≈ 79 → faithful. Economy-7 off-peak pricing is CORRECT (Table 12a/13, fixed PR #1217) — NOT a bug. ⚠ Earlier mis-build left Elmhurst meter on Single/Standard → bogus worksheet 66; fixed by setting Dual meter on the Meters sub-tab. 🔧 First non-boiler cert — SOLVED storage-heater automation (see banked findings: clear PCDB boiler → ButtonMainHeatingCode Electric→Electric→Storage→SEB; set Dual meter). build_10022893721.py complete.
+- [x] 10023443426 — RdSAP-21.0.1 (END-TERRACE HOUSE 2-storey, band L, cavity insulated, combi PCDB 17045 Ideal Logic ES35, 300mm loft, party wall 9.2 lodged, 11 dbl-glazed windows ~10.3m², 9 LED, mains-gas room-heater SECONDARY code 612 eff 0.20) · eng 76 / elm 79 (lodged 76) · PINNED engine 76 = lodged EXACTLY (native RdSAP-21.0.1 schema; reproduces every component: space 6129/main 6247/HW 2752/CO2 2232). Elmhurst +3 is a BUILD gap — my build omitted the lodged secondary gas fire (engine models it 3065 kWh = 0.1÷0.20); engine-on-Elmhurst-inputs 79.29 ≈ worksheet 79 → calculator faithful. NB: shared-assessment storage→combi reset needed (clear leftover MainHeatingCode) + meter reset to Single. build_10023443426.py.
+- [x] 10093412452 — SAP-17.1 (full-SAP END-TERRACE HOUSE 2-storey, band L, combi PCDB 17615, party wall 41.01m², natural vent + 3 extract fans, AP50 4.62, 20 LED, measured U 0.25/0.13/0.16; sibling of 10093718424/10091568921) · eng 81 / elm 80 (lodged 84) · PINNED engine 81. +1 = documented full-SAP→RdSAP residual; engine-on-Elmhurst-inputs 79.91 ≈ worksheet 80 → faithful. Built combi (cylinder lodged true but combi PCDB). No mapper change.
 - [⛔] 10014314798 — SAP-16.2 · NOT MAPPABLE (ValueError: RdSapSchema17_1: missing required field )
 - [⚠] 10094601294 — SAP-18.0.0 · eng 81 / lodged 84 · 🚩 MVHR idx 500230 not credited (flagged)
-- [ ] 10090343335 — SAP-17.0 · eng 86 / lodged 88
-- [ ] 10093115480 — SAP-17.1 · eng 81 / lodged 81
-- [ ] 68151071 — RdSAP-17.0 · eng 68 / lodged 70
-- [ ] 100021985993 — SAP-16.2 · eng 74 / lodged 70
-- [ ] 100020665611 — RdSAP-20.0.0 · eng 36 / lodged 37
+- [🔍] 10090343335 — SAP-17.0 (full-SAP SEMI-DETACHED HOUSE, 3-storey incl. ROOM-IN-ROOF top floor h1.95, combi PCDB 16841 Vaillant ecoTEC plus 824, party wall 48.51m², 10 windows 18.46m² + 1 ROOF WINDOW type-12 2.31m², 3 doors, AP50 3.98, 10 LED, measured U walls 0.18/roof 0.15/floor 0.13, TFA 122) · eng 86 / lodged 88 · 🔍 FLAGGED — NEEDS NEW BUILD PATTERN: room-in-roof + roof-window not covered by templates (Elmhurst DIM panel only exposes 2 floors; room-in-roof entered via RoomInRoofMain age-band + separate RiR dimensions TBD; roof window type-12 entry TBD). Elmhurst build + worksheet comparison pending. Skipped in autonomous sweep to keep momentum; revisit with a dedicated room-in-roof build.
+- [x] 10093115480 — SAP-17.1 (full-SAP END-TERRACE BUNGALOW single-storey, band L 2016, combi PCDB 16841, party wall 10.99m² CF filled, natural vent + 2 fans, AP50 3.85, 9 LED, measured U 0.19/0.11/0.12, TFA 56) · eng 81 / elm 78 (lodged 81) · PINNED engine 81 = lodged EXACT. +3 = documented full-SAP→RdSAP residual (measured U beats band-L defaults); engine-on-Elmhurst-inputs 78.29 ≈ worksheet 78 → faithful. Built combi. No mapper change.
+- [🔍] 68151071 — RdSAP-17.0 (semi-detached BUNGALOW single-storey, band H 1991-1995, cavity insulated, REGULAR boiler PCDB 17550 Worcester Greenstar 18Ri + cylinder size2/foam/50mm, pitched 200mm loft, suspended uninsulated floor, party wall 9.48 lodged, TFA 50) · eng 68 / elm 71 (lodged 70) · 🔍 FLAGGED — MAPPER GAP (cylinder insulation thickness dropped). engine -3 vs Elmhurst, -2 vs lodged; engine HW 3446 kWh vs Elmhurst 2911. ROOT CAUSE confirmed: raw cert lodges sap_heating.cylinder_insulation_thickness=50 but the RdSAP-17.0 mapper maps cylinder_size + cylinder_insulation_type only, leaving EpcPropertyData.sap_heating.cylinder_insulation_thickness_mm=None → engine assumes a poorly-insulated cylinder → over-counts HW cylinder loss → under-rates. FIX: carry cylinder_insulation_thickness → cylinder_insulation_thickness_mm in the RdSAP mapper (check blast radius across schemas + regression). engine-on-Elmhurst-inputs 71.34 ≈ worksheet 71 → calculator faithful (gap is purely the dropped input). build_68151071.py + evidence saved.
+- [x] 100021985993 — SAP-16.2 (END-TERRACE BUNGALOW single-storey, band C, solid-brick internal insulation, combi PCDB 10328, double glazed, 100mm loft, suspended uninsulated floor) · eng 74 / elm 72 (lodged 70) · PINNED engine 74. Built in Elmhurst (end-terrace, boiler 10328, control CBE/2106, water from primary, party wall 6.89 solid masonry — Elmhurst requires non-zero for an end-terrace). +2 vs Elmhurst = documented 16.2 reduced-field party-wall gap (gov-API lodges no party_wall_length → engine models none; worksheet's only extra element is party wall 16.19m²×U0.25=4.05 W/K ≈ 2 SAP). engine-on-Elmhurst-PDF-inputs 67 is PDF-parser noise (HW over-parsed), not a calc bug.
+- [x] 100020665611 — RdSAP-20.0.0 (END-TERRACE HOUSE 2-storey, band E 1967-1975, cavity UNINSULATED, ELECTRIC STORAGE HEATERS SAP 402 SEB + manual charge CSA/2401, immersion off-peak Economy-7 Dual + cylinder Normal/110L foam 38mm, double glazed PVC, party wall 9.5m/floor U0.25, secondary portable electric heaters/SAP 691, TFA 87.4) · eng 36 / elm 35 (lodged 37) · PINNED engine 36. Built in Elmhurst (storage SEB, CSA control, immersion Dual + cylinder, **Dual electricity meter**, secondary electric room heater). engine-on-Elmhurst-inputs 35 ≡ worksheet 35 → calculator faithful; eng 36 ≈ lodged 37 (−1). Cylinder thickness 38mm correctly carried by RdSAP-20.0.0 mapper (no cylinder-gap here). ⚠ Secondary had to be set EXPLICITLY: shared-assessment leftover House-Coal secondary (633) survived `present=No` and inflated worksheet to 44 until overwritten with electric room heater. build_100020665611.py. No mapper change.
 - [⚠] 10093388044 — SAP-17.1 · eng 87 / lodged 93 · 🚩 heat-pump fuel-39 (flagged)
-- [ ] 10090944225 — SAP-17.0 · eng 81 / lodged 82
+- [x] 10090944225 — SAP-17.0 (full-SAP GROUND-FLOOR FLAT, band L 2015, combi PCDB 17045 Ideal Logic Combi ES35, control 2110 CBI time+temp zone, NATURAL vent + 2 extract fans, AP50 3.48, party wall 3.41m² U0, measured U walls 0.27/floor 0.14/window 1.41, TFA 49.97, no cylinder/no secondary) · eng 81 / elm 80 (lodged 82) · PINNED engine 81. Built in Elmhurst (boiler 17045 via search, control CBI/2110 under Boilers→Zone control, water from primary, AP50 Blower Door 3.48+cert, Single meter). engine-on-Elmhurst-inputs 80 = worksheet 80 (exact) → faithful; eng 81 ≈ lodged 82 (−1). Remaining +1 = documented full-SAP→RdSAP residual localised to the FLOOR (cert measured U 0.14 vs Elmhurst RdSAP solid default 0.23; walls/party match) — engine correctly uses measured 0.14, NOT a mapper bug. ⚠ Initial worksheet read 79 due to a spurious leftover REA/691 electric secondary (Table-11 frac 0.1, ~1.5 SAP) the shared assessment retained — REMOVED via present=Yes→clear secondary code to empty→present=No (79→80). 🔧 Build notes: storage→combi reset (clear leftover SEB MainHeatingCode pass-1, reset meter Single); control 2110 is Boilers→**Zone control**→CBI (not Standard). No mapper change.
 - [⚠] 10090341811 — SAP-17.0 · eng 80 / lodged 89 · 🚩 MVHR idx 500352 not credited (flagged)
-- [ ] 10010215568 — RdSAP-17.1 · eng 75 / lodged 74
+- [x] 10010215568 — RdSAP-17.1 (NATIVE; END-TERRACE BUNGALOW single-storey, band J 2003-2006, cavity as-built insulated, pitched 200mm loft, solid insulated floor, mains-gas REGULAR boiler PCDB 15100 Vaillant Ecotec plus 415 + cylinder size2/Normal-110L/foam 25mm/thermostat, control 2106 CBE, water from primary, party wall 9.8m, double glazed, natural vent, 86% LED, TFA 75.46) · eng 75 / elm 73 (lodged 74) · PINNED engine 75. Built in Elmhurst (House+1 storey bungalow, boiler 15100 via search, CBE/2106 control, cylinder Normal/foam/25mm/thermostat, water from primary, window 11.1m² double-unknown-date). engine-on-Elmhurst-inputs 74 ≈ worksheet 73 → calculator faithful; eng 75 ≈ lodged 74 (+1). +2 vs Elmhurst = documented engine-uses-lodged vs Elmhurst-RdSAP-default (wall thickness 250 default vs lodged 310, baths 2 vs lodged 1). Cylinder thickness 25mm correctly carried by RdSAP-17.1 mapper (no cylinder-gap). ⚠ window width fill bug: `.fill("11.1")` rendered 111.10 (keystroke formatter doubled it) → validation "Width ≤ 50"; fixed by JS-setting width. No mapper change. build_10010215568.py.
 - [⚠] 10093117227 — SAP-17.1 · eng 90 / lodged 80 · 🚩 heat-pump fuel-39 (flagged)
 - [⚠] 10023444170 — SAP-17.0 · eng 80 / lodged 83 · 🚩 MVHR idx 500167 not credited (flagged)
 - [ ] 100020980961 — SAP-16.3 · eng 66 / lodged 65
@@ -86,7 +334,7 @@ Skip the 🚩 MVHR / 🚩 heat-pump-fuel and ⛔ sparse certs.
 - [⚠] 10091636116 — SAP-17.0 · eng 80 / lodged 88 · 🚩 MVHR idx 500249 not credited (flagged)
 - [ ] 10093049853 — SAP-17.0 · eng 82 / lodged 87
 - [ ] 10093390790 — SAP-17.1 · eng 79 / lodged 82
-- [ ] 10093116330 — SAP-17.1 · eng 82 / lodged 83
+- [x] 10093116330 — SAP-17.1 (2017 gas-combi 2-storey semi HOUSE, TFA 73) · eng 82 / elm 78 (lodged 83) · PINNED engine 82. +4 = documented full-SAP→RdSAP residual. Build clean. No mapper change.
 - [ ] 10093116326 — SAP-17.1 · eng 82 / lodged 82
 - [ ] 10090317693 — SAP-17.0 · eng 81 / lodged 88
 - [ ] 10090034872 — SAP-17.0 · eng 83 / lodged 85
@@ -100,7 +348,7 @@ Skip the 🚩 MVHR / 🚩 heat-pump-fuel and ⛔ sparse certs.
 - [⚠] 10093305101 — SAP-17.1 · eng 81 / lodged 85 · 🚩 MVHR idx 500140 not credited (flagged)
 - [ ] 100020933894 — SAP-16.0 · eng 61 / lodged 56
 - [ ] 100020937013 — RdSAP-20.0.0 · eng 70 / lodged 73
-- [ ] 🔧 10023444320 — SAP-17.0 schema coverage (full-SAP ≡ 17.1) · eng 81 / lodged 81 · Elmhurst pin pending
+- [x] 🔧 10023444320 — SAP-17.0 (full-SAP MID-FLOOR FLAT, sibling of 324, combi PCDB 16211, MEV, AP50 3.09, no party wall) · eng 81 / elm 82 (lodged 81) · PINNED engine 81. Built in Elmhurst (boiler 16211, control CBE/2106, water from primary, MEV, AP50 3.09, mid-floor=floor to dwelling below). Within ~1 (81.38 vs 82); engine-on-Elmhurst-inputs 82.46 ≈ 82 → faithful. No mapper change beyond coverage.
 - [ ] 100062188801 — SAP-16.3 · eng 68 / lodged 70
 - [ ] 10008048040 — SAP-16.2 · eng 77 / lodged 75
 - [ ] 10093101966 — SAP-17.1 · eng 82 / lodged 84

.devcontainer/backend/requirements.txt

@@ -24,7 +24,13 @@ ipykernel>=6.25,<7
 dotenv
 psycopg[binary]
 pytest-postgresql
+moto[s3,sqs]==5.0.28  # mock_aws (moto 5.x) for S3/SQS in orchestration tests
 # Formatting
 black==26.1.0
 boto3-stubs
 openai
+# Type checking — strict pyright gate (CLAUDE.md). The pip `pyright` wrapper uses
+# the container's Node. pandas-stubs lets pandas-typed modules check cleanly
+# (CLAUDE.md: add pandas-stubs when introducing pandas to a module).
+pyright==1.1.411
+pandas-stubs

.github/workflows/_deploy_lambda.yml

@@ -97,6 +97,9 @@ on:
 
       TF_VAR_openai_api_key:
         required: false
+
+      TF_VAR_open_epc_api_token:
+        required: false
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -170,6 +173,7 @@ jobs:
           TF_VAR_magicplan_customer_id: ${{ secrets.TF_VAR_magicplan_customer_id }}
           TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }}
           TF_VAR_openai_api_key: ${{ secrets.TF_VAR_openai_api_key }}
+          TF_VAR_open_epc_api_token: ${{ secrets.TF_VAR_open_epc_api_token }}
         run: |
           ECR_REPO_URL_VAR=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then
@@ -222,6 +226,7 @@ jobs:
           TF_VAR_magicplan_customer_id: ${{ secrets.TF_VAR_magicplan_customer_id }}
           TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }}
           TF_VAR_openai_api_key: ${{ secrets.TF_VAR_openai_api_key }}
+          TF_VAR_open_epc_api_token: ${{ secrets.TF_VAR_open_epc_api_token }}
         run: |
           EXTRA_VARS=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then

.github/workflows/deploy_fastapi_backend.yml

@@ -1,170 +1,173 @@
-name: Fast Api Backend Deploy
-
-on:
-  push:
-    branches: [ dev, prod ]
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.10.12
-
-      - name: Install Serverless and plugins
-        run: |
-          npm install -g serverless@^3.38.0
-          npm install -g serverless-domain-manager@^7.3.8
-          npm install -g serverless-python-requirements
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-
-      - name: AWS credentials for dev
-        if: github.ref == 'refs/heads/dev'
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
-
-      - name: AWS credentials for prod
-        if: github.ref == 'refs/heads/prod'
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
-
-      - name: Set domain name
-        id: set_domain
-        run: echo "::set-output name=domain::${{ secrets[format('{0}_DOMAIN_NAME', github.ref_name)] }}"
-
-      - name: Set EPC auth token
-        id: set_auth_token
-        run: echo "::set-output name=auth_token::${{ secrets[format('{0}_EPC_AUTH_TOKEN', github.ref_name)] }}"
-
-      - name: Set Open EPC API token
-        id: set_open_epc_token
-        run: echo "::set-output name=open_epc_token::${{ secrets[format('{0}_OPEN_EPC_API_TOKEN', github.ref_name)] }}"
-
-      # Store port, name and host in github secrets
-      - name: Set DB credentials
-        id: set_db_credentials
-        run: |
-          echo "::set-output name=db_host::${{ secrets[format('{0}_DB_HOST', github.ref_name)] }}"
-          echo "::set-output name=db_port::${{ secrets[format('{0}_DB_PORT', github.ref_name)] }}"
-          echo "::set-output name=db_name::${{ secrets[format('{0}_DB_NAME', github.ref_name)] }}"
-
-      - name: Set ECR credentials
-        id: set_ecr_credentials
-        run: |
-          echo "::set-output name=ecr_uri::${{ secrets[format('{0}_ECR_URI', github.ref_name)] }}"
-
-      - name: Set Secrets
-        id: set_api_secrets
-        run: |
-          echo "::set-output name=sap_predictions_bucket::${{ secrets[format('{0}_SAP_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=carbon_predictions_bucket::${{ secrets[format('{0}_CARBON_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=heat_predictions_bucket::${{ secrets[format('{0}_HEAT_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=lighting_cost_predictions_bucket::${{ secrets[format('{0}_LIGHTING_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=heating_cost_predictions_bucket::${{ secrets[format('{0}_HEATING_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=hot_water_cost_predictions_bucket::${{ secrets[format('{0}_HOT_WATER_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=heating_kwh_predictions_bucket::${{ secrets[format('{0}_HEATING_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=hotwater_kwh_predictions_bucket::${{ secrets[format('{0}_HOTWATER_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=energy_asessments_bucket::${{ secrets[format('{0}_ENERGY_ASSESSMENTS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=google_solar_api_key::${{ secrets[format('{0}_GOOGLE_SOLAR_API_KEY', github.ref_name)] }}"
-          echo "::set-output name=sap_baseline_predictions_bucket::${{ secrets[format('{0}_SAP_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=carbon_baseline_predictions_bucket::${{ secrets[format('{0}_CARBON_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=heat_baseline_predictions_bucket::${{ secrets[format('{0}_HEAT_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
-
-      - name: Setup Docker
-        uses: docker/setup-buildx-action@v1
-
-      #      - name: Setup Docker Buildx
-      #        run: |
-      #          docker buildx create --use
-
-      - name: Build Docker Image For Engine
-        run: |
-          docker buildx build \
-            --platform linux/amd64 \
-            --provenance=false \
-            --output=type=docker \
-            -t fastapi-lambda-image:${{ github.sha }} \
-            -f backend/docker/engine.Dockerfile \
-            .
-
-      - name: Login to ECR
-        run: |
-          aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
-
-      - name: Tag and Push Docker Image to ECR
-        run: |
-          docker tag fastapi-lambda-image:${{ github.sha }} ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
-          docker push ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
-
-      - name: Deploy to AWS Lambda via Serverless
-        env:
-          API_KEY: ${{ secrets.FASTAPI_API_KEY }}
-          ENVIRONMENT: ${{ github.ref_name }}
-          SECRET_KEY: ${{ secrets.NEXTAUTH_SECRET }}
-          PLAN_TRIGGER_BUCKET: 'retrofit-plan-inputs-${{ github.ref_name }}'
-          DATA_BUCKET: 'retrofit-data-${{ github.ref_name }}'
-          PREDICTIONS_BUCKET: 'retrofit-sap-predictions-${{ github.ref_name }}'
-          SAP_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_predictions_bucket }}
-          CARBON_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_predictions_bucket }}
-          HEAT_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_predictions_bucket }}
-          LIGHTING_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.lighting_cost_predictions_bucket }}
-          HEATING_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heating_cost_predictions_bucket }}
-          HOT_WATER_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.hot_water_cost_predictions_bucket }}
-          HEATING_KWH_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heating_kwh_predictions_bucket }}
-          HOTWATER_KWH_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.hotwater_kwh_predictions_bucket }}
-          ENERGY_ASSESSMENTS_BUCKET: ${{ steps.set_api_secrets.outputs.energy_asessments_bucket }}
-          GOOGLE_SOLAR_API_KEY: ${{ steps.set_api_secrets.outputs.google_solar_api_key }}
-          DOMAIN_NAME: ${{ steps.set_domain.outputs.domain }}
-          EPC_AUTH_TOKEN: ${{ steps.set_auth_token.outputs.auth_token }}
-          OPEN_EPC_API_TOKEN: ${{ steps.set_open_epc_token.outputs.open_epc_token }}
-          DB_HOST: ${{ steps.set_db_credentials.outputs.db_host }}
-          DB_PORT: ${{ steps.set_db_credentials.outputs.db_port }}
-          DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }}
-          ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
-          GITHUB_SHA: ${{ github.sha }}
-          SAP_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_baseline_predictions_bucket }}
-          CARBON_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_baseline_predictions_bucket }}
-          HEAT_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_baseline_predictions_bucket }}
-        run: |
-          # Fetch database credentials from AWS Secrets Manager
-          SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString)
-          DB_USERNAME=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_username')
-          DB_PASSWORD=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_password')
-
-          # Set the database credentials as environment variables
-          export DB_USERNAME
-          export DB_PASSWORD
-
-          # Deploy to AWS Lambda via Serverless
-          sls deploy --stage ${{ github.ref_name }} --verbose
-
-      - name: Smoke test deployed /health
-        env:
-          EXPECTED_SHA: ${{ github.sha }}
-          HEALTH_URL: https://api.${{ steps.set_domain.outputs.domain }}/health
-        run: |
-          set -euo pipefail
-          echo "Probing $HEALTH_URL"
-          RESPONSE=$(curl -fsSL --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$HEALTH_URL")
-          echo "Response: $RESPONSE"
-          ACTUAL_SHA=$(echo "$RESPONSE" | jq -r '.sha')
-          if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then
-            echo "::error::SHA mismatch. expected=$EXPECTED_SHA actual=$ACTUAL_SHA"
-            exit 1
-          fi
-          echo "Health check passed. sha=$ACTUAL_SHA"
+# Temporarily disabled — the FastAPI backend deploy pipeline (deploys must be run manually via `sls deploy` while disabled).
+# Commented out to cut GitHub Actions minutes; uncomment to re-enable.
+#
+# name: Fast Api Backend Deploy
+#
+# on:
+#   push:
+#     branches: [ dev, prod ]
+#
+# jobs:
+#   deploy:
+#     runs-on: ubuntu-latest
+#
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@v3
+#
+#       - name: Set up Python
+#         uses: actions/setup-python@v2
+#         with:
+#           python-version: 3.10.12
+#
+#       - name: Install Serverless and plugins
+#         run: |
+#           npm install -g serverless@^3.38.0
+#           npm install -g serverless-domain-manager@^7.3.8
+#           npm install -g serverless-python-requirements
+#
+#       - name: Install dependencies
+#         run: |
+#           python -m pip install --upgrade pip
+#
+#       - name: AWS credentials for dev
+#         if: github.ref == 'refs/heads/dev'
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+#           aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+#           aws-region: eu-west-2
+#
+#       - name: AWS credentials for prod
+#         if: github.ref == 'refs/heads/prod'
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }}
+#           aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }}
+#           aws-region: eu-west-2
+#
+#       - name: Set domain name
+#         id: set_domain
+#         run: echo "::set-output name=domain::${{ secrets[format('{0}_DOMAIN_NAME', github.ref_name)] }}"
+#
+#       - name: Set EPC auth token
+#         id: set_auth_token
+#         run: echo "::set-output name=auth_token::${{ secrets[format('{0}_EPC_AUTH_TOKEN', github.ref_name)] }}"
+#
+#       - name: Set Open EPC API token
+#         id: set_open_epc_token
+#         run: echo "::set-output name=open_epc_token::${{ secrets[format('{0}_OPEN_EPC_API_TOKEN', github.ref_name)] }}"
+#
+#       # Store port, name and host in github secrets
+#       - name: Set DB credentials
+#         id: set_db_credentials
+#         run: |
+#           echo "::set-output name=db_host::${{ secrets[format('{0}_DB_HOST', github.ref_name)] }}"
+#           echo "::set-output name=db_port::${{ secrets[format('{0}_DB_PORT', github.ref_name)] }}"
+#           echo "::set-output name=db_name::${{ secrets[format('{0}_DB_NAME', github.ref_name)] }}"
+#
+#       - name: Set ECR credentials
+#         id: set_ecr_credentials
+#         run: |
+#           echo "::set-output name=ecr_uri::${{ secrets[format('{0}_ECR_URI', github.ref_name)] }}"
+#
+#       - name: Set Secrets
+#         id: set_api_secrets
+#         run: |
+#           echo "::set-output name=sap_predictions_bucket::${{ secrets[format('{0}_SAP_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=carbon_predictions_bucket::${{ secrets[format('{0}_CARBON_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=heat_predictions_bucket::${{ secrets[format('{0}_HEAT_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=lighting_cost_predictions_bucket::${{ secrets[format('{0}_LIGHTING_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=heating_cost_predictions_bucket::${{ secrets[format('{0}_HEATING_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=hot_water_cost_predictions_bucket::${{ secrets[format('{0}_HOT_WATER_COST_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=heating_kwh_predictions_bucket::${{ secrets[format('{0}_HEATING_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=hotwater_kwh_predictions_bucket::${{ secrets[format('{0}_HOTWATER_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=energy_asessments_bucket::${{ secrets[format('{0}_ENERGY_ASSESSMENTS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=google_solar_api_key::${{ secrets[format('{0}_GOOGLE_SOLAR_API_KEY', github.ref_name)] }}"
+#           echo "::set-output name=sap_baseline_predictions_bucket::${{ secrets[format('{0}_SAP_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=carbon_baseline_predictions_bucket::${{ secrets[format('{0}_CARBON_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#           echo "::set-output name=heat_baseline_predictions_bucket::${{ secrets[format('{0}_HEAT_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
+#
+#       - name: Setup Docker
+#         uses: docker/setup-buildx-action@v1
+#
+#       #      - name: Setup Docker Buildx
+#       #        run: |
+#       #          docker buildx create --use
+#
+#       - name: Build Docker Image For Engine
+#         run: |
+#           docker buildx build \
+#             --platform linux/amd64 \
+#             --provenance=false \
+#             --output=type=docker \
+#             -t fastapi-lambda-image:${{ github.sha }} \
+#             -f backend/docker/engine.Dockerfile \
+#             .
+#
+#       - name: Login to ECR
+#         run: |
+#           aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
+#
+#       - name: Tag and Push Docker Image to ECR
+#         run: |
+#           docker tag fastapi-lambda-image:${{ github.sha }} ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
+#           docker push ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
+#
+#       - name: Deploy to AWS Lambda via Serverless
+#         env:
+#           API_KEY: ${{ secrets.FASTAPI_API_KEY }}
+#           ENVIRONMENT: ${{ github.ref_name }}
+#           SECRET_KEY: ${{ secrets.NEXTAUTH_SECRET }}
+#           PLAN_TRIGGER_BUCKET: 'retrofit-plan-inputs-${{ github.ref_name }}'
+#           DATA_BUCKET: 'retrofit-data-${{ github.ref_name }}'
+#           PREDICTIONS_BUCKET: 'retrofit-sap-predictions-${{ github.ref_name }}'
+#           SAP_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_predictions_bucket }}
+#           CARBON_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_predictions_bucket }}
+#           HEAT_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_predictions_bucket }}
+#           LIGHTING_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.lighting_cost_predictions_bucket }}
+#           HEATING_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heating_cost_predictions_bucket }}
+#           HOT_WATER_COST_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.hot_water_cost_predictions_bucket }}
+#           HEATING_KWH_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heating_kwh_predictions_bucket }}
+#           HOTWATER_KWH_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.hotwater_kwh_predictions_bucket }}
+#           ENERGY_ASSESSMENTS_BUCKET: ${{ steps.set_api_secrets.outputs.energy_asessments_bucket }}
+#           GOOGLE_SOLAR_API_KEY: ${{ steps.set_api_secrets.outputs.google_solar_api_key }}
+#           DOMAIN_NAME: ${{ steps.set_domain.outputs.domain }}
+#           EPC_AUTH_TOKEN: ${{ steps.set_auth_token.outputs.auth_token }}
+#           OPEN_EPC_API_TOKEN: ${{ steps.set_open_epc_token.outputs.open_epc_token }}
+#           DB_HOST: ${{ steps.set_db_credentials.outputs.db_host }}
+#           DB_PORT: ${{ steps.set_db_credentials.outputs.db_port }}
+#           DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }}
+#           ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
+#           GITHUB_SHA: ${{ github.sha }}
+#           SAP_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_baseline_predictions_bucket }}
+#           CARBON_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_baseline_predictions_bucket }}
+#           HEAT_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_baseline_predictions_bucket }}
+#         run: |
+#           # Fetch database credentials from AWS Secrets Manager
+#           SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString)
+#           DB_USERNAME=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_username')
+#           DB_PASSWORD=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_password')
+#
+#           # Set the database credentials as environment variables
+#           export DB_USERNAME
+#           export DB_PASSWORD
+#
+#           # Deploy to AWS Lambda via Serverless
+#           sls deploy --stage ${{ github.ref_name }} --verbose
+#
+#       - name: Smoke test deployed /health
+#         env:
+#           EXPECTED_SHA: ${{ github.sha }}
+#           HEALTH_URL: https://api.${{ steps.set_domain.outputs.domain }}/health
+#         run: |
+#           set -euo pipefail
+#           echo "Probing $HEALTH_URL"
+#           RESPONSE=$(curl -fsSL --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$HEALTH_URL")
+#           echo "Response: $RESPONSE"
+#           ACTUAL_SHA=$(echo "$RESPONSE" | jq -r '.sha')
+#           if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then
+#             echo "::error::SHA mismatch. expected=$EXPECTED_SHA actual=$ACTUAL_SHA"
+#             exit 1
+#           fi
+#           echo "Health check passed. sha=$ACTUAL_SHA"

.github/workflows/deploy_terraform.yml

@@ -736,6 +736,44 @@ jobs:
       TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
       TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
 
+  # ============================================================
+  # Build Modelling E2E image and Push
+  # ============================================================
+  modelling_e2e_image:
+    needs: [determine_stage, shared_terraform]
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      ecr_repo: modelling-e2e-${{ needs.determine_stage.outputs.stage }}
+      dockerfile_path: applications/modelling_e2e/Dockerfile
+      build_context: .
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+
+  # ============================================================
+  # Deploy Modelling E2E Lambda
+  # ============================================================
+  modelling_e2e_lambda:
+    needs: [modelling_e2e_image, determine_stage]
+    uses: ./.github/workflows/_deploy_lambda.yml
+    with:
+      lambda_name: modelling_e2e
+      lambda_path: deployment/terraform/lambda/modelling_e2e
+      stage: ${{ needs.determine_stage.outputs.stage }}
+      ecr_repo: modelling-e2e-${{ needs.determine_stage.outputs.stage }}
+      image_digest: ${{ needs.modelling_e2e_image.outputs.image_digest }}
+      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+      TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
+      TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
+      TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
+      TF_VAR_open_epc_api_token: ${{ secrets.DEV_OPEN_EPC_API_TOKEN }}
+      TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
+
   # ============================================================
   # Deploy Hubspot ETL Lambda
   # ============================================================

.github/workflows/integration_tests.yml

@@ -1,35 +1,38 @@
-name: Rebaselining Integration Test
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  rebaselining-integration-test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Install tox via Makefile
-        run: |
-          make setup
-
-      - name: Configure AWS credentials for dev
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
-
-      - name: Run only rebaselining integration test
-        env:
-          EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
-        run: |
-          make test ARGS="-m integration"
+# Temporarily disabled — the rebaselining integration suite.
+# Commented out to cut GitHub Actions minutes; uncomment to re-enable.
+#
+# name: Rebaselining Integration Test
+#
+# on:
+#   pull_request:
+#     branches:
+#       - main
+#
+# jobs:
+#   rebaselining-integration-test:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@v4
+#
+#       - name: Set up Python 3.11
+#         uses: actions/setup-python@v4
+#         with:
+#           python-version: '3.11'
+#
+#       - name: Install tox via Makefile
+#         run: |
+#           make setup
+#
+#       - name: Configure AWS credentials for dev
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+#           aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+#           aws-region: eu-west-2
+#
+#       - name: Run only rebaselining integration test
+#         env:
+#           EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
+#         run: |
+#           make test ARGS="-m integration"

.github/workflows/lambda_smoke_tests.yml

@@ -1,144 +1,154 @@
-name: Lambda Smoke Tests
+# name: Lambda Smoke Tests
 
-on:
-  pull_request:
-    branches:
-      - main
+# on:
+#   pull_request:
+#     branches:
+#       - main
 
-jobs:
-  # ============================================================
-  # Ara Engine
-  # ============================================================
-  ara_engine_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/docker/engine.Dockerfile
-      build_context: .
-      service_name: ara-engine
+# jobs:
+#   # ============================================================
+#   # Ara Engine
+#   # ============================================================
+#   ara_engine_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/docker/engine.Dockerfile
+#       build_context: .
+#       service_name: ara-engine
 
-  # ============================================================
-  # Address 2 UPRN
-  # ============================================================
-  address2uprn_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/address2UPRN/handler/Dockerfile
-      build_context: .
-      service_name: address2uprn
+#   # ============================================================
+#   # Address 2 UPRN
+#   # ============================================================
+#   address2uprn_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/address2UPRN/handler/Dockerfile
+#       build_context: .
+#       service_name: address2uprn
 
-  # ============================================================
-  # Postcode Splitter
-  # ============================================================
-  postcode_splitter_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/postcode_splitter/handler/Dockerfile
-      build_context: .
-      service_name: postcode-splitter
+#   # ============================================================
+#   # Postcode Splitter
+#   # ============================================================
+#   postcode_splitter_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/postcode_splitter/handler/Dockerfile
+#       build_context: .
+#       service_name: postcode-splitter
 
-  postcode_splitter_ddd_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: applications/postcode_splitter/Dockerfile
-      build_context: .
-      service_name: postcode-splitter-ddd
+#   postcode_splitter_ddd_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/postcode_splitter/Dockerfile
+#       build_context: .
+#       service_name: postcode-splitter-ddd
 
-  # ============================================================
-  # Landlord Description Overrides
-  # ============================================================
-  landlord_description_overrides_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: applications/landlord_description_overrides/Dockerfile
-      build_context: .
-      service_name: landlord-description-overrides
+#   # ============================================================
+#   # Landlord Description Overrides
+#   # ============================================================
+#   landlord_description_overrides_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/landlord_description_overrides/Dockerfile
+#       build_context: .
+#       service_name: landlord-description-overrides
 
-  # ============================================================
-  # Bulk Address2UPRN Combiner
-  # ============================================================
-  bulk_address2uprn_combiner_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/bulk_address2uprn_combiner/handler/Dockerfile
-      build_context: .
-      service_name: bulk-address2uprn-combiner
+#   # ============================================================
+#   # Bulk Address2UPRN Combiner
+#   # ============================================================
+#   bulk_address2uprn_combiner_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/bulk_address2uprn_combiner/handler/Dockerfile
+#       build_context: .
+#       service_name: bulk-address2uprn-combiner
 
-  # ============================================================
-  # Bulk Upload Finaliser
-  # ============================================================
-  bulk_upload_finaliser_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: applications/bulk_upload_finaliser/Dockerfile
-      build_context: .
-      service_name: bulk-upload-finaliser
+#   # ============================================================
+#   # Bulk Upload Finaliser
+#   # ============================================================
+#   bulk_upload_finaliser_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/bulk_upload_finaliser/Dockerfile
+#       build_context: .
+#       service_name: bulk-upload-finaliser
 
-  # ============================================================
-  # Condition ETL
-  # ============================================================
-  condition_etl_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/condition/handler/Dockerfile
-      build_context: .
-      service_name: condition-etl
+#   # ============================================================
+#   # Condition ETL
+#   # ============================================================
+#   condition_etl_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/condition/handler/Dockerfile
+#       build_context: .
+#       service_name: condition-etl
 
-  # ============================================================
-  # Categorisation
-  # ============================================================
-  categorisation_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/categorisation/handler/Dockerfile
-      build_context: .
-      service_name: categorisation
+#   # ============================================================
+#   # Categorisation
+#   # ============================================================
+#   categorisation_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/categorisation/handler/Dockerfile
+#       build_context: .
+#       service_name: categorisation
 
-  # ============================================================
-  # Ordnance Survey
-  # ============================================================
-  ordnance_survey_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/ordnanceSurvey/handler/Dockerfile
-      build_context: .
-      service_name: ordnance-survey
+#   # ============================================================
+#   # Ordnance Survey
+#   # ============================================================
+#   ordnance_survey_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/ordnanceSurvey/handler/Dockerfile
+#       build_context: .
+#       service_name: ordnance-survey
 
-  # ============================================================
-  # Pas Hub Fetcher
-  # ============================================================
-  pashub_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: backend/pashub_fetcher/handler/Dockerfile
-      build_context: .
-      service_name: pashub
+#   # ============================================================
+#   # Pas Hub Fetcher
+#   # ============================================================
+#   pashub_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: backend/pashub_fetcher/handler/Dockerfile
+#       build_context: .
+#       service_name: pashub
 
-  # ============================================================
-  # MagicPlan
-  # ============================================================
-  magic_plan_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: applications/magic_plan/handler/Dockerfile
-      build_context: .
-      service_name: magic-plan
+#   # ============================================================
+#   # MagicPlan
+#   # ============================================================
+#   magic_plan_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/magic_plan/handler/Dockerfile
+#       build_context: .
+#       service_name: magic-plan
 
-  # ============================================================
-  # Audit Generator
-  # ============================================================
-  audit_generator_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: applications/audit_generator/handler/Dockerfile
-      build_context: .
-      service_name: audit-generator
+#   # ============================================================
+#   # Audit Generator
+#   # ============================================================
+#   audit_generator_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/audit_generator/handler/Dockerfile
+#       build_context: .
+#       service_name: audit-generator
 
-  # ============================================================
-  # HubSpot Scraper
-  # ============================================================
-  hubspot_scraper_smoke_test:
-    uses: ./.github/workflows/_smoke_test_lambda.yml
-    with:
-      dockerfile_path: etl/hubspot/scripts/scraper/handler/Dockerfile
-      build_context: .
-      service_name: hubspot-scraper
+#   # ============================================================
+#   # HubSpot Scraper
+#   # ============================================================
+#   hubspot_scraper_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: etl/hubspot/scripts/scraper/handler/Dockerfile
+#       build_context: .
+#       service_name: hubspot-scraper
+
+#   # ============================================================
+#   # Modelling E2E
+#   # ============================================================
+#   hubspot_scraper_smoke_test:
+#     uses: ./.github/workflows/_smoke_test_lambda.yml
+#     with:
+#       dockerfile_path: applications/modelling_e2e/Dockerfile
+#       build_context: .
+#       service_name: modelling_e2e

.github/workflows/protect_releases.yml

@@ -1,17 +1,20 @@
-name: Restrict PR source
-
-on:
-  pull_request:
-    branches:
-      - dev
-
-jobs:
-  check-source-branch:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fail if PR is not from main
-        run: |
-          if [[ "${{ github.head_ref }}" != "main" ]]; then
-            echo "Only PRs from main are allowed into dev"
-            exit 1
-          fi
+# Temporarily disabled — the main→dev PR-source guardrail.
+# Commented out to cut GitHub Actions minutes; uncomment to re-enable.
+#
+# name: Restrict PR source
+#
+# on:
+#   pull_request:
+#     branches:
+#       - dev
+#
+# jobs:
+#   check-source-branch:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Fail if PR is not from main
+#         run: |
+#           if [[ "${{ github.head_ref }}" != "main" ]]; then
+#             echo "Only PRs from main are allowed into dev"
+#             exit 1
+#           fi

.github/workflows/unit_tests.yml

@@ -1,67 +1,70 @@
-name: Run unit tests
-
-on:
-  pull_request:
-    branches:
-      - "**"
-
-
-jobs:
-  test-docker:
-    name: Tests (Docker)
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres:15
-        env:
-          POSTGRES_USER: test
-          POSTGRES_PASSWORD: test
-          POSTGRES_DB: test
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Build test image
-        run: docker build -f Dockerfile.test -t model-test .
-
-      - name: Initialise database schema
-        run: |
-          docker run --rm \
-            --network host \
-            -e DB_HOST=localhost \
-            -e DB_NAME=test \
-            -e DB_USERNAME=test \
-            -e DB_PASSWORD=test \
-            -e DB_PORT=5432 \
-            model-test python scripts/init_db.py
-
-      - name: Run tests
-        run: |
-          docker run --rm \
-            --network host \
-            -e EPC_AUTH_TOKEN=${{ secrets.DEV_EPC_AUTH_TOKEN }} \
-            -e OPEN_EPC_API_TOKEN=${{ secrets.DEV_OPEN_EPC_API_TOKEN }} \
-            -e HUBSPOT_API_KEY=${{ secrets.HUBSPOT_API_KEY }} \
-            -e AWS_ACCESS_KEY_ID=${{ secrets.DEV_AWS_ACCESS_KEY_ID }} \
-            -e AWS_SECRET_ACCESS_KEY=${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} \
-            -e AWS_DEFAULT_REGION=${{ secrets.DEV_AWS_REGION }} \
-            -e DB_HOST=localhost \
-            -e DB_NAME=test \
-            -e DB_USERNAME=test \
-            -e DB_PASSWORD=test \
-            -e DB_PORT=5432 \
-            model-test pytest -vv -m 'not integration'
-
-      # The DDD rewrite (tests/) runs in its own workflow (ddd_tests.yml): its
-      # SQLModel table classes map to the same physical tables as the legacy
-      # backend models and share the one global SQLModel.metadata, so the two
-      # suites cannot be imported into the same pytest process.
+# Temporarily disabled — this Docker-based suite was consuming too many GitHub
+# Actions minutes. Uncomment to re-enable.
+#
+# name: Run unit tests
+#
+# on:
+#   pull_request:
+#     branches:
+#       - "**"
+#
+#
+# jobs:
+#   test-docker:
+#     name: Tests (Docker)
+#     runs-on: ubuntu-latest
+#
+#     services:
+#       postgres:
+#         image: postgres:15
+#         env:
+#           POSTGRES_USER: test
+#           POSTGRES_PASSWORD: test
+#           POSTGRES_DB: test
+#         ports:
+#           - 5432:5432
+#         options: >-
+#           --health-cmd pg_isready
+#           --health-interval 10s
+#           --health-timeout 5s
+#           --health-retries 5
+#
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@v4
+#
+#       - name: Build test image
+#         run: docker build -f Dockerfile.test -t model-test .
+#
+#       - name: Initialise database schema
+#         run: |
+#           docker run --rm \
+#             --network host \
+#             -e DB_HOST=localhost \
+#             -e DB_NAME=test \
+#             -e DB_USERNAME=test \
+#             -e DB_PASSWORD=test \
+#             -e DB_PORT=5432 \
+#             model-test python scripts/init_db.py
+#
+#       - name: Run tests
+#         run: |
+#           docker run --rm \
+#             --network host \
+#             -e EPC_AUTH_TOKEN=${{ secrets.DEV_EPC_AUTH_TOKEN }} \
+#             -e OPEN_EPC_API_TOKEN=${{ secrets.DEV_OPEN_EPC_API_TOKEN }} \
+#             -e HUBSPOT_API_KEY=${{ secrets.HUBSPOT_API_KEY }} \
+#             -e AWS_ACCESS_KEY_ID=${{ secrets.DEV_AWS_ACCESS_KEY_ID }} \
+#             -e AWS_SECRET_ACCESS_KEY=${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} \
+#             -e AWS_DEFAULT_REGION=${{ secrets.DEV_AWS_REGION }} \
+#             -e DB_HOST=localhost \
+#             -e DB_NAME=test \
+#             -e DB_USERNAME=test \
+#             -e DB_PASSWORD=test \
+#             -e DB_PORT=5432 \
+#             model-test pytest -vv -m 'not integration'
+#
+#       # The DDD rewrite (tests/) runs in its own workflow (ddd_tests.yml): its
+#       # SQLModel table classes map to the same physical tables as the legacy
+#       # backend models and share the one global SQLModel.metadata, so the two
+#       # suites cannot be imported into the same pytest process.

.gitignore

@@ -304,6 +304,7 @@ backlog/*
 # Local Claude config files
 .claude/*modelling_cohort.csv
 .claude/settings.local.json
+.claude/settings.json
 
 # Local EPC debug cache (scripts/eon)
 scripts/eon/epc_cache.pkl
@@ -312,3 +313,8 @@ scripts/eon/epc_cache.pkl
 scripts/hyde/.elmhurst-session/
 scripts/hyde/elmhurst_downloads/
 scripts/hyde/.elmhurst-creds.json
+
+# Hyde property-overrides script artifacts
+overrides_cache.json
+overrides_unknowns.csv
+overrides_edits.csv

CONTEXT.md

@@ -35,10 +35,22 @@ _Avoid_: energy rating, EPC grade, EPC score
 The versioned RdSAP or SAP schema that describes the structure of an EPC's raw data (e.g. `RdSAP-Schema-21.0.1`).
 _Avoid_: schema version, EPC format
 
+**Full-SAP Assessment**:
+A cert produced by a **full SAP** calculation rather than the reduced **RdSAP** procedure — lodged with `assessment_type = "SAP"` and a measured-openings structure (`sap_opening_types`, per-opening U-values), typically on an older spec (`SAP-Schema-16.x/17.x`, `sap_version` ~9.9x = SAP 2012). Structurally distinct from an **RdSAP Assessment** (reduced categorical fabric, no measured openings), so it has its own mapper (`from_sap_schema_17_1`), routed to by **structure not label** (a structurally full-SAP cert is mapped here regardless of a mislabelled `schema_type`). Because full-SAP certs lodge pre-10.2 figures they are **rebaselined** to SAP 10.2 like any pre-SAP10 cert (Rebaselining trigger (a)); a mapper that drops their `sap_version` silently suppresses that rebaseline and leaves **Effective Performance** stuck on the lodged value (ADR-0037).
+_Avoid_: SAP cert (ambiguous — every EPC is a SAP-derived cert), on-construction EPC (overlaps but not synonymous)
+
 **Domestic Certificate**:
 An EPC issued for a residential dwelling, as opposed to a commercial one.
 _Avoid_: residential EPC, home EPC
 
+**Non-Separated Conservatory**:
+A conservatory joined to the dwelling without an external-quality thermal separation (open archway or only an internal-quality door). RdSAP treats it as **part of the dwelling** (§6.1): the assessor measures its glazed perimeter, floor area, glazing type and storey height, and its floor area, glazed walls (as a window), glazed roof (as a rooflight) and ground-loss all fold into the dwelling's score. On the gov register this is `conservatory_type = 4`, lodged as a glazed "building part" carrying only those four measurements — never fabric or floor dimensions. It is the only conservatory kind that affects the score, so it must be **carried as a distinct part of the dwelling picture**, not treated as a generic fabric building part (which mis-scores it and lacks the construction fields the register requires).
+_Avoid_: "floor-only building part" / "degenerate part" (it is a fully-measured glazed conservatory — it only *looks* empty when a schema drops its four fields); "heated conservatory" (heating is the question asked of a **Separated Conservatory**, not this one)
+
+**Separated Conservatory**:
+A conservatory partitioned from the dwelling by an external-quality door (substantial, draught-sealed). RdSAP **disregards** it entirely (§6.2) — no measurements are taken and it contributes nothing to the score; the register records only whether it has fixed heaters (`conservatory_type = 2` unheated / `3` heated) and lodges no glazed building part. `conservatory_type = 1` means no conservatory at all.
+_Avoid_: conflating with a **Non-Separated Conservatory** (the separated one is scored as if absent); "thermally separated" as a synonym for "heated" (separation is about the door, heating is a further question asked only once separated)
+
 ### Properties and addresses
 
 **Property**:
@@ -116,7 +128,7 @@ The SAP / EPC Band / carbon emissions / Primary Energy Intensity the modelling p
 _Avoid_: modelled performance, rebaselined performance (only correct when rebaselining ran), scored values
 
 **Calculated SAP10 Performance**:
-The SAP score, EPC Band, CO2 emissions, Primary Energy Intensity, space heating kWh, and hot water kWh produced by **SAP10 Calculation** from a Property's EpcPropertyData. It is **not** a separately-persisted third value-set beside Lodged and Effective: in every baselining scenario the calculator's output *is* the **Effective Performance** (real lodged SAP10 EPC with no overrides ⇒ Calculated = Lodged = Effective; overrides or an estimated / pre-SAP10 EPC ⇒ Calculated = Effective, there being no lodged SAP10 figure to compare against). The calculator is therefore the mechanism that produces Effective Performance, having superseded the old ML-API rebaseliner. The calculator is **load-bearing**: for `sap_version < 10.2` (lodged under a superseded methodology) its output *is* the Effective Performance; for `≥ 10.2` the API's lodged figures are kept and the calculator runs **alongside, logging any divergence** (SAP > 0.5, PEUI/CO2 beyond tolerance) as a validation signal (see [[sap-spec-version]]). It is load-bearing for **Bill Derivation regardless of version** (the EPC lodges no per-end-use kWh), so a calculator strict-raise **aborts the batch** and the un-mapped cert is fixed immediately. ADR-0009 introduced the term, amended by ADR-0010, realized by ADR-0013 (whose shadow stepping-stone is superseded) and ADR-0014.
+The SAP score, EPC Band, CO2 emissions, Primary Energy Intensity, space heating kWh, and hot water kWh produced by **SAP10 Calculation** from a Property's EpcPropertyData. It is **not** a separately-persisted third value-set beside Lodged and Effective: in every baselining scenario the calculator's output *is* the **Effective Performance** (real lodged SAP10 EPC with no overrides ⇒ Calculated = Lodged = Effective; overrides or an estimated / pre-SAP10 EPC ⇒ Calculated = Effective, there being no lodged SAP10 figure to compare against). The calculator is therefore the mechanism that produces Effective Performance, having superseded the old ML-API rebaseliner. The calculator is **load-bearing**: for `sap_version < 10.2` (lodged under a superseded methodology) its output *is* the Effective Performance; for `≥ 10.2` the API's lodged figures are kept **only when the picture is a pristine lodged cert** — when Landlord Overrides or EPC Prediction changed the physical state (Rebaselining trigger (b)/(c)), the calculator's output *is* the Effective Performance even at ≥ 10.2 (ADR-0039), because the accredited lodged figure no longer describes the dwelling. Divergence (SAP > 0.5, PEUI/CO2 beyond tolerance) is logged **only in that pristine-lodged case** — the one case where lodged-vs-calc is a meaningful validation signal (see [[sap-spec-version]]); when the calculator's output is adopted there is nothing to validate against. It is load-bearing for **Bill Derivation regardless of version** (the EPC lodges no per-end-use kWh), so a calculator strict-raise **aborts the batch** and the un-mapped cert is fixed immediately. ADR-0009 introduced the term, amended by ADR-0010, realized by ADR-0013 (whose shadow stepping-stone is superseded) and ADR-0014.
 _Avoid_: calculator output, computed performance, worksheet performance, SAP10 output, calculated value-set (it is not a stored third set)
 
 **SAP10 Calculation**:
@@ -140,9 +152,17 @@ The process that translates an Optimised Package into cert-field changes and pro
 _Avoid_: measure overrides (rejected during ADR-0009 grill — phantom mid-layer), package applier, retrofit simulator
 
 **Bill Derivation**:
-The deterministic process that derives a Property's annual energy **bill**, composed into per-end-use sections (heating, hot water, lighting, appliances, cooking, pumps/fans, …) plus a **total**, by pricing **SAP10 Calculation**'s delivered kWh per end use at **current Fuel Rates** — each end use billed at its fuel's rate, rolled up per fuel for **standing charges** (metered fuels only — gas/electricity; oil/LPG/solid have none) minus **SEG** export credit on PV. Implemented by `BillDerivation` in `domain/billing/` (a cross-stage concern — the Baseline stage derives the current bill, the Modelling stage re-runs it on the post-package end-state for post-retrofit bills; deterministic, ADR-0006). Reads Fuel Rates from a committed static snapshot via `FuelRatesRepository` (no live ETL yet). **Distinct from the calculator's `total_fuel_cost_gbp`**, which is the SAP-rating notional cost at RdSAP Table 32 standardised prices (~half the real electricity price) — not what the household pays. Raises on a fuel it has no rate for (e.g. house coal, heat network). ADR-0014.
+The deterministic process that derives a Property's annual energy **bill**, composed into per-end-use sections (heating, hot water, lighting, appliances, cooking, pumps/fans, …) plus a **total**, by pricing **SAP10 Calculation**'s delivered kWh per end use at **current Fuel Rates** — each end use billed at its fuel's rate, rolled up per fuel for **standing charges** (metered fuels only — gas/electricity; oil/LPG/solid have none) minus **SEG** export credit on PV. Implemented by `BillDerivation` in `domain/billing/` (a cross-stage concern — the Baseline stage derives the current bill, the Modelling stage re-runs it on the post-package end-state for post-retrofit bills; deterministic, ADR-0006). Reads Fuel Rates from a committed static snapshot via `FuelRatesRepository` (no live ETL yet). **Distinct from the calculator's `total_fuel_cost_gbp`**, which is the SAP-rating notional cost at RdSAP Table 32 standardised prices (~half the real electricity price) — not what the household pays. Electricity on an **Off-Peak Meter** is billed day/night, each end use split by its own **High-Rate Fraction**. Raises on a fuel it has no rate for (e.g. house coal, heat network). ADR-0014.
 _Avoid_: EPC Energy Derivation (renamed), EpcEnergyDerivationService (no "service" suffix), kWh prediction, baseline kWh, energy estimation
 
+**Off-Peak Meter**:
+A dwelling's dual-rate electricity meter (Economy-7-style) charging a cheaper **night** rate and a dearer **day** rate. It is a property of the **meter, not of any one end use**: *every* electric end use — heating, hot water, lighting, appliances, cooking — is billed on it, each split day/night by its own **High-Rate Fraction**. Derived from the RdSAP `meter_type` (or, in synthesis, from the heating SAP code per the heating-system cluster). The calculator already prices the whole meter this way; **Bill Derivation** mirrors it. Pricing an off-peak dwelling's electricity at the single standard rate is wrong in *both* directions — it over-bills night-shifted storage heat and under-bills daytime appliances.
+_Avoid_: Economy 7 (the common brand, but the model is tariff-agnostic — 7/10/18/24-hour all map here), off-peak fuel (it is a meter arrangement, not a distinct fuel carrier)
+
+**High-Rate Fraction**:
+The fraction of one electric end use's annual kWh billed at the **day** (high) rate on an **Off-Peak Meter**; the remainder bills at the **night** (low) rate. A **calculator output** per end use (SAP 10.2 Table 12a — storage heating ≈ 0 / all-night, lighting & appliances ≈ 0.8–0.9 / mostly-day), reused by **Bill Derivation** rather than re-derived, so the bill's day/night split never diverges from the rating's. Unregulated loads SAP does not rate (appliances, cooking) inherit the Table-12a `ALL_OTHER_USES` fraction.
+_Avoid_: day/night ratio, peak fraction, split factor
+
 **UCL Correction**:
 The per-band linear correction (Few et al. 2023, _Energy & Buildings_ 288 113024) that aligns EPC-modelled Primary Energy Intensity with metered consumption. Folded into ML training labels at fit time (per ADR-0007) rather than applied at runtime — the trained model emits metered-equivalent PEUI directly, avoiding the discontinuities at EPC band boundaries that arose when the per-band linear correction was applied post-prediction. Calibrated against gas-heated, non-PV homes in England and Wales rated under SAP 2012; the current implementation extrapolates it to all properties (open question §15.14).
 _Avoid_: UCL adjustment, energy correction, metered correction
@@ -261,11 +281,19 @@ The catalogue classification of a retrofit measure (e.g. `solar_pv`, `loft_insul
 _Avoid_: measure (ambiguous), category
 
 **Solar Potential**:
-The typed domain projection over a Property's Google Solar `buildingInsights` response (fetched by **Ingestion**, persisted whole as JSONB, read by Modelling — never re-fetched). It carries the per-**roof-segment** geometry the PV simulation needs — `azimuthDegrees` (→ SAP orientation octant), `pitchDegrees` (→ SAP pitch), per-segment panel counts, panel capacity, and `sunshineQuantiles` (→ overshading) — plus the candidate panel layouts (`solarPanelConfigs`). It is the source of the **PV Overlay**: the solar Recommendation Generator reads the Solar Potential (NOT the EPC's `photovoltaic_arrays`, which is the dwelling's *existing* PV) and builds competing PV Options from it. Distinct from existing/installed PV.
+The typed domain projection over a Property's Google Solar `buildingInsights` response (fetched by **Ingestion**, persisted whole as JSONB, read by Modelling — never re-fetched). It carries the per-**roof-segment** geometry the PV simulation needs — `azimuthDegrees` (→ SAP orientation octant), `pitchDegrees` (→ SAP pitch), per-segment panel counts, **per-segment centre + area** (for the **Dwelling-Roof Cap**), panel capacity, and `sunshineQuantiles` (→ overshading) — plus the candidate panel layouts (`solarPanelConfigs`). It is the source of the **PV Overlay**: the solar Recommendation Generator reads the Solar Potential (NOT the EPC's `photovoltaic_arrays`, which is the dwelling's *existing* PV) and builds competing PV Options from it. Distinct from existing/installed PV.
 _Avoid_: solar config (ambiguous — the API response vs the chosen array set), Google insights (the raw JSON, not the typed projection)
 
+**Footprint Conflation**:
+When Google Solar (typically on medium-quality imagery, and most often for **semi-detached / terraced** homes) reports the **whole building's** roof for a single dwelling — merging the target with its attached neighbour — so the **Solar Potential** describes more roof than the dwelling owns. The failure mode the **Dwelling-Roof Cap** corrects; without it the array is sized to two or more homes.
+_Avoid_: duplicate surfaces (the legacy heuristic's name, not the concept)
+
+**Dwelling-Roof Cap**:
+The bound that limits a **Solar PV Recommendation**'s array to the dwelling's *own* usable roof, defeating **Footprint Conflation** (ADR-0038). A surface-area budget — the EPC's RdSAP §3.8 roof plan area (`roof_area`) ÷ cos(pitch) × a usable fraction (~0.5, one non-north plane) — caps the panel count via `min(budget, 0.70 × maxArrayPanelsCount)`, so it is a **no-op on correctly-matched homes** and only bites under conflation. Segments are ranked by proximity to the **dwelling's** own lat/long (not Google's building centre) to keep the home's roof and drop the neighbour's. Used only for **sizing** — distinct from **Solar PV Eligibility**, which still uses the real Solar Potential, never a roof-area-from-floor-area estimate.
+_Avoid_: roof-area haircut (overloads the MCS coverage haircut), double-property halving (the legacy semi-detached-only mechanism this replaces)
+
 **Solar PV Recommendation**:
-The single "Solar PV" **Recommendation** for a Property, carrying competing whole-array **Measure Options** built from the **Solar Potential** (ADR-0026). Up to **five conservatively-sized array configs** (capped, ranked by energy generation as the size-suitability proxy; deliberately *not* full-roof — imagery can miss obstructions, so a coverage/edge-setback haircut applies per MCS), each offered **with and without a battery** (≤ 10 Options). Each Option is priced at a **single price point** from the rate sheet (kWp band + scaffolding-by-elevation + optional battery/diverter) — no multi-product price variants. The PV Overlay sets `photovoltaic_arrays` (one per segment: peak_power, orientation, pitch, overshading) plus `pv_diverter_present`, `pv_connection`, and **`is_dwelling_export_capable=True`** (an export meter is ensured post-install); the battery variant adds `pv_batteries`.
+The single "Solar PV" **Recommendation** for a Property, carrying competing whole-array **Measure Options** built from the **Solar Potential** (ADR-0026). Up to **five conservatively-sized array configs** (capped, ranked by energy generation as the size-suitability proxy; deliberately *not* full-roof — imagery can miss obstructions, so a coverage/edge-setback haircut applies per MCS, and the **Dwelling-Roof Cap** bounds the array to the dwelling's own roof under **Footprint Conflation**), each offered **with and without a battery** (≤ 10 Options). Each Option is priced at a **single price point** from the rate sheet (kWp band + scaffolding-by-elevation + optional battery/diverter) — no multi-product price variants. The PV Overlay sets `photovoltaic_arrays` (one per segment: peak_power, orientation, pitch, overshading) plus `pv_diverter_present`, `pv_connection`, and **`is_dwelling_export_capable=True`** (an export meter is ensured post-install); the battery variant adds `pv_batteries`.
 _Avoid_: solar bundle (PV is competing sized Options, not one fixed bundle like ASHP)
 
 **Solar PV Eligibility**:
@@ -292,6 +320,10 @@ _Avoid_: "windows" as a Measure (name **double glazing** / **secondary glazing**
 The rule fixing the single lighting Measure the **Lighting** Recommendation offers. We convert **all non-LED bulbs** (incandescent + CFL + low-energy-unknown) to **LED** — all the way to LED, not the legacy "fill to low energy", because SAP rates LED efficacy above CFL (ADR-0023). One Measure, no planning gate (lighting isn't planning-restricted). Offered only when the dwelling lodges at least one non-LED bulb; a dwelling already all-LED, or one that lodged **no** bulb counts (nothing to size against), gets no Recommendation. Unlike the fabric measures it is a **whole-dwelling** Measure — its **Simulation Overlay** writes the four top-level bulb counts directly (`led = total`, others 0), the first overlay surface that isn't a building part / window / system sub-object. Priced at a flat **average price per bulb** × the count of non-LED bulbs replaced. A free Optimiser candidate (it *improves* SAP), contrast the forced ventilation **Measure Dependency**.
 _Avoid_: "low energy lighting" as the upgrade target (we go to **LED**); treating it as a forced dependency (it is a free candidate); pricing by floor area (it's per-bulb count × average)
 
+**Coherent Heating System**:
+The set of fields that together describe a dwelling's heating and must move **as a unit** whenever a heating system is *synthesised* — never a subset. It spans the main heating cluster (fuel, system, emitter, **controls**), the **hot water** it implies, the **off-peak meter** an electric-storage or CPSU system runs on, and the hot-water cylinder / solar flags. The fields are physically scattered across the `EpcPropertyData` (a gov-API schema mirror — the cluster, a loose meter on the energy source, loose top-level flags), but the *concept* is one system. Three places synthesise a heating system and each must cover the whole set with its own **fill policy**: a **Measure Option** sets a designed end-state (HHRSH / ASHP), a **Landlord Override** *completes* the partial system the landlord named with conservative assumptions for the fields they couldn't tell us (the off-peak meter is *derived* from the SAP code; an unobserved storage charge control defaults to the conservative manual one), and **EPC Prediction**'s heating donor *carries* a real neighbour's whole coherent set. Setting only a subset mis-scores: an electric-storage system left on a single-rate meter bills its overnight heat at the peak rate and collapses the SAP. **Coherence is owned by the synthesiser, never by the calculator** — a real lodged cert is scored exactly as lodged (ADR-0035).
+_Avoid_: heating overlay (that is one *mechanism*, not the field set); meter as a separate concern (it is part of the system); normalising a lodged EPC (coherence is a synthesis-time obligation only)
+
 **Heating Eligibility**:
 The rule fixing which **Measure Options** the single **Heating & Hot Water** Recommendation offers (ADR-0024, expanded). The competing Options are **mutually-exclusive** (the Optimiser picks at most one) and fall in two families: **whole-system replacements** — `high_heat_retention_storage_heaters`, `air_source_heat_pump` — which change main heating + **controls + fuel + meter + the implied hot water** at once (never a separate HW measure; the legacy heating-vs-HW split double-counted); and, for a dwelling keeping a serviceable wet boiler, **partial upgrades** — `gas_boiler_upgrade` (a like-for-like condensing **gas** boiler: gas→gas, or non-gas→gas only where mains gas is present; combi or regular-plus-cylinder, shaped by the dwelling) and the **system tune-up** (keep the boiler; install better **controls** + fix the **cylinder**), the tune-up offered at two competing control levels: `system_tune_up` (standard, SAP code 2106) and `system_tune_up_zoned` (time-and-temperature zone control, 2110 — more SAP uplift, more cost). Each Option is a **fixed, real, contractor-installable end-state** (ASHP via a fixed PCDB heat-pump index; HHR storage via `sap_main_heating_code=409`; the gas boiler via Table 4b code 102/104; controls via 2106/2110), not a derived ideal; **Product** stays cost-only, but a partial/bundle cost is **composed per dwelling** from the components the overlay installs (ADR-0025/0027), not a flat scalar. Eligibility encodes **only physical/planning installability** — the **Optimiser owns the economics**, so it must not re-gate on cost proxies: **ASHP** → houses/bungalows that are not **listed**/**heritage** and not already a heat pump (flats excluded — individual siting needs a survey; a **conservation area** still gets the offer, unlike glazing); **HHR storage** → off-gas or currently-electric dwellings, not community-heated or already HHR; **boiler upgrade / tune-up** → an existing (non-electric) wet boiler, the gas end-state gated on a mains-gas connection, a partial control upgrade offered only when it genuinely improves the existing control (never a downgrade or no-op). Floor area, fabric, fuel, and built form are **not** gates (the legacy ASHP built-form / 120 m² rule is dropped — no authoritative basis). A free Optimiser candidate, not a forced **Measure Dependency**.
 _Avoid_: separate "heating" and "hot water" recommendations (HW folds into each Option); gating ASHP on floor area / built form / fabric (eligibility is physical/planning only — the Optimiser decides cost-effectiveness); treating the whole-system replacements and the partial boiler/tune-up upgrades as **separate** Recommendations (they are mutually-exclusive Options within the one heating rec — separate recs would let the Optimiser co-select and double-charge); a standalone hot-water-only or controls-only Recommendation (controls + cylinder fold into the boiler/tune-up Option)

applications/landlord_description_overrides/handler.py

@@ -7,11 +7,16 @@ import boto3
 from applications.landlord_description_overrides.landlord_description_overrides_trigger_body import (
     LandlordDescriptionOverridesTriggerBody,
 )
-from domain.epc.built_form_type import BuiltFormType
-from domain.epc.property_type import PropertyType
-from domain.epc.roof_type import RoofType
-from domain.epc.wall_type import WallType
-from domain.epc.wall_type_construction_dates import (
+from domain.epc.property_overrides.built_form_type import BuiltFormType
+from domain.epc.property_overrides.construction_age_band import ConstructionAgeBand
+from domain.epc.property_overrides.glazing_type import GlazingType
+from domain.epc.property_overrides.main_fuel_type import MainFuelType
+from domain.epc.property_overrides.main_heating_system_type import MainHeatingSystemType
+from domain.epc.property_overrides.property_type import PropertyType
+from domain.epc.property_overrides.roof_type import RoofType
+from domain.epc.property_overrides.water_heating_type import WaterHeatingType
+from domain.epc.property_overrides.wall_type import WallType
+from domain.epc.property_overrides.wall_type_construction_dates import (
     wall_type_construction_date_prompt_hint,
 )
 from infrastructure.chatgpt.chatgpt import ChatGPT
@@ -24,6 +29,21 @@ from infrastructure.postgres.engine import commit_scope, make_engine, make_sessi
 from infrastructure.postgres.landlord_built_form_type_override_table import (
     LandlordBuiltFormTypeOverrideRow,
 )
+from infrastructure.postgres.landlord_construction_age_band_override_table import (
+    LandlordConstructionAgeBandOverrideRow,
+)
+from infrastructure.postgres.landlord_glazing_override_table import (
+    LandlordGlazingOverrideRow,
+)
+from infrastructure.postgres.landlord_main_fuel_override_table import (
+    LandlordMainFuelOverrideRow,
+)
+from infrastructure.postgres.landlord_main_heating_system_override_table import (
+    LandlordMainHeatingSystemOverrideRow,
+)
+from infrastructure.postgres.landlord_water_heating_override_table import (
+    LandlordWaterHeatingOverrideRow,
+)
 from infrastructure.postgres.landlord_property_type_override_table import (
     LandlordPropertyTypeOverrideRow,
 )
@@ -102,6 +122,56 @@ def _build_columns(
                 session, LandlordRoofTypeOverrideRow
             ),
         ),
+        "main_fuel": lambda src: ClassifiableColumn(
+            name="main_fuel",
+            source_column=src,
+            classifier=ChatGptColumnClassifier(
+                chat_gpt, MainFuelType, MainFuelType.UNKNOWN
+            ),
+            repo=LandlordOverridesRepository[MainFuelType](
+                session, LandlordMainFuelOverrideRow
+            ),
+        ),
+        "glazing": lambda src: ClassifiableColumn(
+            name="glazing",
+            source_column=src,
+            classifier=ChatGptColumnClassifier(
+                chat_gpt, GlazingType, GlazingType.UNKNOWN
+            ),
+            repo=LandlordOverridesRepository[GlazingType](
+                session, LandlordGlazingOverrideRow
+            ),
+        ),
+        "construction_age_band": lambda src: ClassifiableColumn(
+            name="construction_age_band",
+            source_column=src,
+            classifier=ChatGptColumnClassifier(
+                chat_gpt, ConstructionAgeBand, ConstructionAgeBand.UNKNOWN
+            ),
+            repo=LandlordOverridesRepository[ConstructionAgeBand](
+                session, LandlordConstructionAgeBandOverrideRow
+            ),
+        ),
+        "water_heating": lambda src: ClassifiableColumn(
+            name="water_heating",
+            source_column=src,
+            classifier=ChatGptColumnClassifier(
+                chat_gpt, WaterHeatingType, WaterHeatingType.UNKNOWN
+            ),
+            repo=LandlordOverridesRepository[WaterHeatingType](
+                session, LandlordWaterHeatingOverrideRow
+            ),
+        ),
+        "main_heating_system": lambda src: ClassifiableColumn(
+            name="main_heating_system",
+            source_column=src,
+            classifier=ChatGptColumnClassifier(
+                chat_gpt, MainHeatingSystemType, MainHeatingSystemType.UNKNOWN
+            ),
+            repo=LandlordOverridesRepository[MainHeatingSystemType](
+                session, LandlordMainHeatingSystemOverrideRow
+            ),
+        ),
     }
 
     columns: list[ClassifiableColumn[Any]] = []

applications/modelling_e2e/Dockerfile

@@ -0,0 +1,24 @@
+FROM public.ecr.aws/lambda/python:3.11
+
+WORKDIR /var/task
+
+COPY applications/modelling_e2e/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY datatypes/ datatypes/
+COPY domain/ domain/
+COPY infrastructure/ infrastructure/
+COPY orchestration/ orchestration/
+COPY repositories/ repositories/
+COPY utilities/ utilities/
+COPY harness/ harness/
+
+# harness/console.py imports in-memory fakes from tests/orchestration/ at module
+# load time; the fakes have no pytest dependency and are safe to ship.
+COPY tests/__init__.py tests/__init__.py
+COPY tests/orchestration/__init__.py tests/orchestration/__init__.py
+COPY tests/orchestration/fakes.py tests/orchestration/fakes.py
+
+COPY applications/ applications/
+
+CMD ["applications.modelling_e2e.handler.handler"]

applications/modelling_e2e/errors.py

@@ -0,0 +1,144 @@
+"""Specific, debuggable failures for the modelling_e2e prediction path.
+
+When an EPC-less Property cannot be modelled the per-property failure handler
+records ``str(exc)`` in the child SubTask's outputs (see ``handler.run_subtask``).
+A single generic "not predictable" string conflated three unrelated causes —
+unresolved property_type, an empty same-type cohort, and a degenerate
+prediction — so the output could not tell an operator *which* one fired or which
+data to fix. Each cause is its own exception carrying the Property's identity
+plus the cause-specific context needed to debug it from the output alone.
+
+All subclass ``ValueError`` so the existing ``except Exception`` per-property
+boundary (and any ``except ValueError`` callers) keep catching them.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+
+class PropertyNotModellableError(ValueError):
+    """Base: an EPC-less Property could not be predicted. Subclasses name the
+    specific cause. Carries the Property's identity so the failed SubTask output
+    identifies exactly which Property (and portfolio) to investigate."""
+
+    def __init__(
+        self,
+        *,
+        property_id: int,
+        uprn: int,
+        postcode: str,
+        portfolio_id: int,
+        cause: str,
+    ) -> None:
+        self.property_id = property_id
+        self.uprn = uprn
+        self.postcode = postcode
+        self.portfolio_id = portfolio_id
+        self.cause = cause
+        super().__init__(
+            f"cannot model property_id={property_id} uprn={uprn} "
+            f"postcode={postcode!r} portfolio_id={portfolio_id}: {cause}"
+        )
+
+
+class UnresolvedPropertyTypeError(PropertyNotModellableError):
+    """No lodged EPC and the Property's ``property_type`` could not be resolved,
+    so the hard same-type cohort filter cannot fire. Almost always a missing or
+    contradictory Landlord Override (e.g. a House/Bungalow lodged with an
+    "another dwelling above" roof is skipped by the override build), or a
+    ``property_type`` value with no gov-EPC code."""
+
+    def __init__(
+        self,
+        *,
+        property_id: int,
+        uprn: int,
+        postcode: str,
+        portfolio_id: int,
+        property_type: Optional[str],
+        built_form: Optional[str],
+    ) -> None:
+        self.property_type = property_type
+        self.built_form = built_form
+        super().__init__(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            cause=(
+                "no lodged EPC and property_type could not be resolved "
+                f"(property_type={property_type!r}, built_form={built_form!r}), "
+                "so no same-type cohort can be selected. Usually a missing or "
+                "contradictory Landlord Override (e.g. a House/Bungalow with an "
+                "'another dwelling above' roof is skipped by the override build). "
+                "Resolve this Property's property_type override to model it."
+            ),
+        )
+
+
+class NoSameTypeComparablesError(PropertyNotModellableError):
+    """``property_type`` resolved, but no same-type comparable was found in the
+    Property's own postcode or — after broadening — the nearby-postcode cohort,
+    so it cannot be sized from a mixed-type cohort (ADR-0031)."""
+
+    def __init__(
+        self,
+        *,
+        property_id: int,
+        uprn: int,
+        postcode: str,
+        portfolio_id: int,
+        property_type: str,
+        broadened: bool,
+    ) -> None:
+        self.property_type = property_type
+        self.broadened = broadened
+        scope = (
+            "its own postcode or the broadened nearby-postcode cohort"
+            if broadened
+            else "its own postcode"
+        )
+        super().__init__(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            cause=(
+                f"no lodged EPC; property_type={property_type!r} resolved but no "
+                f"same-type comparable was found in {scope}. Cannot size the "
+                "Property from a mixed-type cohort."
+            ),
+        )
+
+
+class DegeneratePredictionError(PropertyNotModellableError):
+    """A same-type cohort was found, but the synthesised EPC carried no MAIN
+    building part — every comparable seeding the structure was lodged with a
+    null/unrecognised part identifier — so the SAP calculation has no main
+    dwelling to anchor on."""
+
+    def __init__(
+        self,
+        *,
+        property_id: int,
+        uprn: int,
+        postcode: str,
+        portfolio_id: int,
+        property_type: str,
+        cohort_size: int,
+    ) -> None:
+        self.property_type = property_type
+        self.cohort_size = cohort_size
+        super().__init__(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            cause=(
+                f"no lodged EPC; predicted from a {cohort_size}-member "
+                f"property_type={property_type!r} cohort, but the synthesised EPC "
+                "had no MAIN building part (the structural template was lodged "
+                "with a null part identifier). Cannot anchor the SAP calculation."
+            ),
+        )

applications/modelling_e2e/handler.py

@@ -0,0 +1,722 @@
+"""SQS-triggered Lambda: fetch EPC (or predict) → run modelling → persist plan.
+
+One SQS message = one batch of properties sharing a portfolio, scenario, and
+(by caller convention) postcode. The handler reads ``property_ids``,
+``portfolio_id``, ``scenario_id``, ``no_solar``, and ``dry_run`` from the
+message body, fetches or predicts each property's EPC, runs the full modelling
+pipeline (SAP10 → optimiser) via ``harness.console.run_modelling``, buffers each
+resulting Plan in memory, and persists the whole batch via ``PostgresUnitOfWork``
+in one atomic transaction at the end.
+
+When no lodged EPC is found, EPC Prediction (Path 3, ADR-0031) synthesises one
+from the postcode cohort. ``_cohort_cache`` is module-level so warm Lambda
+containers re-processing the same postcode avoid redundant fetches.
+
+All Measure Types are considered: pricing goes through
+``catalogue_snapshot_with_off_catalogue_overrides`` so the measures the live
+``material`` catalogue cannot supply (``secondary_heating_removal``, the glazing
+and heating gaps) are priced from the committed off-catalogue overlay instead of
+crashing.
+
+The DB engine is module-scoped (ADR-0012). Architecturally each invocation uses
+one DB connection at a time: the handler reads everything up front — overrides,
+Scenario, a catalogue snapshot, and stored Solar — through one short-lived read
+Session, closes it, models the batch (buffering each Plan in memory), then
+persists the whole batch in one end-of-batch Unit of Work whose overrides resolve
+on its own session, so no two Sessions ever overlap. The engine uses ``NullPool``
+rather than a fixed pool so that target is a graceful ceiling, not a hard one: a
+fresh connection is opened per checkout and closed on return, so there is no pool
+slot to exhaust — any future accidental overlap opens a transient second
+connection instead of dead-locking the Lambda.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import io
+import os
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from typing import Any, Optional, cast
+
+import boto3
+import pandas as pd  # pyright: ignore[reportMissingTypeStubs]
+from sqlalchemy import Engine, text
+from sqlalchemy.pool import NullPool
+from sqlmodel import Session
+
+from datatypes.epc.domain.epc_property_data import (
+    BuildingPartIdentifier,
+    EpcPropertyData,
+)
+from domain.epc_prediction.comparable_properties import (
+    ComparableProperty,
+    select_comparables,
+)
+from domain.epc_prediction.epc_prediction import EpcPrediction
+from domain.epc_prediction.prediction_target import (
+    PredictionTarget,
+    build_prediction_target,
+)
+from domain.geospatial.coordinates import Coordinates
+from domain.geospatial.planning_restrictions import PlanningRestrictions
+from domain.geospatial.spatial_reference import SpatialReference
+from domain.modelling.plan import Plan
+from domain.property.property import Property, PropertyIdentity
+from domain.property_baseline.calculator_rebaseliner import CalculatorRebaseliner
+from domain.sap10_calculator.calculator import Sap10Calculator
+from domain.tasks.subtasks import SubTask
+from domain.tasks.tasks import Source
+from harness.console import run_modelling
+from orchestration.task_orchestrator import TaskOrchestrator
+from orchestration.property_baseline_orchestrator import (
+    PropertyBaselineOrchestrator,
+)
+from infrastructure.epc_client.epc_client_service import EpcClientService
+from infrastructure.postcodes_io.postcodes_io_client import PostcodesIoClient
+from infrastructure.postgres.config import PostgresConfig
+from infrastructure.postgres.engine import make_engine
+from infrastructure.solar.google_solar_api_client import (
+    BuildingInsightsNotFoundError,
+    GoogleSolarApiClient,
+)
+from applications.modelling_e2e.errors import (
+    DegeneratePredictionError,
+    NoSameTypeComparablesError,
+    UnresolvedPropertyTypeError,
+)
+from applications.modelling_e2e.modelling_e2e_trigger_body import (
+    ModellingE2ETriggerBody,
+)
+from repositories.comparable_properties.epc_comparable_properties_repository import (
+    EpcComparablePropertiesRepository,
+    SkippedCohortCert,
+)
+from repositories.epc.epc_postgres_repository import EpcPostgresRepository, EpcSaveRequest
+from repositories.plan.plan_repository import PlanSaveRequest
+from repositories.geospatial.geospatial_s3_repository import (
+    GeospatialS3Repository,
+    ParquetReader,
+)
+from repositories.fuel_rates.fuel_rates_static_file_repository import (
+    FuelRatesStaticFileRepository,
+)
+from repositories.postgres_unit_of_work import PostgresUnitOfWork
+from repositories.product.composite_product_repository import (
+    catalogue_snapshot_with_off_catalogue_overrides,
+)
+from repositories.property.in_memory_property_overrides_reader import (
+    InMemoryPropertyOverridesReader,
+)
+from repositories.property.landlord_override_overlays import overlays_from
+from repositories.property.override_backed_prediction_attributes_reader import (
+    OverrideBackedPredictionAttributesReader,
+)
+from repositories.property.property_overrides_postgres_reader import (
+    PropertyOverridesPostgresReader,
+)
+from repositories.property.property_overrides_reader import (
+    ResolvedPropertyOverrides,
+)
+from repositories.scenario.scenario_postgres_repository import (
+    ScenarioPostgresRepository,
+)
+from repositories.solar.solar_postgres_repository import SolarPostgresRepository
+from repositories.tasks.subtask_postgres_repository import (
+    SubTaskPostgresRepository,
+)
+from repositories.tasks.task_postgres_repository import TaskPostgresRepository
+from utilities.aws_lambda.task_handler import task_handler
+from uuid import UUID
+from utilities.logger import setup_logger
+
+_engine: Optional[Engine] = None
+_cohort_cache: dict[str, list[ComparableProperty]] = {}
+# Broadened (nearby-postcode) cohorts, keyed by (seed postcode, target property
+# type): the early-stop walk depends on the type it is filling for, so two types
+# in the same postcode must not share a cached result.
+_nearby_cohort_cache: dict[tuple[str, str], list[ComparableProperty]] = {}
+
+logger = setup_logger()
+
+
+@dataclasses.dataclass(frozen=True)
+class _SolarWrite:
+    """A freshly-fetched Solar insight queued for persistence. Only set when the
+    insight was fetched this run — stored insights are never re-written."""
+
+    uprn: int
+    longitude: float
+    latitude: float
+    insights: dict[str, Any]
+
+
+@dataclasses.dataclass(frozen=True)
+class _PropertyWrite:
+    """One modelled Property's full persistence intent, accumulated in memory
+    during the compute loop and replayed in a single end-of-batch Unit of Work.
+
+    Buffering the writes (rather than committing per property) keeps the single
+    pooled connection idle through the CPU-bound modelling loop, then collapses
+    the whole batch into one transaction — far fewer statements for RDS to parse,
+    plan, and commit, which is the RDS-CPU bottleneck this targets (ADR-0012)."""
+
+    property_id: int
+    uprn: int
+    portfolio_id: int
+    scenario_id: int
+    is_default: bool
+    lodged_epc: Optional[EpcPropertyData]
+    lodged_epc_is_new: bool
+    predicted_epc: Optional[EpcPropertyData]
+    predicted_epc_is_new: bool
+    spatial: Optional[SpatialReference]
+    solar: Optional[_SolarWrite]
+    plan: Plan
+    has_recommendations: bool
+
+
+def _flush_writes(engine: Engine, writes: list[_PropertyWrite]) -> None:
+    """Persist a whole batch of modelled Properties in one Unit of Work.
+
+    EPC writes are batched by source (lodged group first, predicted group second)
+    so each source emits one DELETE pass + one INSERT pass regardless of batch
+    size, rather than N×per-property round-trips (ADR-0012). All other writes
+    (spatial, solar, plan, mark-modelled) remain per-property inside the same
+    transaction. All-or-nothing per batch: a failed save rolls the whole
+    transaction back so the SQS message is retried — every save is an idempotent
+    upsert. Per-property failures are isolated earlier, in the modelling loop,
+    before a write is ever queued."""
+    lodged_requests = [
+        EpcSaveRequest(w.lodged_epc, property_id=w.property_id, portfolio_id=w.portfolio_id, source="lodged")
+        for w in writes
+        if w.lodged_epc is not None and w.lodged_epc_is_new
+    ]
+    predicted_requests = [
+        EpcSaveRequest(w.predicted_epc, property_id=w.property_id, portfolio_id=w.portfolio_id, source="predicted")
+        for w in writes
+        if w.predicted_epc is not None and w.predicted_epc_is_new
+    ]
+    with PostgresUnitOfWork(lambda: Session(engine)) as uow:
+        if lodged_requests:
+            uow.epc.save_batch(lodged_requests)
+        if predicted_requests:
+            uow.epc.save_batch(predicted_requests)
+        plan_requests = [
+            PlanSaveRequest(
+                w.plan,
+                property_id=w.property_id,
+                scenario_id=w.scenario_id,
+                portfolio_id=w.portfolio_id,
+                is_default=w.is_default,
+            )
+            for w in writes
+        ]
+        uow.plan.save_batch(plan_requests)
+        for w in writes:
+            if w.spatial is not None:
+                uow.spatial.save(w.uprn, w.spatial)
+            if w.solar is not None:
+                uow.solar.save(
+                    w.solar.uprn,
+                    longitude=w.solar.longitude,
+                    latitude=w.solar.latitude,
+                    insights=w.solar.insights,
+                )
+            uow.property.mark_modelled(
+                w.property_id, has_recommendations=w.has_recommendations
+            )
+        uow.commit()
+
+
+def _get_engine() -> Engine:
+    global _engine
+    if _engine is None:
+        config = PostgresConfig.from_env(dict(os.environ))
+        # Architecturally one connection per invocation: the handler reads
+        # everything up front through one short-lived read Session, closes it,
+        # then writes each Property in a sequential Unit of Work — and the Unit of
+        # Work resolves overrides on its own session — so no two Sessions overlap
+        # and a single connection suffices. 12 concurrent containers × 1 = 12
+        # against RDS.
+        #
+        # NullPool, not a fixed pool, enforces that as a *graceful* ceiling rather
+        # than a hard one: each checkout opens a fresh connection and closes it on
+        # return, so there is no pool slot to exhaust. If a future code path ever
+        # holds two Sessions at once it opens a second connection for that instant
+        # instead of dead-locking on a 1-slot pool and failing the whole
+        # invocation (the "QueuePool limit of size 1 overflow 0 reached" timeout).
+        # The design target stays one connection; NullPool just keeps the Lambda
+        # running if we ever regress it.
+        _engine = make_engine(config, poolclass=NullPool)
+    return _engine
+
+
+@contextmanager
+def _shared_engine_orchestrator() -> Generator[TaskOrchestrator, None, None]:
+    """A ``TaskOrchestrator`` on the same module-scoped engine as the modelling
+    work, not a separate one.
+
+    Its repositories commit on every ``save``/``create``, releasing the
+    connection between bookkeeping calls, so it holds none while the wrapped
+    handler body runs. Combined with the read-then-write handler structure, the
+    whole invocation uses one DB connection at a time."""
+    engine = _get_engine()
+    with Session(engine) as session:
+        yield TaskOrchestrator(
+            task_repo=TaskPostgresRepository(session=session),
+            subtask_repo=SubTaskPostgresRepository(session=session),
+        )
+
+
+def _s3_parquet_reader() -> ParquetReader:
+    bucket = os.environ["DATA_BUCKET"]
+
+    def read(key: str) -> pd.DataFrame:
+        s3: Any = cast(
+            Any, boto3.client("s3")
+        )  # pyright: ignore[reportUnknownMemberType]
+        raw = cast(bytes, s3.get_object(Bucket=bucket, Key=key)["Body"].read())
+        return pd.read_parquet(io.BytesIO(raw))  # type: ignore[return-value]
+
+    return read
+
+
+def _spatial_for(
+    geospatial: GeospatialS3Repository, uprn: int
+) -> Optional[SpatialReference]:
+    try:
+        return geospatial.spatial_for(uprn)
+    except Exception:  # noqa: BLE001
+        return None
+
+
+# The 32-wide fallback gap between this container's Solar calls: 0.8 (safety
+# headroom) × 600 QPM ÷ 60 ÷ 32 containers ≈ one call every 4s. Used when the
+# env var is unset so the Lambda self-protects even if terraform wiring is missed.
+_DEFAULT_SOLAR_MIN_REQUEST_INTERVAL_SECONDS: float = 4.0
+
+
+def _solar_min_request_interval_seconds() -> float:
+    """Per-container minimum gap (seconds) between Google Solar API calls, read
+    from ``SOLAR_MIN_REQUEST_INTERVAL_SECONDS``. Terraform derives the value from
+    the queue's ``maximum_concurrency`` (0.8 × 10 QPS ÷ N) so the up-to-32-wide
+    fleet stays under the hard 600 QPM Solar ceiling. Falls back to the 32-wide
+    default when unset or unparseable."""
+    raw = os.environ.get("SOLAR_MIN_REQUEST_INTERVAL_SECONDS")
+    if raw is None:
+        return _DEFAULT_SOLAR_MIN_REQUEST_INTERVAL_SECONDS
+    try:
+        return float(raw)
+    except ValueError:
+        return _DEFAULT_SOLAR_MIN_REQUEST_INTERVAL_SECONDS
+
+
+def _solar_insights_for(
+    solar_client: GoogleSolarApiClient, spatial: Optional[SpatialReference]
+) -> Optional[dict[str, Any]]:
+    if spatial is None or spatial.coordinates is None:
+        return None
+    try:
+        return solar_client.get_building_insights(
+            spatial.coordinates.longitude, spatial.coordinates.latitude
+        )
+    except BuildingInsightsNotFoundError:
+        return None
+
+
+def _dedupe_skipped(
+    skipped: list[SkippedCohortCert],
+) -> list[SkippedCohortCert]:
+    """First occurrence of each skipped cert number (the same cert can appear in
+    more than one postcode cohort across a batch)."""
+    seen: set[str] = set()
+    unique: list[SkippedCohortCert] = []
+    for cert in skipped:
+        if cert.certificate_number not in seen:
+            seen.add(cert.certificate_number)
+            unique.append(cert)
+    return unique
+
+
+def _predict_epc(
+    *,
+    property_id: int,
+    uprn: int,
+    postcode: str,
+    portfolio_id: int,
+    attributes_reader: OverrideBackedPredictionAttributesReader,
+    coordinates: Optional[Coordinates],
+    cohort_for: Callable[[str], list[ComparableProperty]],
+    broaden: Callable[[PredictionTarget], list[ComparableProperty]],
+    predictor: EpcPrediction,
+) -> EpcPropertyData:
+    """Synthesise an EpcPropertyData for an EPC-less property from its postcode
+    cohort (EPC Prediction Path 3, ADR-0031).
+
+    When the property's own postcode holds no same-type comparables (a sparse
+    postcode — e.g. the only flat among houses), the cohort is broadened to the
+    real unit postcodes physically nearest it (``broaden``) before giving up.
+
+    Raises a specific ``PropertyNotModellableError`` subclass — naming the cause
+    and carrying the property's identity — when it cannot predict: property_type
+    unresolved, an empty same-type cohort, or a degenerate (no MAIN part)
+    prediction. The per-property handler records ``str(exc)`` in the SubTask
+    output, so the cause is debuggable from the output alone.
+    """
+    attributes = attributes_reader.attributes_for(property_id)
+    identity = PropertyIdentity(
+        portfolio_id=portfolio_id, postcode=postcode, address="", uprn=uprn
+    )
+    target = build_prediction_target(identity, coordinates, attributes)
+    if target is None:
+        raise UnresolvedPropertyTypeError(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            property_type=attributes.property_type,
+            built_form=attributes.built_form,
+        )
+    comparables = select_comparables(target, cohort_for(target.postcode))
+    broadened = False
+    if not comparables.members:
+        broadened = True
+        comparables = select_comparables(target, broaden(target))
+    if not comparables.members:
+        raise NoSameTypeComparablesError(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            property_type=target.property_type,
+            broadened=broadened,
+        )
+    predicted = predictor.predict(target, comparables)
+    if not any(
+        part.identifier is BuildingPartIdentifier.MAIN
+        for part in predicted.sap_building_parts
+    ):
+        raise DegeneratePredictionError(
+            property_id=property_id,
+            uprn=uprn,
+            postcode=postcode,
+            portfolio_id=portfolio_id,
+            property_type=target.property_type,
+            cohort_size=len(comparables.members),
+        )
+    return predicted
+
+
+@task_handler(
+    task_source="modelling_e2e",
+    source=Source.PROPERTY,
+    orchestrator_cm=_shared_engine_orchestrator,
+    pass_task_orchestrator=True,
+)
+def handler(
+    body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator, task_id: UUID
+) -> None:
+    trigger = ModellingE2ETriggerBody.model_validate(body)
+    property_ids = trigger.property_ids
+    portfolio_id = trigger.portfolio_id
+    scenario_id = trigger.scenario_id
+    refetch_solar = trigger.refetch_solar
+    refetch_epc = trigger.refetch_epc
+    repredict_epc = trigger.repredict_epc
+    dry_run = trigger.dry_run
+
+    logger.info(
+        f"start property_ids={property_ids} portfolio={portfolio_id} "
+        f"scenario={scenario_id} refetch_solar={refetch_solar} "
+        f"refetch_epc={refetch_epc} repredict_epc={repredict_epc} dry_run={dry_run}"
+    )
+
+    engine = _get_engine()
+    epc_client = EpcClientService(os.environ["OPEN_EPC_API_TOKEN"])
+    geospatial = GeospatialS3Repository(_s3_parquet_reader())
+    solar_client = GoogleSolarApiClient(
+        os.environ["GOOGLE_SOLAR_API_KEY"],
+        min_request_interval_seconds=_solar_min_request_interval_seconds(),
+    )
+
+    with engine.connect() as conn:
+        property_rows = conn.execute(
+            text("SELECT id, uprn, postcode FROM property WHERE id = ANY(:ids)"),
+            {"ids": property_ids},
+        ).fetchall()
+
+    uprns: dict[int, int] = {int(row[0]): int(row[1]) for row in property_rows}
+    postcodes: dict[int, str] = {int(row[0]): (row[2] or "") for row in property_rows}
+
+    # Pre-fetch every Property's overrides up front in one query (one short read
+    # Session, opened and closed before the write loop) and serve them from memory
+    # through the loop, so no override read Session is held open alongside a write
+    # Unit of Work.
+    overrides_postgres_reader = PropertyOverridesPostgresReader(lambda: Session(engine))
+    overrides_by_pid: dict[int, ResolvedPropertyOverrides] = (
+        overrides_postgres_reader.overrides_for_many(property_ids)
+    )
+    overrides_reader = InMemoryPropertyOverridesReader(overrides_by_pid)
+    prediction_attrs_reader = OverrideBackedPredictionAttributesReader(overrides_reader)
+    comparables_repo = EpcComparablePropertiesRepository(
+        epc_client, geospatial, nearby_postcodes=PostcodesIoClient()
+    )
+    predictor = EpcPrediction()
+
+    def _get_cohort(postcode: str) -> list[ComparableProperty]:
+        if postcode not in _cohort_cache:
+            _cohort_cache[postcode] = (
+                comparables_repo.candidates_for(postcode) if postcode else []
+            )
+        return _cohort_cache[postcode]
+
+    def _broaden(target: PredictionTarget) -> list[ComparableProperty]:
+        """The nearby-postcode cohort for a gated-out target — the real unit
+        postcodes nearest it, walked until enough same-type comparables surface
+        (ADR-0034). Memoised per (postcode, property_type) so co-located
+        same-type misses share one walk."""
+        key = (target.postcode, target.property_type)
+        if key not in _nearby_cohort_cache:
+            _nearby_cohort_cache[key] = (
+                comparables_repo.candidates_near(
+                    target.postcode,
+                    target.coordinates,
+                    enough=lambda c: c.epc.property_type == target.property_type,
+                )
+                if target.postcode
+                else []
+            )
+        return _nearby_cohort_cache[key]
+
+    # Re-establishes every written Property's Baseline Performance from the just-
+    # persisted EPCs. Run once for the whole batch after the write flush — the
+    # orchestrator already does the batch in one UoW (ADR-0012) — rather than once
+    # per property, so the batch costs one baseline transaction, not N.
+    baseline_orchestrator = PropertyBaselineOrchestrator(
+        unit_of_work=lambda: PostgresUnitOfWork(lambda: Session(engine)),
+        rebaseliner=CalculatorRebaseliner(Sap10Calculator()),
+        fuel_rates=FuelRatesStaticFileRepository(),
+    )
+
+    read_session = Session(engine)
+    try:
+        # Read everything the modelling loop needs up front: the Scenario, an
+        # in-memory snapshot of the catalogue (priced after the Session closes),
+        # and each UPRN's stored Solar insights. Then close the read Session
+        # immediately so its pooled connection is free for the single end-of-batch
+        # write Unit of Work — no write ever opens a second connection alongside a
+        # held-open read Session. (The ``finally`` is the safety net.)
+        scenario = ScenarioPostgresRepository(read_session).get_many([scenario_id])[0]
+        products = catalogue_snapshot_with_off_catalogue_overrides(read_session)
+        stored_solar: dict[int, Optional[dict[str, Any]]] = (
+            {}
+            if not refetch_solar
+            else SolarPostgresRepository(read_session).get_many(
+                list(set(uprns.values()))
+            )
+        )
+        epc_repo = EpcPostgresRepository(read_session)
+        stored_lodged_epcs: dict[int, EpcPropertyData] = (
+            epc_repo.get_for_properties(property_ids) if not refetch_epc else {}
+        )
+        stored_predicted_epcs: dict[int, EpcPropertyData] = (
+            epc_repo.get_predicted_for_properties(property_ids)
+            if not repredict_epc
+            else {}
+        )
+        read_session.close()
+
+        # Each Property models in its own child SubTask (failures isolated here),
+        # appending its persistence intent to this buffer instead of writing — the
+        # whole batch is flushed in one transaction after the loop.
+        accumulated: list[_PropertyWrite] = []
+
+        def _work(subtask: SubTask) -> None:
+            inputs = subtask.inputs or {}
+            pid = int(inputs["property_id"])
+            uprn = uprns[pid]
+            postcode = postcodes.get(pid, "")
+            logger.info(f"property={pid} uprn={uprn} postcode={postcode!r}")
+
+            spatial = _spatial_for(geospatial, uprn)
+            restrictions = (
+                spatial.restrictions if spatial is not None else PlanningRestrictions()
+            )
+            coordinates: Optional[Coordinates] = (
+                spatial.coordinates if spatial is not None else None
+            )
+
+            stored_lodged = stored_lodged_epcs.get(pid)
+            lodged_epc_is_new = False
+            if refetch_epc:
+                epc: Optional[EpcPropertyData] = epc_client.get_by_uprn(uprn)
+                lodged_epc_is_new = epc is not None
+            elif stored_lodged is not None:
+                logger.info(
+                    f"property={pid} using stored lodged EPC (refetch_epc=False)"
+                )
+                epc = stored_lodged
+            else:
+                epc = (
+                    None  # no stored lodged EPC; prediction path handles this property
+                )
+            overrides = overlays_from(overrides_reader.overrides_for(pid))
+            predicted_epc: Optional[EpcPropertyData] = None
+            predicted_epc_is_new = False
+
+            if epc is not None:
+                logger.info(f"property={pid} lodged EPC found")
+                effective_epc = Property(
+                    identity=PropertyIdentity(
+                        portfolio_id=portfolio_id,
+                        postcode=postcode,
+                        address="",
+                        uprn=uprn,
+                    ),
+                    epc=epc,
+                    landlord_overrides=overrides,
+                ).effective_epc
+            else:
+                logger.info(f"property={pid} no lodged EPC — attempting prediction")
+                stored_predicted = stored_predicted_epcs.get(pid)
+                if not repredict_epc and stored_predicted is not None:
+                    logger.info(
+                        f"property={pid} using stored predicted EPC (repredict_epc=False)"
+                    )
+                    predicted_epc = stored_predicted
+                else:
+                    predicted_epc = _predict_epc(
+                        property_id=pid,
+                        uprn=uprn,
+                        postcode=postcode,
+                        portfolio_id=portfolio_id,
+                        attributes_reader=prediction_attrs_reader,
+                        coordinates=coordinates,
+                        cohort_for=_get_cohort,
+                        broaden=_broaden,
+                        predictor=predictor,
+                    )
+                    predicted_epc_is_new = True
+                effective_epc = Property(
+                    identity=PropertyIdentity(
+                        portfolio_id=portfolio_id,
+                        postcode=postcode,
+                        address="",
+                        uprn=uprn,
+                    ),
+                    epc=None,
+                    predicted_epc=predicted_epc,
+                    landlord_overrides=overrides,
+                ).effective_epc
+
+            solar_insights: Optional[dict[str, Any]]
+            solar_was_fetched = False
+            if not refetch_solar:
+                solar_insights = None
+            else:
+                solar_insights = stored_solar.get(uprn)
+                if solar_insights is None:
+                    solar_insights = _solar_insights_for(solar_client, spatial)
+                    solar_was_fetched = solar_insights is not None
+
+            plan = run_modelling(
+                effective_epc,
+                planning_restrictions=restrictions,
+                solar_insights=solar_insights,
+                considered_measures=None,
+                products=products,
+                scenario=scenario,
+                print_table=False,
+            )
+            logger.info(
+                f"property={pid} modelling complete " f"measures={len(plan.measures)}"
+            )
+
+            if dry_run:
+                measure_types = (
+                    ", ".join(m.measure_type for m in plan.measures) or "none"
+                )
+                logger.info(
+                    f"[dry_run] property={pid} "
+                    f"measures=[{measure_types}] — skipping DB write"
+                )
+                return
+
+            solar_write: Optional[_SolarWrite] = None
+            if (
+                solar_was_fetched
+                and solar_insights is not None
+                and spatial is not None
+                and spatial.coordinates is not None
+            ):
+                solar_write = _SolarWrite(
+                    uprn=uprn,
+                    longitude=spatial.coordinates.longitude,
+                    latitude=spatial.coordinates.latitude,
+                    insights=solar_insights,
+                )
+
+            # Queue this Property's writes rather than committing now — the
+            # whole batch is persisted in one Unit of Work after the loop
+            # (see _flush_writes). The *_is_new flags gate EPC saves so that
+            # EPCs read from DB unchanged are not re-written.
+            accumulated.append(
+                _PropertyWrite(
+                    property_id=pid,
+                    uprn=uprn,
+                    portfolio_id=portfolio_id,
+                    scenario_id=scenario_id,
+                    is_default=scenario.is_default,
+                    lodged_epc=epc,
+                    lodged_epc_is_new=lodged_epc_is_new,
+                    predicted_epc=predicted_epc,
+                    predicted_epc_is_new=predicted_epc_is_new,
+                    spatial=spatial,
+                    solar=solar_write,
+                    plan=plan,
+                    has_recommendations=bool(plan.measures),
+                )
+            )
+            logger.info(f"property={pid} queued for write")
+
+        # Fan the batch out into one child SubTask per property and run them in
+        # a single batched pass: create all children, model each (failures
+        # isolated per child), then persist all their statuses in two writes +
+        # one cascade — not ~5 writes and a full parent re-roll-up per property
+        # (see TaskOrchestrator.run_subtasks).
+        orchestrator.run_subtasks(
+            task_id,
+            [{"property_id": pid} for pid in property_ids],
+            work=_work,
+        )
+
+        # Persist the whole batch in one transaction, then re-establish every
+        # written Property's Baseline (the orchestrator batches its own UoW). The
+        # N per-property write transactions plus N baseline transactions collapse
+        # to two — the RDS-CPU win. Skipped entirely on a dry run or an all-failed
+        # batch, where nothing was queued.
+        if accumulated:
+            _flush_writes(engine, accumulated)
+            baseline_orchestrator.run([w.property_id for w in accumulated])
+            logger.info(
+                f"persisted {len(accumulated)} "
+                f"{'property' if len(accumulated) == 1 else 'properties'} "
+                f"and baselines"
+            )
+
+        # Cohort certs the mapper could not consume were skipped (not aborted on)
+        # so prediction could proceed; surface them — with cert numbers — in the
+        # subtask outputs so the mapper gaps can be closed later.
+        skipped_certs: list[dict[str, str]] = [
+            {"certificate_number": s.certificate_number, "error": s.error}
+            for s in _dedupe_skipped(comparables_repo.skipped)
+        ]
+        if skipped_certs:
+            logger.info(
+                f"skipped {len(skipped_certs)} unmappable cohort cert(s): "
+                f"{[s['certificate_number'] for s in skipped_certs]}"
+            )
+
+    finally:
+        read_session.close()

applications/modelling_e2e/local_handler/.env.local.example

@@ -0,0 +1,36 @@
+# Local-test environment for the modelling_e2e Lambda.
+#
+#   cp .env.local.example .env.local   then fill in the values below.
+#
+# .env.local is gitignored. The container hits REAL AWS and a REAL Postgres,
+# so every value here points at infrastructure that exists.
+#
+# Set dry_run=true in invoke_local_lambda.py to run the full pipeline without
+# writing anything to the DB — safe for local testing.
+#
+# Keep comments on their own lines — docker-compose's env_file parser folds a
+# trailing "# ..." into the value.
+
+# --- Postgres (infrastructure/postgres/config.py -> PostgresConfig.from_env) ---
+# POSTGRES_HOST <- DB_HOST, PORT <- DB_PORT, USERNAME <- DB_USERNAME,
+# PASSWORD <- DB_PASSWORD, DATABASE <- DB_NAME.
+POSTGRES_HOST=
+POSTGRES_PORT=5432
+POSTGRES_USERNAME=
+POSTGRES_PASSWORD=
+POSTGRES_DATABASE=
+# POSTGRES_DRIVER=psycopg2   (optional; defaults to psycopg2)
+
+# --- Handler config (applications/modelling_e2e/handler.py) ---
+# OPEN_EPC_API_TOKEN: gov.uk EPC API token (root .env: OPEN_EPC_API_TOKEN).
+# GOOGLE_SOLAR_API_KEY: Google Solar API key (root .env: GOOGLE_SOLAR_API_KEY).
+# DATA_BUCKET: S3 bucket holding geospatial parquet files (root .env: DATA_BUCKET).
+OPEN_EPC_API_TOKEN=
+GOOGLE_SOLAR_API_KEY=
+DATA_BUCKET=
+
+# --- AWS credentials for boto3 (S3 + EPC client) ---
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=eu-west-2
+# AWS_SESSION_TOKEN=   (only if using temporary/SSO credentials)

applications/modelling_e2e/local_handler/docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  modelling-e2e:
+    build:
+      context: ../../../
+      dockerfile: applications/modelling_e2e/Dockerfile
+    ports:
+      - "9004:8080"
+    env_file:
+      - ../../../.env

applications/modelling_e2e/local_handler/invoke_local_lambda.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import json
+import requests
+
+HOST = "localhost"
+PORT = "9004"
+
+LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations"
+
+payload = {
+    "Records": [
+        {
+            "body": json.dumps(
+                {
+                    "property_ids": [727220, 727229],
+                    "portfolio_id": 796,
+                    "scenario_id": 1268,
+                    "refetch_solar": True,
+                    "dry_run": True,
+                }
+            )
+        }
+    ]
+}
+
+response = requests.post(LAMBDA_URL, json=payload)
+
+print("Status code:", response.status_code)
+print("Response:")
+print(response.text)

applications/modelling_e2e/modelling_e2e_trigger_body.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class ModellingE2ETriggerBody(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    property_ids: list[int]
+    portfolio_id: int
+    scenario_id: int
+    refetch_solar: bool = True
+    refetch_epc: bool = True
+    repredict_epc: bool = True
+    dry_run: bool = False

applications/modelling_e2e/requirements.txt

@@ -0,0 +1,11 @@
+awslambdaric
+boto3
+numpy==2.1.2
+pandas==2.2.2
+pyarrow==17.0.0
+httpx
+requests
+pydantic
+sqlalchemy==2.0.36
+sqlmodel
+psycopg2-binary==2.9.10

backend/epc_api/json_samples/real_life_examples/RdSAP-Schema-20.0.0/uprn_22086693/epc.json

@@ -0,0 +1,438 @@
+{
+  "uprn": 22086693,
+  "roofs": [
+    {
+      "description": {
+        "value": "Pitched, 200 mm loft insulation",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    },
+    {
+      "description": {
+        "value": "Flat, insulated (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 3
+    }
+  ],
+  "walls": [
+    {
+      "description": {
+        "value": "Cavity wall, as built, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 2,
+      "environmental_efficiency_rating": 2
+    },
+    {
+      "description": {
+        "value": "Cavity wall, as built, insulated (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "floors": [
+    {
+      "description": {
+        "value": "Suspended, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    },
+    {
+      "description": {
+        "value": "Solid, no insulation (assumed)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "status": "entered",
+  "tenure": 2,
+  "window": {
+    "description": {
+      "value": "Fully double glazed",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 3,
+    "environmental_efficiency_rating": 3
+  },
+  "addendum": {
+    "addendum_numbers": [
+      8
+    ],
+    "cavity_fill_recommended": "true"
+  },
+  "lighting": {
+    "description": {
+      "value": "Low energy lighting in all fixed outlets",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "postcode": "BN2 9ZN",
+  "hot_water": {
+    "description": {
+      "value": "From main system",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "post_town": "BRIGHTON",
+  "built_form": 2,
+  "created_at": "2022-03-22 14:09:56",
+  "door_count": 1,
+  "glazed_area": 1,
+  "glazing_gap": "16+",
+  "region_code": 14,
+  "report_type": 2,
+  "sap_heating": {
+    "cylinder_size": 1,
+    "water_heating_code": 901,
+    "water_heating_fuel": 26,
+    "instantaneous_wwhrs": {
+      "rooms_with_bath_and_or_shower": 1,
+      "rooms_with_mixer_shower_no_bath": 0,
+      "rooms_with_bath_and_mixer_shower": 0
+    },
+    "secondary_fuel_type": 29,
+    "main_heating_details": [
+      {
+        "has_fghrs": "N",
+        "main_fuel_type": 26,
+        "boiler_flue_type": 2,
+        "fan_flue_present": "Y",
+        "heat_emitter_type": 1,
+        "emitter_temperature": 0,
+        "main_heating_number": 1,
+        "main_heating_control": 2106,
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "sap_main_heating_code": 113,
+        "central_heating_pump_age": 0,
+        "main_heating_data_source": 2
+      }
+    ],
+    "immersion_heating_type": "NA",
+    "secondary_heating_type": 691,
+    "has_fixed_air_conditioning": "false"
+  },
+  "sap_version": 9.94,
+  "schema_type": "RdSAP-Schema-20.0.0",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": {
+        "value": "Boiler and radiators, mains gas",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "dwelling_type": {
+    "value": "Semi-detached house",
+    "language": "1"
+  },
+  "language_code": 1,
+  "property_type": 0,
+  "address_line_1": "30 Hallett Road",
+  "assessment_type": "RdSAP",
+  "completion_date": "2022-03-22",
+  "inspection_date": "2022-03-17",
+  "extensions_count": 1,
+  "measurement_type": 1,
+  "total_floor_area": 80,
+  "transaction_type": 8,
+  "conservatory_type": 1,
+  "heated_room_count": 4,
+  "pvc_window_frames": "true",
+  "registration_date": "2022-03-22",
+  "sap_energy_source": {
+    "mains_gas": "Y",
+    "meter_type": 1,
+    "photovoltaic_supply": [
+      [
+        {
+          "pitch": 2,
+          "peak_power": {
+            "value": 1.14,
+            "quantity": "kW"
+          },
+          "orientation": 5,
+          "overshading": 1,
+          "pv_connection": 2
+        }
+      ],
+      [
+        {
+          "pitch": 2,
+          "peak_power": {
+            "value": 1.14,
+            "quantity": "kW"
+          },
+          "orientation": 7,
+          "overshading": 1,
+          "pv_connection": 2
+        }
+      ]
+    ],
+    "wind_turbines_count": 0,
+    "wind_turbines_terrain_type": 2
+  },
+  "secondary_heating": {
+    "description": {
+      "value": "Room heaters, electric",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "lzc_energy_sources": [
+    11
+  ],
+  "sap_building_parts": [
+    {
+      "identifier": "Main Dwelling",
+      "wall_dry_lined": "N",
+      "wall_thickness": 280,
+      "floor_heat_loss": 7,
+      "roof_construction": 4,
+      "wall_construction": 4,
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "floor_insulation": 1,
+          "total_floor_area": {
+            "value": 36.86,
+            "quantity": "square metres"
+          },
+          "party_wall_length": {
+            "value": 6.8,
+            "quantity": "metres"
+          },
+          "floor_construction": 2,
+          "heat_loss_perimeter": {
+            "value": 13.4,
+            "quantity": "metres"
+          }
+        },
+        {
+          "floor": 1,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "total_floor_area": {
+            "value": 36.86,
+            "quantity": "square metres"
+          },
+          "party_wall_length": {
+            "value": 6.8,
+            "quantity": "metres"
+          },
+          "heat_loss_perimeter": {
+            "value": 17.4,
+            "quantity": "metres"
+          }
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "C",
+      "party_wall_construction": 0,
+      "wall_thickness_measured": "Y",
+      "roof_insulation_location": 2,
+      "roof_insulation_thickness": "200mm",
+      "wall_insulation_thickness": "NI",
+      "floor_insulation_thickness": "NI"
+    },
+    {
+      "identifier": "Extension",
+      "wall_dry_lined": "N",
+      "floor_heat_loss": 7,
+      "roof_construction": 1,
+      "wall_construction": 4,
+      "building_part_number": 2,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": {
+            "value": 2.3,
+            "quantity": "metres"
+          },
+          "floor_insulation": 1,
+          "total_floor_area": {
+            "value": 6,
+            "quantity": "square metres"
+          },
+          "party_wall_length": 0,
+          "floor_construction": 1,
+          "heat_loss_perimeter": {
+            "value": 7,
+            "quantity": "metres"
+          }
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "H",
+      "party_wall_construction": 0,
+      "wall_thickness_measured": "N",
+      "roof_insulation_location": 6,
+      "wall_insulation_thickness": "NI",
+      "floor_insulation_thickness": "NI",
+      "flat_roof_insulation_thickness": "AB"
+    }
+  ],
+  "low_energy_lighting": 100,
+  "solar_water_heating": "N",
+  "habitable_room_count": 4,
+  "heating_cost_current": {
+    "value": 710,
+    "currency": "GBP"
+  },
+  "insulated_door_count": 0,
+  "co2_emissions_current": 2.4,
+  "energy_rating_average": 60,
+  "energy_rating_current": 72,
+  "lighting_cost_current": {
+    "value": 74,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": {
+        "value": "Programmer, room thermostat and TRVs",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "multiple_glazing_type": 3,
+  "open_fireplaces_count": 0,
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 548,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 95,
+    "currency": "GBP"
+  },
+  "mechanical_ventilation": 0,
+  "percent_draughtproofed": 100,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 112,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3500 - \u00a31,500",
+      "improvement_type": "B",
+      "improvement_details": {
+        "improvement_number": 6
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 77,
+      "environmental_impact_rating": 75
+    },
+    {
+      "sequence": 2,
+      "typical_saving": {
+        "value": 49,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3800 - \u00a31,200",
+      "improvement_type": "W1",
+      "improvement_details": {
+        "improvement_number": 57
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 79,
+      "environmental_impact_rating": 78
+    },
+    {
+      "sequence": 3,
+      "typical_saving": {
+        "value": 30,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a34,000 - \u00a36,000",
+      "improvement_type": "N",
+      "improvement_details": {
+        "improvement_number": 19
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 81,
+      "environmental_impact_rating": 80
+    }
+  ],
+  "co2_emissions_potential": 1.5,
+  "energy_rating_potential": 81,
+  "lighting_cost_potential": {
+    "value": 74,
+    "currency": "GBP"
+  },
+  "schema_version_original": "LIG-19.0",
+  "alternative_improvements": [
+    {
+      "improvement": {
+        "sequence": 1,
+        "typical_saving": {
+          "value": 59,
+          "currency": "GBP"
+        },
+        "improvement_type": "Q2",
+        "improvement_details": {
+          "improvement_number": 55
+        },
+        "improvement_category": 6,
+        "energy_performance_rating": 80,
+        "environmental_impact_rating": 78
+      }
+    }
+  ],
+  "hot_water_cost_potential": {
+    "value": 66,
+    "currency": "GBP"
+  },
+  "renewable_heat_incentive": {
+    "water_heating": 2107,
+    "impact_of_cavity_insulation": -1824,
+    "space_heating_existing_dwelling": 9574
+  },
+  "energy_consumption_current": 171,
+  "has_fixed_air_conditioning": "false",
+  "multiple_glazed_proportion": 100,
+  "calculation_software_version": "2.1.0.1",
+  "energy_consumption_potential": 103,
+  "environmental_impact_current": 69,
+  "fixed_lighting_outlets_count": 10,
+  "windows_transmission_details": {
+    "u_value": 2.6,
+    "data_source": 2,
+    "solar_transmittance": 0.76
+  },
+  "current_energy_efficiency_band": "C",
+  "environmental_impact_potential": 80,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "B",
+  "co2_emissions_current_per_floor_area": 30,
+  "low_energy_fixed_lighting_outlets_count": 10
+}

backend/epc_api/json_samples/real_life_examples/SAP-Schema-16.0/uprn_10070004512/epc.json

@@ -0,0 +1,281 @@
+{
+  "uprn": 10070004512,
+  "roofs": [
+    {
+      "description": "(another dwelling above)",
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "walls": [
+    {
+      "description": "Cavity wall, filled cavity",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "floors": [
+    {
+      "description": "To external air, no insulation (assumed)",
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "status": "entered",
+  "windows": [
+    {
+      "description": "Fully double glazed",
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 3
+    }
+  ],
+  "lighting": {
+    "description": "Low energy lighting in all fixed outlets",
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "postcode": "BR1 4QF",
+  "hot_water": {
+    "description": "Electric immersion, off-peak",
+    "energy_efficiency_rating": 3,
+    "environmental_efficiency_rating": 1
+  },
+  "post_town": "BROMLEY",
+  "created_at": "2012-04-20 10:59:22.000000",
+  "door_count": 1,
+  "glazed_area": 1,
+  "region_code": 14,
+  "report_type": 2,
+  "sap_heating": {
+    "wwhrs": {
+      "rooms_with_bath_and_or_shower": 1,
+      "rooms_with_mixer_shower_no_bath": 0,
+      "rooms_with_bath_and_mixer_shower": 0
+    },
+    "cylinder_size": 1,
+    "water_heating_code": 903,
+    "water_heating_fuel": 29,
+    "main_heating_details": [
+      {
+        "has_fghrs": "N",
+        "main_fuel_type": 29,
+        "heat_emitter_type": 0,
+        "main_heating_number": 1,
+        "main_heating_control": 2401,
+        "main_heating_category": 7,
+        "main_heating_fraction": 1,
+        "sap_main_heating_code": 402,
+        "main_heating_data_source": 2
+      }
+    ],
+    "immersion_heating_type": 1,
+    "has_fixed_air_conditioning": "false"
+  },
+  "sap_version": 9.91,
+  "schema_type": "SAP-Schema-16.0",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": "Electric storage heaters",
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 1
+    }
+  ],
+  "dwelling_type": "Ground-floor flat",
+  "language_code": 1,
+  "property_type": 2,
+  "address_line_1": "54a, Boyland Road",
+  "schema_version": "LIG-16.0",
+  "assessment_type": "RdSAP",
+  "completion_date": "2012-04-20",
+  "inspection_date": "2012-04-20",
+  "extensions_count": 0,
+  "measurement_type": 1,
+  "sap_flat_details": {
+    "level": 1,
+    "top_storey": "N",
+    "flat_location": 0,
+    "heat_loss_corridor": 0
+  },
+  "total_floor_area": 33,
+  "transaction_type": 3,
+  "conservatory_type": 1,
+  "heated_room_count": 2,
+  "registration_date": "2012-04-20",
+  "restricted_access": 1,
+  "sap_energy_source": {
+    "main_gas": "N",
+    "meter_type": 1,
+    "photovoltaic_supply": {
+      "percent_roof_area": 0
+    },
+    "wind_turbines_count": 0,
+    "wind_turbines_terrain_type": 1
+  },
+  "secondary_heating": {
+    "description": "Portable electric heaters (assumed)",
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "sap_building_parts": [
+    {
+      "identifier": "Main Dwelling",
+      "wall_dry_lined": "N",
+      "floor_heat_loss": 1,
+      "roof_construction": 3,
+      "wall_construction": 4,
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": 2.42,
+          "floor_insulation": 1,
+          "total_floor_area": 33.24,
+          "floor_construction": 0,
+          "heat_loss_perimeter": 10.39
+        }
+      ],
+      "wall_insulation_type": 2,
+      "construction_age_band": "B",
+      "wall_thickness_measured": "N",
+      "roof_insulation_location": "ND",
+      "roof_insulation_thickness": "ND"
+    }
+  ],
+  "low_energy_lighting": 100,
+  "solar_water_heating": "N",
+  "bedf_revision_number": 321,
+  "habitable_room_count": 2,
+  "heating_cost_current": {
+    "value": 303,
+    "currency": "GBP"
+  },
+  "insulated_door_count": 0,
+  "co2_emissions_current": {
+    "value": 2.9,
+    "quantity": "tonnes per year"
+  },
+  "energy_rating_average": 60,
+  "energy_rating_current": 66,
+  "lighting_cost_current": {
+    "value": 23,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": "Manual charge control",
+      "energy_efficiency_rating": 2,
+      "environmental_efficiency_rating": 2
+    }
+  ],
+  "multiple_glazing_type": 3,
+  "open_fireplaces_count": 0,
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 190,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 106,
+    "currency": "GBP"
+  },
+  "mechanical_ventilation": 0,
+  "percent_draughtproofed": 100,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 86,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3800 - \u00a31,200",
+      "improvement_type": "W",
+      "improvement_details": {
+        "improvement_number": 47
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 73,
+      "environmental_impact_rating": 58
+    },
+    {
+      "sequence": 2,
+      "typical_saving": {
+        "value": 27,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3600 - \u00a3800",
+      "improvement_type": "L",
+      "improvement_details": {
+        "improvement_number": 25
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 75,
+      "environmental_impact_rating": 60
+    }
+  ],
+  "co2_emissions_potential": {
+    "value": 2.1,
+    "quantity": "tonnes per year"
+  },
+  "energy_rating_potential": 75,
+  "lighting_cost_potential": {
+    "value": 23,
+    "currency": "GBP"
+  },
+  "alternative_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 24,
+        "currency": "GBP"
+      },
+      "improvement_type": "J2",
+      "improvement_details": {
+        "improvement_number": 54
+      },
+      "improvement_category": 6,
+      "energy_performance_rating": 76,
+      "environmental_impact_rating": 93
+    },
+    {
+      "sequence": 2,
+      "typical_saving": {
+        "value": 84,
+        "currency": "GBP"
+      },
+      "improvement_type": "Z1",
+      "improvement_details": {
+        "improvement_number": 51
+      },
+      "improvement_category": 6,
+      "energy_performance_rating": 80,
+      "environmental_impact_rating": 81
+    }
+  ],
+  "hot_water_cost_potential": {
+    "value": 106,
+    "currency": "GBP"
+  },
+  "renewable_heat_incentive": {
+    "water_heating": 1434,
+    "space_heating_existing_dwelling": 4064
+  },
+  "seller_commission_report": "Y",
+  "energy_consumption_current": 497,
+  "has_fixed_air_conditioning": "false",
+  "multiple_glazed_proportion": 100,
+  "calculation_software_version": 4.1,
+  "energy_consumption_potential": 365,
+  "environmental_impact_current": 47,
+  "fixed_lighting_outlets_count": 5,
+  "current_energy_efficiency_band": "D",
+  "environmental_impact_potential": 60,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "C",
+  "co2_emissions_current_per_floor_area": {
+    "value": 88,
+    "quantity": "kg/m2 per year"
+  },
+  "low_energy_fixed_lighting_outlets_count": 5
+}

backend/epc_api/json_samples/real_life_examples/SAP-Schema-16.1/uprn_100021943298/elmhurst_inputs.md

@@ -0,0 +1,98 @@
+# Elmhurst RdSAP inputs — UPRN 100021943298 (cert 0498-6963-7270-0822-1980, SAP-Schema-16.1)
+
+**Lodged SAP:** 72   **Our engine:** 76 (continuous ~75.x)   ← compare Elmhurst against the engine
+**Property:** Ground-floor FLAT, age band B (1900–1929), mains-gas combi + radiators, TFA 78 m²
+
+**Schema note:** SAP-Schema-16.1 is reduced-field (RdSAP-shaped) — mapped via the
+new `from_sap_schema_16_1` → `_normalize_sap_schema_16_x` → `from_rdsap_schema_17_1`.
+Descriptions + glazed-area band, no measured U-values, so engine and Elmhurst should
+both use RdSAP age-band/description U-values — expect TIGHTER agreement (the +4 over
+lodged is the reconciliation target).
+
+**Known divergences / gaps to watch:**
+- **Party wall:** 16.1 lodges no `party_wall_length`; normalizer defaults 0 → engine
+  models no party wall. For a ground-floor flat the side/party walls to other dwellings
+  are handled by Elmhurst's Flat position + corridor — keep party-wall length 0 and check
+  the worksheet party-wall line.
+- Age band B → Elmhurst on-screen band **B 1900-1929**.
+
+## Property Description
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Property Type | **Flat** | property_type 2 |
+| Built form | **End-Terrace** | built_form 3 |
+| Age band | **B 1900-1929** | construction_age_band B |
+| Storeys | 1 | ground-floor flat, single storey |
+| Habitable rooms | 3 | habitable_room_count 3 |
+
+## Flats
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Position of flat | **Ground Floor** | dwelling_type "Ground-floor flat" |
+| Floor | 0 | ground = storey 0 |
+| Corridor | None | all-exposed (no unheated corridor lodged) |
+
+## Dimensions
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Floor area (lowest) | 77.69 m² | floor 0 |
+| Room height | 2.65 m | room_height |
+| Heat-loss perimeter | 28.9 m | heat_loss_perimeter |
+| Party-wall length | **0** | 16.1 lodges none |
+
+## Walls
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Construction | **Solid Brick** | wall_construction 3 |
+| Insulation | **Internal** | wall_insulation_type 3 = internal insulation ("Solid brick, with internal insulation") |
+| Thickness | (as built) | wall_thickness_measured True |
+
+## Roofs
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Type | **Another dwelling above** | roof "(another dwelling above)" → no roof heat loss |
+
+## Floors
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Location / type | Ground floor / **Suspended** | "Suspended, no insulation (assumed)" |
+| Insulation | **As built** (none) | no insulation |
+
+## Openings
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Windows | **Double glazed**, total 11.544 m² (4 × 2.886, N/E/S/W) | glazed_area band 1, multiple_glazing_type 2 (double). Enter ONE combined "Double …" row of 11.544 m². |
+| Doors | **2, uninsulated** | door_count 2, insulated_door_count 0 |
+
+## Ventilation & Lighting
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Ventilation | Natural | no MV |
+| Extract fans / flues | 0 / 0 | none lodged |
+| Sheltered sides | 1 | sheltered_sides 1 |
+| Air Pressure Test | **Not available** (no test lodged) | clear any Blower Door |
+| Lighting | **100% low-energy** (6 of 6 outlets) | low_energy 6 / total 6 |
+
+## Space Heating
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Main heating | Mains gas **combi** + radiators | fuel 26, emitter 1, control 2106, fan flue, PCDB 10328, category 2, NO cylinder → BGW condensing combi cascade |
+| Secondary | None | secondary "None" |
+
+## Water Heating
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Water heating | From main (901), gas | water_heating from combi |
+| Hot water cylinder | **None** | has_hot_water_cylinder False (combi/instantaneous) |
+| Solar / WWHRS / FGHRS | None | — |
+
+## Fields to clear in Elmhurst (do NOT map)
+| Elmhurst field | Set to | Why absent |
+|---|---|---|
+| Conservatory | none | not lodged |
+| Air Pressure Test method | **Not available** | no test lodged (clear any prior Blower Door) |
+| Mechanical ventilation | off | natural ventilation |
+| Cylinder size/insulation | (gone — combi) | no cylinder |
+| Main Heating 2 / Secondary | none | single combi system |
+| PV / Wind / Hydro | none | none lodged |
+| Room-in-roof / extensions | blank | single building part, no RiR |

backend/epc_api/json_samples/real_life_examples/SAP-Schema-16.1/uprn_100021943298/epc.json

@@ -0,0 +1,232 @@
+{
+  "uprn": 100021943298,
+  "roofs": [
+    {
+      "description": "(another dwelling above)",
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "walls": [
+    {
+      "description": "Solid brick, with internal insulation",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "floors": [
+    {
+      "description": "Suspended, no insulation (assumed)",
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "status": "entered",
+  "tenure": 2,
+  "windows": [
+    {
+      "description": "Fully double glazed",
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 3
+    }
+  ],
+  "lighting": {
+    "description": "Low energy lighting in all fixed outlets",
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "postcode": "SE4 1YN",
+  "hot_water": {
+    "description": "From main system",
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "post_town": "LONDON",
+  "built_form": 3,
+  "created_at": "2012-10-02 23:16:59.000000",
+  "door_count": 2,
+  "glazed_area": 1,
+  "region_code": 17,
+  "report_type": 2,
+  "sap_heating": {
+    "wwhrs": {
+      "rooms_with_bath_and_or_shower": 1,
+      "rooms_with_mixer_shower_no_bath": 0,
+      "rooms_with_bath_and_mixer_shower": 0
+    },
+    "cylinder_size": 1,
+    "water_heating_code": 901,
+    "water_heating_fuel": 26,
+    "main_heating_details": [
+      {
+        "has_fghrs": "N",
+        "main_fuel_type": 26,
+        "boiler_flue_type": 2,
+        "fan_flue_present": "Y",
+        "heat_emitter_type": 1,
+        "boiler_index_number": 10328,
+        "main_heating_number": 1,
+        "main_heating_control": 2106,
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "main_heating_data_source": 1
+      }
+    ],
+    "has_fixed_air_conditioning": "false"
+  },
+  "sap_version": 9.91,
+  "schema_type": "SAP-Schema-16.1",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": "Boiler and radiators, mains gas",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "dwelling_type": "Ground-floor flat",
+  "language_code": 1,
+  "property_type": 2,
+  "address_line_1": "11a, Chalsey Road",
+  "schema_version": "LIG-16.1",
+  "assessment_type": "RdSAP",
+  "completion_date": "2012-10-02",
+  "inspection_date": "2012-10-01",
+  "extensions_count": 0,
+  "measurement_type": 1,
+  "sap_flat_details": {
+    "level": 1,
+    "top_storey": "N",
+    "flat_location": 0,
+    "heat_loss_corridor": 0
+  },
+  "total_floor_area": 78,
+  "transaction_type": 8,
+  "conservatory_type": 1,
+  "heated_room_count": 3,
+  "registration_date": "2012-10-02",
+  "restricted_access": 0,
+  "sap_energy_source": {
+    "main_gas": "Y",
+    "meter_type": 2,
+    "photovoltaic_supply": {
+      "percent_roof_area": 0
+    },
+    "wind_turbines_count": 0,
+    "wind_turbines_terrain_type": 2
+  },
+  "secondary_heating": {
+    "description": "None",
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "sap_building_parts": [
+    {
+      "identifier": "Main Dwelling",
+      "wall_dry_lined": "N",
+      "wall_thickness": 300,
+      "floor_heat_loss": 7,
+      "roof_construction": 3,
+      "wall_construction": 3,
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": 2.65,
+          "floor_insulation": 1,
+          "total_floor_area": 77.69,
+          "floor_construction": 2,
+          "heat_loss_perimeter": 28.9
+        }
+      ],
+      "wall_insulation_type": 3,
+      "construction_age_band": "B",
+      "wall_thickness_measured": "Y",
+      "roof_insulation_location": "ND",
+      "roof_insulation_thickness": "ND",
+      "wall_insulation_thickness": "50mm"
+    }
+  ],
+  "low_energy_lighting": 100,
+  "solar_water_heating": "N",
+  "bedf_revision_number": 329,
+  "habitable_room_count": 3,
+  "heating_cost_current": {
+    "value": 421,
+    "currency": "GBP"
+  },
+  "insulated_door_count": 0,
+  "co2_emissions_current": 2.4,
+  "energy_rating_average": 60,
+  "energy_rating_current": 72,
+  "lighting_cost_current": {
+    "value": 46,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": "Programmer, room thermostat and TRVs",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "multiple_glazing_type": 1,
+  "open_fireplaces_count": 0,
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 360,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 85,
+    "currency": "GBP"
+  },
+  "mechanical_ventilation": 0,
+  "percent_draughtproofed": 100,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 61,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00a3800 - \u00a31,200",
+      "improvement_type": "W",
+      "improvement_details": {
+        "improvement_number": 47
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 75,
+      "environmental_impact_rating": 77
+    }
+  ],
+  "co2_emissions_potential": 2.0,
+  "energy_rating_potential": 75,
+  "lighting_cost_potential": {
+    "value": 46,
+    "currency": "GBP"
+  },
+  "hot_water_cost_potential": {
+    "value": 85,
+    "currency": "GBP"
+  },
+  "renewable_heat_incentive": {
+    "water_heating": 2121,
+    "space_heating_existing_dwelling": 7355
+  },
+  "seller_commission_report": "Y",
+  "energy_consumption_current": 160,
+  "has_fixed_air_conditioning": "false",
+  "multiple_glazed_proportion": 100,
+  "calculation_software_version": 8.0,
+  "energy_consumption_potential": 137,
+  "environmental_impact_current": 72,
+  "fixed_lighting_outlets_count": 6,
+  "current_energy_efficiency_band": "C",
+  "environmental_impact_potential": 77,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "C",
+  "co2_emissions_current_per_floor_area": 31,
+  "low_energy_fixed_lighting_outlets_count": 6
+}

backend/epc_api/json_samples/real_life_examples/SAP-Schema-16.3/uprn_100061905751/epc.json

@@ -0,0 +1,335 @@
+{
+  "uprn": 100061905751,
+  "roofs": [
+    {
+      "description": "Pitched, 100 mm loft insulation",
+      "energy_efficiency_rating": 3,
+      "environmental_efficiency_rating": 3
+    }
+  ],
+  "walls": [
+    {
+      "description": "Cavity wall, as built, no insulation (assumed)",
+      "energy_efficiency_rating": 1,
+      "environmental_efficiency_rating": 1
+    }
+  ],
+  "floors": [
+    {
+      "description": "Suspended, no insulation (assumed)",
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "status": "entered",
+  "tenure": 1,
+  "windows": [
+    {
+      "description": "Fully double glazed",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "addendum": {
+    "cavity_fill_recommended": "true"
+  },
+  "lighting": {
+    "description": "Low energy lighting in 17% of fixed outlets",
+    "energy_efficiency_rating": 2,
+    "environmental_efficiency_rating": 2
+  },
+  "postcode": "BN11 4EP",
+  "hot_water": {
+    "description": "From main system",
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "post_town": "WORTHING",
+  "built_form": 3,
+  "created_at": "2014-07-24 12:49:23",
+  "door_count": 2,
+  "glazed_area": 1,
+  "region_code": 14,
+  "report_type": 2,
+  "sap_heating": {
+    "wwhrs": {
+      "rooms_with_bath_and_or_shower": 1,
+      "rooms_with_mixer_shower_no_bath": 0,
+      "rooms_with_bath_and_mixer_shower": 1
+    },
+    "cylinder_size": 1,
+    "water_heating_code": 901,
+    "water_heating_fuel": 26,
+    "secondary_fuel_type": 15,
+    "main_heating_details": [
+      {
+        "has_fghrs": "N",
+        "main_fuel_type": 26,
+        "boiler_flue_type": 2,
+        "fan_flue_present": "Y",
+        "heat_emitter_type": 1,
+        "boiler_index_number": 16366,
+        "main_heating_number": 1,
+        "main_heating_control": 2106,
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "main_heating_data_source": 1
+      }
+    ],
+    "secondary_heating_type": 631,
+    "has_fixed_air_conditioning": "false"
+  },
+  "sap_version": 9.91,
+  "schema_type": "SAP-Schema-16.3",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": "Boiler and radiators, mains gas",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "dwelling_type": "End-terrace house",
+  "language_code": 1,
+  "property_type": 0,
+  "address_line_1": "77, Tarring Road",
+  "schema_version": "LIG-16.1",
+  "assessment_type": "RdSAP",
+  "completion_date": "2014-07-24",
+  "inspection_date": "2014-07-22",
+  "extensions_count": 1,
+  "measurement_type": 1,
+  "total_floor_area": 139,
+  "transaction_type": 5,
+  "conservatory_type": 1,
+  "heated_room_count": 5,
+  "registration_date": "2014-04-25",
+  "restricted_access": 0,
+  "sap_energy_source": {
+    "main_gas": "Y",
+    "meter_type": 2,
+    "photovoltaic_supply": [
+      [
+        {
+          "pitch": 3,
+          "peak_power": 1.25,
+          "orientation": 5,
+          "overshading": 1
+        }
+      ],
+      [
+        {
+          "pitch": 3,
+          "peak_power": 0.5,
+          "orientation": 3,
+          "overshading": 1
+        }
+      ],
+      [
+        {
+          "pitch": 3,
+          "peak_power": 0.5,
+          "orientation": 7,
+          "overshading": 2
+        }
+      ]
+    ],
+    "wind_turbines_count": 0,
+    "wind_turbines_terrain_type": 2
+  },
+  "secondary_heating": {
+    "description": "Room heaters, smokeless fuel",
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "lzc_energy_sources": [
+    11
+  ],
+  "sap_building_parts": [
+    {
+      "identifier": "Main Dwelling",
+      "wall_dry_lined": "N",
+      "wall_thickness": 300,
+      "floor_heat_loss": 7,
+      "roof_construction": 4,
+      "wall_construction": 4,
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": 2.7,
+          "floor_insulation": 1,
+          "total_floor_area": 67.49,
+          "floor_construction": 2,
+          "heat_loss_perimeter": 27.69
+        },
+        {
+          "floor": 1,
+          "room_height": 2.47,
+          "total_floor_area": 67.49,
+          "heat_loss_perimeter": 31.38
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "A",
+      "wall_thickness_measured": "Y",
+      "roof_insulation_location": 2,
+      "roof_insulation_thickness": "100mm"
+    },
+    {
+      "identifier": "Extension",
+      "wall_dry_lined": "N",
+      "wall_thickness": 280,
+      "floor_heat_loss": 7,
+      "roof_construction": 5,
+      "wall_construction": 4,
+      "building_part_number": 2,
+      "sap_floor_dimensions": [
+        {
+          "floor": 0,
+          "room_height": 2.45,
+          "floor_insulation": 1,
+          "total_floor_area": 3.84,
+          "floor_construction": 1,
+          "heat_loss_perimeter": 5.77
+        }
+      ],
+      "wall_insulation_type": 4,
+      "construction_age_band": "B",
+      "wall_thickness_measured": "Y",
+      "roof_insulation_location": 4,
+      "roof_insulation_thickness": "NI"
+    }
+  ],
+  "low_energy_lighting": 17,
+  "solar_water_heating": "N",
+  "bedf_revision_number": 361,
+  "habitable_room_count": 5,
+  "heating_cost_current": {
+    "value": 1430,
+    "currency": "GBP"
+  },
+  "co2_emissions_current": 7.2,
+  "energy_rating_average": 60,
+  "energy_rating_current": 57,
+  "lighting_cost_current": {
+    "value": 132,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": "Programmer, room thermostat and TRVs",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "multiple_glazing_type": 2,
+  "open_fireplaces_count": 2,
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 881,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 105,
+    "currency": "GBP"
+  },
+  "mechanical_ventilation": 0,
+  "percent_draughtproofed": 100,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": {
+        "value": 308,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00c2\u00a3500 - \u00c2\u00a31,500",
+      "improvement_type": "B",
+      "improvement_details": {
+        "improvement_number": 6
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 71,
+      "environmental_impact_rating": 63
+    },
+    {
+      "sequence": 2,
+      "typical_saving": {
+        "value": 73,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00c2\u00a3800 - \u00c2\u00a31,200",
+      "improvement_type": "W",
+      "improvement_details": {
+        "improvement_number": 47
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 74,
+      "environmental_impact_rating": 66
+    },
+    {
+      "sequence": 3,
+      "typical_saving": {
+        "value": 49,
+        "currency": "GBP"
+      },
+      "indicative_cost": "\u00c2\u00a375",
+      "improvement_type": "E",
+      "improvement_details": {
+        "improvement_number": 35
+      },
+      "improvement_category": 5,
+      "energy_performance_rating": 75,
+      "environmental_impact_rating": 67
+    }
+  ],
+  "co2_emissions_potential": 4.0,
+  "energy_rating_potential": 75,
+  "lighting_cost_potential": {
+    "value": 72,
+    "currency": "GBP"
+  },
+  "alternative_improvements": [
+    {
+      "improvement": {
+        "sequence": 1,
+        "typical_saving": {
+          "value": 67,
+          "currency": "GBP"
+        },
+        "improvement_type": "Q2",
+        "improvement_details": {
+          "improvement_number": 55
+        },
+        "improvement_category": 6,
+        "energy_performance_rating": 73,
+        "environmental_impact_rating": 65
+      }
+    }
+  ],
+  "hot_water_cost_potential": {
+    "value": 105,
+    "currency": "GBP"
+  },
+  "renewable_heat_incentive": {
+    "water_heating": 2315,
+    "impact_of_loft_insulation": -869,
+    "impact_of_cavity_insulation": -8658,
+    "space_heating_existing_dwelling": 23216
+  },
+  "seller_commission_report": "Y",
+  "energy_consumption_current": 230,
+  "has_fixed_air_conditioning": "false",
+  "multiple_glazed_proportion": 100,
+  "calculation_software_version": "9.1.2",
+  "energy_consumption_potential": 125,
+  "environmental_impact_current": 45,
+  "fixed_lighting_outlets_count": 18,
+  "current_energy_efficiency_band": "D",
+  "environmental_impact_potential": 67,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "C",
+  "co2_emissions_current_per_floor_area": 52,
+  "low_energy_fixed_lighting_outlets_count": 3
+}

backend/epc_api/json_samples/real_life_examples/SAP-Schema-19.1.0/uprn_10096028301/elmhurst_inputs.md

@@ -0,0 +1,96 @@
+# Elmhurst RdSAP inputs — UPRN 10096028301 (cert 0390-3321-6060-2405-7985, SAP-Schema-19.1.0)
+
+**Lodged SAP:** 85   **Our engine:** 82   ← compare Elmhurst against the engine
+**Property:** Ground-floor FLAT, age band M (2023 onwards), mains-gas combi + radiators, decentralised MEV, TFA 73 m²
+
+**Schema note:** SAP-Schema-19.1.0 is FULL SAP (measured U-values, measured air
+permeability) — mapped via the new `from_sap_schema_19_1_0` (parses with the 17.1
+dataclass, delegates to `from_sap_schema_17_1`). Near-twin of the worked-ref flat
+10092973954 (same PCDB combi 17929, MEV, AP50). Expect the documented full-SAP→RdSAP
+residual: the engine uses the cert's MEASURED U (wall 0.24 / floor 0.13 — WORSE than
+RdSAP band-M defaults), so here the engine *under*-rates vs RdSAP (eng 82 < lodged 85).
+
+**Known divergences / gaps to watch:**
+- **MEV (decentralised, EXTRACT_OR_PIV_OUTSIDE):** engine prices it as extract loss,
+  not heat recovery — the documented MVHR gap. Omit/parallel on both sides.
+- **Measured vs RdSAP U:** Elmhurst can't set the U-value-known override → it uses band-M
+  age defaults (better than the cert's measured 0.24 wall). Note the divergence; don't tune.
+- Age band M → Elmhurst on-screen band **M 2023 onwards**.
+
+## Property Description
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Property Type | **Flat** | ground-floor flat |
+| Built form | **Detached** (block) | built_form not lodged (full SAP) — use Detached as for worked-ref flat |
+| Age band | **M 2023 onwards** | construction_age_band M |
+| Storeys | 1 | ground-floor flat |
+| Habitable rooms | 3 | habitable_room_count 3 |
+
+## Flats
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Position of flat | **Ground Floor** | dwelling_type ground-floor flat |
+| Floor | 0 | ground = storey 0 |
+| Corridor | None | all-exposed |
+
+## Dimensions
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Floor area (lowest) | 72.99 m² | floor 0 |
+| Room height | 2.36 m | room_height |
+| Heat-loss perimeter | 36.04 m | heat_loss_perimeter |
+| Party-wall length | **0** | not lodged |
+
+## Walls
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Construction | **Cavity** | wall_construction 4 |
+| Insulation | **As Built** | measured U 0.24 W/m²K (U-known override not settable → accept band-M default; note divergence) |
+
+## Roofs
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Type | **Another dwelling above** | roof "(other premises above)" → no roof heat loss |
+
+## Floors
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Location / type | Ground floor / Solid | measured U 0.13 (accept band-M default) |
+| Insulation | As built | — |
+
+## Openings
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Windows | total ~11.37 m² (4 windows), U 1.3, g 0.46 | glazing_type 7. Enter ONE combined band row ("Double … 2022" ≈ U1.4/g0.72) of 11.37 m² — `…known data` demands per-row frame-factor/source and fails validation. |
+| Doors | **1, solid (uninsulated)** U 1.2 | sap_opening_types: 1 Solid Door, U 1.2 |
+
+## Ventilation & Lighting
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Ventilation | **Mechanical — decentralised MEV** (extract) | ventilation EXTRACT_OR_PIV_OUTSIDE. 🚩 engine models as extract loss not heat recovery. |
+| Air Pressure Test | **Blower Door**, result **3.5** (AP50) | air_tightness "AP50 = 3.5 as tested"; Elmhurst (18)=AP50/20. Needs a certificate number to pass Recommendations validation. |
+| Sheltered sides | (from built form) | — |
+| Lighting | per cert low-energy % | sap_lighting |
+
+## Space Heating
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Main heating | Mains gas **combi** + radiators | fuel mains gas, PCDB 17929 (Vaillant combi), control 2110, category 2, NO cylinder → BGW condensing combi cascade |
+| Flue | Balanced (room-sealed) | fan_flue_present; balanced flue |
+| Secondary | None | — |
+
+## Water Heating
+| Elmhurst field | Value | Notes |
+|---|---|---|
+| Water heating | From main (901), gas | combi |
+| Hot water cylinder | **None** | has_hot_water_cylinder False (combi) |
+| Solar / WWHRS / FGHRS | None | — |
+
+## Fields to clear in Elmhurst (do NOT map)
+| Elmhurst field | Set to | Why absent |
+|---|---|---|
+| Conservatory | none | not lodged |
+| Cylinder size/insulation | (gone — combi) | no cylinder |
+| Main Heating 2 / Secondary | none | single combi system |
+| PV / Wind / Hydro / WWHRS / FGHRS | none | none lodged |
+| Room-in-roof / extensions | blank | single building part |

backend/epc_api/json_samples/real_life_examples/SAP-Schema-19.1.0/uprn_10096028301/epc.json

@@ -0,0 +1,456 @@
+{
+  "der": 12.66,
+  "ter": 13.84,
+  "dfee": 36.3,
+  "dper": 69.44,
+  "tfee": 38.6,
+  "tper": 72.99,
+  "uprn": 10096028301,
+  "roofs": [
+    {
+      "description": {
+        "value": "(other premises above)",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 0,
+      "environmental_efficiency_rating": 0
+    }
+  ],
+  "walls": [
+    {
+      "description": {
+        "value": "Average thermal transmittance 0.24 W/m\u00b2K",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "floors": [
+    {
+      "description": {
+        "value": "Average thermal transmittance 0.13 W/m\u00b2K",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "status": "entered",
+  "tenure": "ND",
+  "windows": {
+    "description": {
+      "value": "High performance glazing",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "lighting": {
+    "description": {
+      "value": "Good lighting efficiency",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "postcode": "CT5 1TN",
+  "data_type": 2,
+  "hot_water": {
+    "description": {
+      "value": "From main system, waste water heat recovery",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "post_town": "Whitstable",
+  "built_form": 1,
+  "created_at": "2025-06-09 10:49:27",
+  "living_area": 26.89,
+  "orientation": 3,
+  "region_code": 14,
+  "report_type": 3,
+  "sap_heating": {
+    "number_baths": 1,
+    "thermal_store": 1,
+    "shower_outlets": [
+      {
+        "shower_wwhrs": 2,
+        "shower_flow_rate": 9,
+        "shower_outlet_type": 3
+      }
+    ],
+    "water_fuel_type": 1,
+    "water_heating_code": 901,
+    "instantaneous_wwhrs": {
+      "wwhrs_index_number1": 80137
+    },
+    "main_heating_details": [
+      {
+        "has_fghrs": "false",
+        "main_fuel_type": 1,
+        "heat_emitter_type": 1,
+        "is_flue_fan_present": "true",
+        "main_heating_number": 1,
+        "is_condensing_boiler": "true",
+        "main_heating_control": 2110,
+        "is_interlocked_system": "true",
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "gas_or_oil_boiler_type": 2,
+        "main_heating_flue_type": 2,
+        "central_heating_pump_age": 2,
+        "main_heating_data_source": 1,
+        "main_heating_index_number": 17929,
+        "has_separate_delayed_start": "true",
+        "is_oil_pump_in_heated_space": "false",
+        "is_main_heating_hetas_approved": "false",
+        "is_central_heating_pump_in_heated_space": "true"
+      }
+    ],
+    "has_hot_water_cylinder": "false",
+    "has_cylinder_thermostat": "false",
+    "has_fixed_air_conditioning": "false",
+    "secondary_heating_category": 1,
+    "is_immersion_for_summer_use": "false",
+    "is_hot_water_separately_timed": "false",
+    "is_heat_pump_assisted_by_immersion": "false"
+  },
+  "sap_version": 10.2,
+  "schema_type": "SAP-Schema-19.1.0",
+  "uprn_source": "Address Matched",
+  "country_code": "ENG",
+  "main_heating": [
+    {
+      "description": {
+        "value": "Boiler and radiators, mains gas",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "sap_lighting": [
+    [
+      {
+        "lighting_power": 9,
+        "lighting_outlets": 6,
+        "lighting_efficacy": 80
+      }
+    ]
+  ],
+  "terrain_type": 2,
+  "air_tightness": {
+    "description": {
+      "value": "Air permeability [AP50] = 3.5 m\u00b3/h.m\u00b2 (as tested)",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "dwelling_type": "Ground-floor flat",
+  "language_code": 1,
+  "property_type": 2,
+  "pv_connection": 1,
+  "address_line_1": "1 Ransome Way",
+  "assessment_date": "2025-06-09",
+  "assessment_type": "SAP",
+  "completion_date": "2025-06-09",
+  "inspection_date": "2025-06-09",
+  "sap_ventilation": {
+    "psv_count": 0,
+    "wall_type": 2,
+    "pressure_test": 1,
+    "wet_rooms_count": 2,
+    "air_permeability": 3.52,
+    "open_flues_count": 0,
+    "ventilation_type": 6,
+    "has_draught_lobby": "true",
+    "other_flues_count": 0,
+    "closed_flues_count": 0,
+    "extract_fans_count": 0,
+    "boilers_flues_count": 0,
+    "open_chimneys_count": 0,
+    "sheltered_sides_count": 2,
+    "blocked_chimneys_count": 0,
+    "kitchen_duct_fans_count": 0,
+    "kitchen_room_fans_count": 0,
+    "kitchen_wall_fans_count": 1,
+    "flueless_gas_fires_count": 0,
+    "mechanical_vent_duct_type": 1,
+    "non_kitchen_duct_fans_count": 0,
+    "non_kitchen_room_fans_count": 0,
+    "non_kitchen_wall_fans_count": 1,
+    "mechanical_ventilation_data_source": 1,
+    "mechanical_vent_system_index_number": 500769,
+    "is_mechanical_vent_approved_installer_scheme": "false"
+  },
+  "design_water_use": 1,
+  "sap_data_version": 10.2,
+  "sap_flat_details": {
+    "level": 1,
+    "storeys": 3
+  },
+  "total_floor_area": 73,
+  "transaction_type": 6,
+  "cold_water_source": 1,
+  "conservatory_type": 1,
+  "registration_date": "2025-06-09",
+  "sap_energy_source": {
+    "pv_arrays": [
+      {
+        "pitch": 3,
+        "peak_power": 0.38,
+        "orientation": 5,
+        "overshading": 1
+      }
+    ],
+    "electricity_tariff": 1
+  },
+  "sap_opening_types": [
+    {
+      "name": "Solid Door",
+      "type": 1,
+      "u_value": 1.2,
+      "data_source": 2,
+      "glazing_type": 1,
+      "isargonfilled": "false"
+    },
+    {
+      "name": "Windows",
+      "type": 4,
+      "u_value": 1.3,
+      "data_source": 4,
+      "frame_factor": 1.0,
+      "glazing_type": 7,
+      "isargonfilled": "false",
+      "solar_transmittance": 0.46
+    }
+  ],
+  "secondary_heating": {
+    "description": {
+      "value": "None",
+      "language": "1"
+    },
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "lowest_storey_area": 72.99,
+  "lzc_energy_sources": [
+    11
+  ],
+  "sap_building_parts": [
+    {
+      "sap_roofs": [
+        {
+          "name": "Party roof 1",
+          "u_value": 0,
+          "roof_type": 4,
+          "kappa_value": 30,
+          "total_roof_area": 72.99
+        }
+      ],
+      "sap_walls": [
+        {
+          "name": "Walls (1)",
+          "u_value": 0.24,
+          "wall_type": 2,
+          "kappa_value": 60,
+          "total_wall_area": 68.76,
+          "is_curtain_walling": "false"
+        },
+        {
+          "name": "Walls (2)",
+          "u_value": 0.26,
+          "wall_type": 3,
+          "kappa_value": 110,
+          "total_wall_area": 16.3,
+          "is_curtain_walling": "false"
+        },
+        {
+          "name": "Internal Wall (1)",
+          "u_value": 0,
+          "wall_type": 5,
+          "kappa_value": 9,
+          "total_wall_area": 114.18,
+          "is_curtain_walling": "false"
+        }
+      ],
+      "sap_openings": [
+        {
+          "name": "Front Door",
+          "type": "Solid Door",
+          "width": 2.15,
+          "height": 1,
+          "location": "Walls (1)",
+          "orientation": 3
+        },
+        {
+          "name": "Front Windows",
+          "type": "Windows",
+          "width": 4.77,
+          "height": 1,
+          "location": "Walls (1)",
+          "orientation": 3
+        },
+        {
+          "name": "RH Windows",
+          "type": "Windows",
+          "width": 4.39,
+          "height": 1,
+          "location": "Walls (1)",
+          "orientation": 1
+        },
+        {
+          "name": "Rear Window",
+          "type": "Windows",
+          "width": 0.62,
+          "height": 1,
+          "location": "Walls (1)",
+          "orientation": 7
+        },
+        {
+          "name": "LH Window",
+          "type": "Windows",
+          "width": 1.59,
+          "height": 1,
+          "location": "Walls (1)",
+          "orientation": 5
+        }
+      ],
+      "construction_year": 2025,
+      "sap_thermal_bridges": {
+        "thermal_bridges": [
+          {
+            "length": 9.65,
+            "psi_value": 0.021,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E2"
+          },
+          {
+            "length": 8.16,
+            "psi_value": 0.017,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E3"
+          },
+          {
+            "length": 16.95,
+            "psi_value": 0.013,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E4"
+          },
+          {
+            "length": 29.14,
+            "psi_value": 0.054,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E5"
+          },
+          {
+            "length": 29.14,
+            "psi_value": 0.052,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E7"
+          },
+          {
+            "length": 11.61,
+            "psi_value": 0.046,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E16"
+          },
+          {
+            "length": 6.91,
+            "psi_value": 0.111,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E5"
+          },
+          {
+            "length": 6.91,
+            "psi_value": 0.056,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E7"
+          },
+          {
+            "length": 4.72,
+            "psi_value": -0.1,
+            "psi_value_source": 1,
+            "thermal_bridge_type": "E17"
+          }
+        ],
+        "thermal_bridge_code": 5
+      },
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "storey": 0,
+          "u_value": 0.13,
+          "floor_type": 2,
+          "kappa_value": 75,
+          "storey_height": 2.36,
+          "heat_loss_area": 72.99,
+          "total_floor_area": 72.99
+        }
+      ]
+    }
+  ],
+  "user_interface_name": "Design SAP 10",
+  "windows_overshading": 2,
+  "heating_cost_current": {
+    "value": 218,
+    "currency": "GBP"
+  },
+  "co2_emissions_current": 0.8,
+  "energy_rating_average": 60,
+  "energy_rating_current": 85,
+  "lighting_cost_current": {
+    "value": 57,
+    "currency": "GBP"
+  },
+  "main_heating_controls": [
+    {
+      "description": {
+        "value": "Time and temperature zone control",
+        "language": "1"
+      },
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "has_hot_water_cylinder": "false",
+  "heating_cost_potential": {
+    "value": 218,
+    "currency": "GBP"
+  },
+  "hot_water_cost_current": {
+    "value": 118,
+    "currency": "GBP"
+  },
+  "user_interface_version": "2.22.1",
+  "co2_emissions_potential": 0.8,
+  "energy_rating_potential": 85,
+  "gas_smart_meter_present": "false",
+  "lighting_cost_potential": {
+    "value": 57,
+    "currency": "GBP"
+  },
+  "schema_version_original": "SAP-Schema-19.1.0",
+  "hot_water_cost_potential": {
+    "value": 118,
+    "currency": "GBP"
+  },
+  "is_in_smoke_control_area": "unknown",
+  "seller_commission_report": "Y",
+  "energy_consumption_current": 59,
+  "has_fixed_air_conditioning": "false",
+  "is_dwelling_export_capable": "true",
+  "multiple_glazed_percentage": 100,
+  "calculation_software_version": "2.22.1",
+  "energy_consumption_potential": 59,
+  "environmental_impact_current": 90,
+  "current_energy_efficiency_band": "B",
+  "environmental_impact_potential": 90,
+  "electricity_smart_meter_present": "false",
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "B",
+  "co2_emissions_current_per_floor_area": 10.7
+}

datatypes/epc/domain/epc_property_data.py

@@ -35,15 +35,18 @@ class BuildingPartIdentifier(Enum):
     OTHER = "other"
 
     @classmethod
-    def from_api_string(cls, api_identifier: Optional[str]) -> "BuildingPartIdentifier":
+    def from_api_string(
+        cls, api_identifier: Optional[object]
+    ) -> "BuildingPartIdentifier":
         """Map a gov-EPC API `BuildingPart.identifier` to its canonical
         member. "Main Dwelling" → MAIN; "Extension N" → EXTENSION_N
-        (for N in 1..4). `None` (permitted by the 21_0_1 schema) and
-        anything unrecognised fall to OTHER.
+        (for N in 1..4). `None` (permitted by the 21_0_1 schema), a non-string
+        identifier (some SAP-16.x certs lodge a bare int), and anything else
+        unrecognised fall to OTHER.
         """
         if api_identifier == "Main Dwelling":
             return cls.MAIN
-        if api_identifier is not None:
+        if isinstance(api_identifier, str):
             match = _API_EXTENSION.match(api_identifier)
             if match is not None:
                 return cls.extension(int(match.group(1)))
@@ -367,7 +370,12 @@ class SapEnergySource:
     wind_turbines_terrain_type: str  # int in API, str (e.g. "Suburban") in site notes
     electricity_smart_meter_present: bool
 
-    pv_connection: Optional[Union[int, str]] = None  # int from API; str from site notes
+    # gov-API enum (int): 0 = no PV, 1 = PV present but NOT connected to the
+    # dwelling's own electricity meter (communal / separately metered), 2 = PV
+    # connected to the dwelling's meter. Per RdSAP 10 §11.1 / SAP 10.2 Appendix M,
+    # PV is credited to the dwelling's SAP only when connected (== 2); see
+    # `_pv_connected_to_dwelling_meter` in cert_to_inputs. str from site notes.
+    pv_connection: Optional[Union[int, str]] = None
     photovoltaic_supply: Optional[PhotovoltaicSupply] = None
     photovoltaic_arrays: Optional[List[PhotovoltaicArray]] = None
     wind_turbine_details: Optional[WindTurbineDetails] = None

datatypes/epc/domain/tests/test_from_rdsap_schema.py

@@ -191,6 +191,30 @@ class TestFromRdSapSchema20_0_0:
         schema = from_dict(RdSapSchema20_0_0, load("20_0_0.json"))
         return EpcPropertyDataMapper.from_rdsap_schema_20_0_0(schema)
 
+    def test_photovoltaic_supply_as_dict_list_is_mapped_not_crashed(self) -> None:
+        # 20.0.0 types `photovoltaic_supply` as the wrapper only (not the 21.0.x
+        # Union), so a cert lodging measured arrays as a LIST leaves the leaves as
+        # raw dicts. The mapper previously did `es.photovoltaic_supply.none_or_no_
+        # details` → "'list' object has no attribute 'none_or_no_details'", sinking
+        # the whole prediction cohort. Regression for cert 6102-6227-8000-0083-2292
+        # (uprn 22086693). It now routes through `_map_schema_21_pv`, whose
+        # dict-tolerant array reader captures the arrays.
+        data = load("20_0_0.json")
+        data["sap_energy_source"]["photovoltaic_supply"] = [
+            [{"pitch": 2, "peak_power": {"value": 1.14, "quantity": "kW"},
+              "orientation": 5, "overshading": 1}],
+            [{"pitch": 2, "peak_power": 1.14, "orientation": 5, "overshading": 1}],
+        ]
+        schema = from_dict(RdSapSchema20_0_0, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_20_0_0(schema)
+
+        arrays = result.sap_energy_source.photovoltaic_arrays
+        assert arrays is not None and len(arrays) == 2
+        assert arrays[0].peak_power == 1.14
+        assert arrays[0].orientation == 5
+        assert result.sap_energy_source.photovoltaic_supply is None
+
     def test_uprn(self, result: EpcPropertyData) -> None:
         assert result.uprn == 12457
 
@@ -229,6 +253,42 @@ class TestFromRdSapSchema21_0_0:
         schema = from_dict(RdSapSchema21_0_0, load("21_0_0.json"))
         return EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema)
 
+    def test_optional_fields_omitted_maps_not_crashed(self) -> None:
+        # Real-API 21.0.0 cohort certs omit a set of fields that the dataclass
+        # modelled as required while 21.0.1 already treats as optional. The
+        # modelling_e2e cohort skipped 33 such certs (e.g. 2205-3036-3484-0400-
+        # 5718). This is the exact union of fields those certs omit — all are
+        # optional in 21.0.1; widening 21.0.0 to match maps them into the donor
+        # pool instead of dropping them. (Building-part fields are NOT widened:
+        # the only one those certs omit is roof_insulation_thickness.)
+        data = load("21_0_0.json")
+        for window in data.get("sap_windows", []):
+            for k in ("frame_factor", "glazing_gap", "pvc_frame",
+                      "window_transmission_details"):
+                window.pop(k, None)
+        for k in ("wet_rooms_count", "open_chimneys_count",
+                  "pressure_test_certificate_number", "windows_transmission_details",
+                  "mechanical_ventilation_index_number", "mechanical_vent_duct_type",
+                  "mechanical_vent_duct_placement", "mechanical_vent_duct_insulation",
+                  "mechanical_vent_duct_insulation_level",
+                  "mechanical_vent_measured_installation", "insulated_door_u_value",
+                  "low_energy_fixed_lighting_bulbs_count",
+                  "cfl_fixed_lighting_bulbs_count", "led_fixed_lighting_bulbs_count",
+                  "suggested_improvements"):
+            data.pop(k, None)
+        data["sap_energy_source"].pop("wind_turbine_details", None)
+        data["sap_energy_source"].pop("pv_battery_count", None)
+        for bp in data.get("sap_building_parts", []):
+            bp.pop("roof_insulation_thickness", None)
+
+        schema = from_dict(RdSapSchema21_0_0, data)
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema)
+
+        assert result.uprn == 12457
+        # coalesced, not None → safe for the NOT-NULL epc_property column
+        assert result.wet_rooms_count == 0
+        assert result.open_chimneys_count == 0
+
     def test_uprn(self, result: EpcPropertyData) -> None:
         assert result.uprn == 12457
 
@@ -355,6 +415,66 @@ class TestFromRdSapSchema21_0_1:
 
     # --- general ---
 
+    def test_omitted_wet_rooms_count_defaults_to_zero_not_none(self) -> None:
+        # 21.0.1 lodges wet_rooms_count as Optional, so a cert that omits it
+        # mapped to EpcPropertyData.wet_rooms_count=None. That None then
+        # violated the epc_property.wet_rooms_count NOT-NULL constraint when a
+        # predicted EPC (which deep-copies a comparable template) was persisted,
+        # and would crash the calc's `wet_rooms_count > 0` check. 37 modelling_e2e
+        # properties failed on the 2026-06-23 run for this reason. Default to 0,
+        # matching every other mapper (RdSAP 0 → calc's Table-?? minimum 1 fan).
+        data = load("21_0_1.json")
+        data.pop("wet_rooms_count", None)
+        schema = from_dict(RdSapSchema21_0_1, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_1(schema)
+
+        assert result.wet_rooms_count == 0
+
+    def test_omitted_open_chimneys_count_defaults_to_zero_not_none(self) -> None:
+        data = load("21_0_1.json")
+        data.pop("open_chimneys_count", None)
+        schema = from_dict(RdSapSchema21_0_1, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_1(schema)
+
+        assert result.open_chimneys_count == 0
+
+    def test_omitted_cfl_fixed_lighting_bulbs_count_defaults_to_zero_not_none(self) -> None:
+        data = load("21_0_1.json")
+        data.pop("cfl_fixed_lighting_bulbs_count", None)
+        schema = from_dict(RdSapSchema21_0_1, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_1(schema)
+
+        assert result.cfl_fixed_lighting_bulbs_count == 0
+
+    def test_omitted_led_fixed_lighting_bulbs_count_defaults_to_zero_not_none(self) -> None:
+        data = load("21_0_1.json")
+        data.pop("led_fixed_lighting_bulbs_count", None)
+        schema = from_dict(RdSapSchema21_0_1, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_1(schema)
+
+        assert result.led_fixed_lighting_bulbs_count == 0
+
+    def test_banded_flat_location_string_coerced_to_band_floor_int(self) -> None:
+        # The gov API lodges flat_location (the flat's floor number) as a plain
+        # int on most certs, but on high floors as a banded string "20+" (floor
+        # 20 or above) — e.g. cert 2481-2122-4211-1172-4722 (UPRN 6027561).
+        # The epc_flat_details.flat_location column is a NOT-NULL integer, so the
+        # raw string crashed the modelling_e2e insert (psycopg2
+        # InvalidTextRepresentation, portfolio 796 run). Coerce the band to its
+        # floor (20); Elmhurst caps the floor number at 30.
+        data = load("21_0_1.json")
+        data["sap_flat_details"]["flat_location"] = "20+"
+        schema = from_dict(RdSapSchema21_0_1, data)
+
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_1(schema)
+
+        assert result.sap_flat_details is not None
+        assert result.sap_flat_details.flat_location == 20
+
     def test_uprn(self, result: EpcPropertyData) -> None:
         assert result.uprn == 12457
 
@@ -2357,3 +2477,281 @@ class TestNonSeparatedConservatoryApiMirror:
         # Assert — §6.2: disregarded; no conservatory geometry.
         assert epc.sap_conservatory is None
         assert conservatory_geometry(epc) is None
+
+
+class TestNonSeparatedConservatoryApiMirror19_0:
+    """RdSAP 10 §6.1 — the same glazed-BP conservatory the 21.0.1 mapper
+    handles is also lodged on RdSAP-Schema-19.0 (repro cert 718138). The four
+    glazed fields were undeclared on the 19.0 `SapBuildingPart`, so `from_dict`
+    dropped them: the conservatory mapped to a fabric-less building part that
+    mis-scored and could not persist (NOT-NULL `construction_age_band`). The
+    19.0 mapper now mirrors the 21.0.1 split.
+
+    Unlike 21.0.1, the 19.0 mapper keeps the **lodged** `total_floor_area`
+    scalar (a real 19.0 cert already lodges the non-separated conservatory in
+    it — 718138: dwelling 140.23 + conservatory 9.71 = 149.94 → lodged 150), so
+    appending a conservatory does NOT change `total_floor_area_m2`. The §6.1
+    floor-area fold reaches the score through the calculator's dimensions, not
+    through the scalar."""
+
+    def test_from_api_response_splits_out_conservatory_building_part(
+        self,
+    ) -> None:
+        # Arrange — the 19.0 dwelling plus a non-separated double-glazed
+        # conservatory glazed BP.
+        from datatypes.epc.domain.epc_property_data import SapConservatory
+        from domain.sap10_calculator.worksheet.conservatory import (
+            conservatory_geometry,
+        )
+
+        dwelling_part_count = len(
+            EpcPropertyDataMapper.from_api_response(load("19_0.json")).sap_building_parts
+        )
+
+        cert = load("19_0.json")
+        cert["conservatory_type"] = 4
+        cert["sap_building_parts"].append(
+            {
+                "floor_area": 12.0,
+                "room_height": 1,
+                "double_glazed": "Y",
+                "glazed_perimeter": 9.0,
+            }
+        )
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(cert)
+
+        # Assert — conservatory split out; the glazed BP is NOT a fabric part.
+        assert epc.sap_conservatory == SapConservatory(
+            floor_area_m2=12.0,
+            glazed_perimeter_m=9.0,
+            double_glazed=True,
+            thermally_separated=False,
+            room_height_storeys=1.0,
+        )
+        assert len(epc.sap_building_parts) == dwelling_part_count
+        # §6.1 fold is active (surfaces derived by the shared cascade).
+        assert conservatory_geometry(epc) is not None
+
+    def test_separated_conservatory_lodges_no_glazed_building_part(self) -> None:
+        # Arrange — a separated conservatory (type 2/3) lodges NO glazed BP;
+        # the dwelling is unchanged.
+        from domain.sap10_calculator.worksheet.conservatory import (
+            conservatory_geometry,
+        )
+
+        cert = load("19_0.json")
+        cert["conservatory_type"] = 2
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(cert)
+
+        # Assert — §6.2: disregarded; no conservatory geometry.
+        assert epc.sap_conservatory is None
+        assert conservatory_geometry(epc) is None
+
+
+class TestNonSeparatedConservatoryApiMirror20_0_0:
+    """RdSAP 10 §6.1 — the same glazed-BP conservatory, on RdSAP-Schema-20.0.0.
+    Surfaced by the modelling_e2e triage: 5 type-4 certs (e.g. property 725213)
+    failed `NotNullViolation: construction_age_band` because the 20.0.0 schema
+    declared none of the four glazed fields, so the conservatory mapped to a
+    null-age fabric part. Mirrors the 19.0 / 21.0.1 split."""
+
+    def test_from_api_response_splits_out_conservatory_building_part(
+        self,
+    ) -> None:
+        from datatypes.epc.domain.epc_property_data import SapConservatory
+        from domain.sap10_calculator.worksheet.conservatory import (
+            conservatory_geometry,
+        )
+
+        dwelling_part_count = len(
+            EpcPropertyDataMapper.from_api_response(load("20_0_0.json")).sap_building_parts
+        )
+
+        cert = load("20_0_0.json")
+        cert["conservatory_type"] = 4
+        cert["sap_building_parts"].append(
+            {
+                "floor_area": 12.0,
+                "room_height": 1,
+                "double_glazed": "Y",
+                "glazed_perimeter": 9.0,
+            }
+        )
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(cert)
+
+        # Assert — conservatory split out; the glazed BP is NOT a fabric part
+        # (so it never hits the NOT-NULL construction_age_band on persist).
+        assert epc.sap_conservatory == SapConservatory(
+            floor_area_m2=12.0,
+            glazed_perimeter_m=9.0,
+            double_glazed=True,
+            thermally_separated=False,
+            room_height_storeys=1.0,
+        )
+        assert len(epc.sap_building_parts) == dwelling_part_count
+        assert all(
+            bp.construction_age_band is not None for bp in epc.sap_building_parts
+        )
+        assert conservatory_geometry(epc) is not None
+
+    def test_separated_conservatory_lodges_no_glazed_building_part(self) -> None:
+        from domain.sap10_calculator.worksheet.conservatory import (
+            conservatory_geometry,
+        )
+
+        cert = load("20_0_0.json")
+        cert["conservatory_type"] = 2
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(cert)
+
+        # Assert — §6.2: disregarded; no conservatory geometry.
+        assert epc.sap_conservatory is None
+        assert conservatory_geometry(epc) is None
+
+
+@pytest.mark.parametrize("fixture", ["17_0.json", "17_1.json", "18_0.json"])
+def test_gov_mappers_split_non_separated_conservatory(fixture: str) -> None:
+    """RdSAP 10 §6.1 — every gov mapper splits a non-separated conservatory
+    (conservatory_type=4) glazed BP out of the fabric parts and into
+    `sap_conservatory`, so it never persists as a null-age fabric part (the
+    modelling_e2e `construction_age_band` NOT-NULL triage). 19.0 / 20.0.0 /
+    21.0.1 are covered by their own classes; this pins the rest.
+
+    21.0.0 is intentionally excluded: unlike every other gov schema it never
+    adopted the ADR-0027 "every SapBuildingPart field is Optional" treatment, so
+    a sparse conservatory part fails `from_dict` on its required fields. Porting
+    it needs that broader strictness change — tracked as a follow-up; no 21.0.0
+    conservatory cert surfaced in the triage."""
+    from datatypes.epc.domain.epc_property_data import SapConservatory
+
+    base_parts = len(
+        EpcPropertyDataMapper.from_api_response(load(fixture)).sap_building_parts
+    )
+    cert = load(fixture)
+    cert["conservatory_type"] = 4
+    cert["sap_building_parts"].append(
+        {"floor_area": 12.0, "room_height": 1, "double_glazed": "Y", "glazed_perimeter": 9.0}
+    )
+
+    # Act
+    epc = EpcPropertyDataMapper.from_api_response(cert)
+
+    # Assert — split out, not a (null-age) fabric part.
+    assert epc.sap_conservatory == SapConservatory(
+        floor_area_m2=12.0,
+        glazed_perimeter_m=9.0,
+        double_glazed=True,
+        thermally_separated=False,
+        room_height_storeys=1.0,
+    )
+    assert len(epc.sap_building_parts) == base_parts
+    assert all(bp.construction_age_band is not None for bp in epc.sap_building_parts)
+
+
+@pytest.mark.parametrize(
+    "schema_cls, mapper, fixture, expected",
+    [
+        (
+            RdSapSchema17_1,
+            EpcPropertyDataMapper.from_rdsap_schema_17_1,
+            "17_1.json",
+            "Programmer and room thermostat",
+        ),
+        (
+            RdSapSchema18_0,
+            EpcPropertyDataMapper.from_rdsap_schema_18_0,
+            "18_0.json",
+            "Programmer, room thermostat and TRVs",
+        ),
+        (
+            RdSapSchema19_0,
+            EpcPropertyDataMapper.from_rdsap_schema_19_0,
+            "19_0.json",
+            "Programmer, room thermostat and TRVs",
+        ),
+        (
+            RdSapSchema20_0_0,
+            EpcPropertyDataMapper.from_rdsap_schema_20_0_0,
+            "20_0_0.json",
+            "Programmer, room thermostat and TRVs",
+        ),
+        (
+            RdSapSchema21_0_0,
+            EpcPropertyDataMapper.from_rdsap_schema_21_0_0,
+            "21_0_0.json",
+            "Programmer, room thermostat and TRVs",
+        ),
+    ],
+)
+def test_rdsap_mappers_carry_main_heating_controls_display(
+    schema_cls: Any, mapper: Any, fixture: str, expected: str
+) -> None:
+    # "Heating Control" panel row. 17.0 and 21.0.1 already map it; these five
+    # dropped it, leaving the FE "Unknown" despite the cert lodging it. Display
+    # only — the calculator reads the control CODE separately (ADR-0037 family).
+    schema = from_dict(schema_cls, load(fixture))
+
+    result = mapper(schema)
+
+    assert result.main_heating_controls is not None
+    assert result.main_heating_controls.description == expected
+
+
+@pytest.mark.parametrize(
+    "schema_cls, mapper, fixture",
+    [
+        (RdSapSchema17_1, EpcPropertyDataMapper.from_rdsap_schema_17_1, "17_1.json"),
+        (RdSapSchema18_0, EpcPropertyDataMapper.from_rdsap_schema_18_0, "18_0.json"),
+        (RdSapSchema20_0_0, EpcPropertyDataMapper.from_rdsap_schema_20_0_0, "20_0_0.json"),
+        (RdSapSchema19_0, EpcPropertyDataMapper.from_rdsap_schema_19_0, "19_0.json"),
+        (RdSapSchema21_0_0, EpcPropertyDataMapper.from_rdsap_schema_21_0_0, "21_0_0.json"),
+    ],
+)
+def test_rdsap_mappers_map_mechanical_ventilation_kind(
+    schema_cls: Any, mapper: Any, fixture: str
+) -> None:
+    # Calc-facing: an MVHR cert (mechanical_ventilation=4) must reach the §2
+    # ventilation cascade as MVHR, not be silently treated as natural. 17.1 /
+    # 18.0 / 20.0.0 built a bare SapVentilation(sheltered_sides=…) and dropped
+    # the top-level mechanical_ventilation (fixed in #1333); 19.0 + 21.0.0 set
+    # no sap_ventilation at all, dropping the whole block (fixed here — the
+    # remaining ADR-0037 ventilation follow-up). 17.0 + 21.0.1 + full-SAP
+    # already map it (via _sap_17_1_ventilation / their own wiring). Natural
+    # certs (code 0/5 → None) are unchanged, so the fix only moves genuine
+    # MEV/MVHR certs.
+    data = load(fixture)
+    data["mechanical_ventilation"] = 4
+
+    result = mapper(from_dict(schema_cls, data))
+
+    assert result.sap_ventilation.mechanical_ventilation_kind == "MVHR"
+
+
+@pytest.mark.parametrize(
+    "schema_cls, mapper, fixture",
+    [
+        (RdSapSchema19_0, EpcPropertyDataMapper.from_rdsap_schema_19_0, "19_0.json"),
+        (RdSapSchema21_0_0, EpcPropertyDataMapper.from_rdsap_schema_21_0_0, "21_0_0.json"),
+    ],
+)
+def test_rdsap_19_0_21_0_0_construct_sap_ventilation_block(
+    schema_cls: Any, mapper: Any, fixture: str
+) -> None:
+    # 19.0 + 21.0.0 previously passed no `sap_ventilation=` at all → the
+    # EpcPropertyData default empty SapVentilation, dropping the entire §2
+    # block (sheltered_sides + kind). Mirroring 21.0.1, they now construct it.
+    # `sheltered_sides` derives from built_form per §S5 (both fixtures lodge
+    # built_form=2 Semi-Detached → 1 sheltered side); a missing/empty block
+    # would leave it None and over-count shelter via the cascade default of 2.
+    schema = from_dict(schema_cls, load(fixture))
+
+    result = mapper(schema)
+
+    assert result.sap_ventilation.sheltered_sides == 1

datatypes/epc/domain/tests/test_from_sap_schema.py

@@ -11,6 +11,7 @@ exercise the shape variation the design decisions hinge on
 """
 
 import json
+import logging
 import os
 from typing import Any, Dict
 
@@ -77,6 +78,137 @@ class TestFromSapSchema17_1Tracer:
         assert result.total_floor_area_m2 == 68.0
 
 
+class TestFromSapSchema17_1RebaselineFields:
+    """The calc-affecting categoricals the WIP mapper dropped (ADR-0037). Their
+    absence left the full-SAP family un-rebaselined and mis-scored, producing the
+    portfolio-796 "impossible downgrade"."""
+
+    def test_maps_sap_version_so_pre_sap10_rebaseline_can_fire(self) -> None:
+        # The full-SAP family lodges SAP 2012 (sap_version 9.92). Without it on
+        # the EpcPropertyData, CalculatorRebaseliner cannot fire trigger (a)
+        # (`sap_version < 10.2` → Effective = Calculated), so Effective
+        # Performance wrongly stays equal to the lodged SAP-2012 score while the
+        # plan models the SAP-10.2 calc — the impossible downgrade.
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.sap_version == 9.92
+
+    def test_maps_property_type_as_gov_code_string(self) -> None:
+        # property_type feeds the calculator (heat_transmission `_is_house` /
+        # `_is_flat_or_maisonette` drive party-wall + heat-loss handling). Left
+        # None, the dwelling is classified as neither → degraded SAP. The fixture
+        # is a flat (gov code 2); mapped as a string like from_rdsap_schema_17_1.
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.property_type == "2"
+
+    def test_maps_built_form_as_gov_code_string(self) -> None:
+        # built_form (gov code 1 in the fixture) → str, mirroring
+        # from_rdsap_schema_17_1; carried for the calculator's exposed-side /
+        # sheltered handling and the property-details panel.
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.built_form == "1"
+
+    def test_maps_secondary_heating_element(self) -> None:
+        # secondary_heating is read by the calculator (cert_to_inputs) — dropping
+        # it under-counts a real secondary heater. The fixture lodges "None" (no
+        # secondary); assert it's carried through rather than silently dropped.
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.secondary_heating is not None
+        assert result.secondary_heating.description == "None"
+
+    def test_maps_assessment_type(self) -> None:
+        # Full-SAP certs lodge assessment_type "SAP"; carried for provenance and
+        # the property-details panel, mirroring from_rdsap_schema_17_1.
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.assessment_type == "SAP"
+
+
+class TestFromSapSchema17_1DisplayElements:
+    """Display EnergyElements the WIP mapper dropped, leaving the FE
+    property-details panel "Unknown" for full-SAP certs (ADR-0037). Brings
+    full-SAP to parity with from_rdsap_schema_17_1's display coverage."""
+
+    @pytest.fixture
+    def result(self) -> EpcPropertyData:
+        schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
+        return EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+    def test_maps_main_heating_display_list(self, result: EpcPropertyData) -> None:
+        assert [e.description for e in result.main_heating] == [
+            "Boiler and radiators, mains gas"
+        ]
+
+    def test_maps_window_display_element(self, result: EpcPropertyData) -> None:
+        assert result.window is not None
+        assert result.window.description == "High performance glazing"
+
+    def test_maps_lighting_display_element(self, result: EpcPropertyData) -> None:
+        assert result.lighting is not None
+        assert result.lighting.description == "Low energy lighting in all fixed outlets"
+
+    def test_maps_hot_water_display_element(self, result: EpcPropertyData) -> None:
+        assert result.hot_water is not None
+        assert result.hot_water.description == "From main system"
+
+    def test_maps_main_heating_controls_display_element(
+        self, result: EpcPropertyData
+    ) -> None:
+        # "Heating Control" panel row. Full-SAP lodges it as a list; mirror the
+        # 21.0.1 mapper and take the first control system.
+        assert result.main_heating_controls is not None
+        assert (
+            result.main_heating_controls.description
+            == "Time and temperature zone control"
+        )
+
+
+class TestFullSapHasHotWaterCylinderFallback:
+    """Some full-SAP certs (e.g. SAP-Schema-17.0 cert 8265-7433-3220-9736-7902)
+    omit the top-level `has_hot_water_cylinder` and lodge it only under
+    `sap_heating`. The required top-level field made `from_dict` raise; it is now
+    optional and the mapper falls back to the nested value."""
+
+    def test_top_level_omitted_falls_back_to_sap_heating(self) -> None:
+        # Arrange — drop the top-level flag, keep the nested one ("true").
+        data = load("sap_17_1.json")
+        data.pop("has_hot_water_cylinder", None)
+        data["sap_heating"]["has_hot_water_cylinder"] = "true"
+
+        # Act
+        schema = from_dict(SapSchema17_1, data)
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        # Assert — resolved from sap_heating, not crashed
+        assert result.has_hot_water_cylinder is True
+
+    def test_top_level_omitted_nested_false_resolves_false(self) -> None:
+        # Arrange
+        data = load("sap_17_1.json")
+        data.pop("has_hot_water_cylinder", None)
+        data["sap_heating"]["has_hot_water_cylinder"] = "false"
+
+        # Act
+        schema = from_dict(SapSchema17_1, data)
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        # Assert
+        assert result.has_hot_water_cylinder is False
+
+
 class TestFromSapSchema17_1FabricDescriptions:
     """Slice 3 (D4): the measured-U fabric descriptions flow through to
     epc.walls/floors/roofs so the engine's u_wall/u_floor/u_roof can parse
@@ -245,6 +377,22 @@ class TestFromSapSchema17_1RoofWindows:
         result = self._map("sap_17_1.json")
         assert not result.sap_roof_windows
 
+    def test_rooflight_opening_type_6_maps_as_roof_window(self) -> None:
+        # SAP-19.1.0 certs lodge rooflights as opening-type 6 ("Ext Rooflight").
+        # SAP 10.2 treats roof windows and rooflights as the same inclined-glazing
+        # family, so code 6 maps onto sap_roof_windows like code 5 — previously it
+        # raised UnmappedApiCode and skipped the whole cert.
+        data = load("sap_17_1_house.json")
+        for ot in data["sap_opening_types"]:
+            if ot["type"] == 5:
+                ot["type"] = 6
+        schema = from_dict(SapSchema17_1, data)
+
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+
+        assert result.sap_roof_windows is not None
+        assert len(result.sap_roof_windows) == 2
+
 
 class TestFromSapSchema17_1UnknownOpeningType:
     """Slice 4d (D2): an opening-type code outside {1,2,3 door, 4 window,
@@ -324,7 +472,7 @@ class TestFromSapSchema17_1CodeBasedHeating:
 
 
 class TestFromSapSchema17_1Ventilation:
-    """Slice D5-vent: full-SAP sap_ventilation → measured air permeability (AP4),
+    """Slice D5-vent: full-SAP sap_ventilation → measured air permeability (AP50),
     ventilation_type → MechanicalVentilationKind, sheltered sides, wet rooms and
     the MEV PCDB index."""
 
@@ -333,9 +481,13 @@ class TestFromSapSchema17_1Ventilation:
         schema = from_dict(SapSchema17_1, load("sap_17_1.json"))
         return EpcPropertyDataMapper.from_sap_schema_17_1(schema)
 
-    def test_measured_air_permeability_fed_as_ap4(self, sample: EpcPropertyData) -> None:
+    def test_measured_air_permeability_fed_as_ap50(self, sample: EpcPropertyData) -> None:
+        # The lodged `air_permeability` is a q50 Blower-Door result, so it feeds the
+        # engine's AP50 path `(18) = AP50/20 + (8)` — NOT the AP4/Pulse formula
+        # `0.263 × AP4^0.924` (the air-permeability AP50 fix, uprn_10093116528).
         assert sample.sap_ventilation is not None
-        assert sample.sap_ventilation.air_permeability_ap4_m3_h_m2 == 2.6
+        assert sample.sap_ventilation.air_permeability_ap50_m3_h_m2 == 2.6
+        assert sample.sap_ventilation.air_permeability_ap4_m3_h_m2 is None
 
     def test_ventilation_type_6_is_extract(self, sample: EpcPropertyData) -> None:
         # ventilation_type 6 = MEV decentralised → EXTRACT_OR_PIV_OUTSIDE.
@@ -357,6 +509,31 @@ class TestFromSapSchema17_1Ventilation:
         with pytest.raises(UnmappedApiCode):
             EpcPropertyDataMapper.from_sap_schema_17_1(schema)
 
+    @pytest.mark.parametrize("code", [9, 10])
+    def test_natural_ventilation_codes_9_and_10_map_to_none(self, code: int) -> None:
+        data = load("sap_17_1.json")
+        data["sap_ventilation"]["ventilation_type"] = code
+        schema = from_dict(SapSchema17_1, data)
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+        assert result.sap_ventilation is not None
+        assert result.sap_ventilation.mechanical_ventilation_kind is None
+
+    def test_ventilation_type_7_is_mv(self) -> None:
+        data = load("sap_17_1.json")
+        data["sap_ventilation"]["ventilation_type"] = 7
+        schema = from_dict(SapSchema17_1, data)
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+        assert result.sap_ventilation is not None
+        assert result.sap_ventilation.mechanical_ventilation_kind == "MV"
+
+    def test_ventilation_type_8_is_mvhr(self) -> None:
+        data = load("sap_17_1.json")
+        data["sap_ventilation"]["ventilation_type"] = 8
+        schema = from_dict(SapSchema17_1, data)
+        result = EpcPropertyDataMapper.from_sap_schema_17_1(schema)
+        assert result.sap_ventilation is not None
+        assert result.sap_ventilation.mechanical_ventilation_kind == "MVHR"
+
 
 class TestFromSapSchema17_1Heating:
     """Slice D6: full-SAP sap_heating (differing field names) maps onto the
@@ -506,7 +683,10 @@ class TestFromSapSchema16_2:
         epc = EpcPropertyDataMapper.from_api_response(load("sap_18_0_0.json"))
         assert isinstance(epc, EpcPropertyData)
         assert epc.uprn == 10094601287
-        assert Sap10Calculator().calculate(epc).sap_score == 80  # lodged 84
+        # 80 -> 81 (closer to lodged 84) after the dMEV intermittent-fan fix:
+        # this cert is EXTRACT_OR_PIV_OUTSIDE with a lodged 0 fans, so the
+        # Table 5 age-band default is no longer substituted (SAP 10.2 §2 (7a)).
+        assert Sap10Calculator().calculate(epc).sap_score == 81  # lodged 84
 
     def test_16_0_dispatches_via_16_x_path_with_tenure_default(self) -> None:
         # SAP-Schema-16.0 is the same reduced-field 16.x shape; it omits the
@@ -519,6 +699,80 @@ class TestFromSapSchema16_2:
         assert epc.uprn == 100020933894
         assert Sap10Calculator().calculate(epc).sap_score == 61  # lodged 56
 
+    def test_16_x_missing_built_form_is_derived_from_dwelling_type(self) -> None:
+        # Some 16.0 certs (e.g. cert 8742-6624-9300-2780-4926) lodge
+        # `property_type`/`dwelling_type` but omit `built_form`, which RdSapSchema17_1
+        # requires — previously aborted the prediction cohort. The normaliser derives
+        # it from the dwelling_type text ("Semi-detached house" → 2). built_form is
+        # ML-only (SAP calc never reads it), so this is SAP- and gate-neutral.
+        data = load("sap_16_0.json")
+        del data["built_form"]
+        assert data["dwelling_type"] == "Semi-detached house"
+
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        assert isinstance(epc, EpcPropertyData)
+        assert epc.built_form == "2"
+
+    def test_16_x_missing_built_form_for_flat_falls_back_to_modal(self) -> None:
+        # A flat's dwelling_type ("Ground-floor flat") does not encode a form, so the
+        # derivation falls back to the modal 4 (Mid-terrace) — keeps the cert mappable.
+        data = load("sap_16_0.json")
+        del data["built_form"]
+        data["dwelling_type"] = "Ground-floor flat"
+
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        assert epc.built_form == "4"
+
+    def test_16_x_missing_insulated_door_count_defaults_to_zero(self) -> None:
+        # Some 16.x certs lodge `door_count` but omit `insulated_door_count`,
+        # which RdSapSchema17_1 requires — previously raised "missing required
+        # field 'insulated_door_count'", aborting the whole prediction cohort
+        # (EpcComparablePropertiesRepository.candidates_for maps every cohort
+        # cert in one list comprehension, so one unmappable cert killed the
+        # predict_epc path). Regression for cert 2628-3046-6233-4584-2970
+        # (SAP-Schema-16.3, uprn 100061905751) seen via modelling_e2e property
+        # 730259. The normaliser defaults the missing count to 0 — no insulated
+        # doors recorded, the conservative RdSAP assumption.
+        data = load("sap_16_3.json")
+        del data["insulated_door_count"]
+        assert "insulated_door_count" not in data
+
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        assert isinstance(epc, EpcPropertyData)
+        assert epc.insulated_door_count == 0
+
+    def test_16_x_missing_multiple_glazed_proportion_fails_loud(self) -> None:
+        # A 16.x cert omitting `multiple_glazed_proportion` (e.g. 16.3 cert
+        # 0418-3986-7250-2884-7970) is deliberately NOT defaulted — it fails the
+        # RdSapSchema17_1 parse and is handled by the cohort skip-and-report path.
+        # Synthesising a value drew the otherwise-unmappable cert into the
+        # EPC-prediction donor pool and regressed the component-accuracy gate;
+        # the field is ML-only so there is no calc cost to leaving it unmapped.
+        data = load("sap_16_3.json")
+        del data["multiple_glazed_proportion"]
+
+        with pytest.raises(
+            ValueError, match="missing required field 'multiple_glazed_proportion'"
+        ):
+            EpcPropertyDataMapper.from_api_response(data)
+
+    def test_recorded_co2_as_measurement_dict_is_coerced_not_crashed(self) -> None:
+        # Some certs (e.g. 16.x cert 2308-4997-7262-0137-9930) lodge
+        # `co2_emissions_current` as a Measurement dict {'value': 3.5, 'quantity':
+        # 'tonnes per year'} rather than a plain number. `_with_recorded_performance`
+        # previously did `float(co2)` → "float() argument must be ... not 'dict'",
+        # crashing the whole prediction cohort. It now coerces via _measurement_value.
+        data = load("sap_16_3.json")
+        data["co2_emissions_current"] = {"value": 3.5, "quantity": "tonnes per year"}
+
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        assert isinstance(epc, EpcPropertyData)
+        assert epc.co2_emissions_current == 3.5
+
     def test_16_2_normalizer_does_not_mutate_caller_dict(self) -> None:
         # Mirror _normalize_shower_outlets' contract: the caller's dict is
         # untouched (deep copy), so a re-dispatch sees the original shape.
@@ -527,3 +781,149 @@ class TestFromSapSchema16_2:
         EpcPropertyDataMapper.from_api_response(data)
         assert "window" not in data  # the rename landed only on the copy
         assert set(data.keys()) == before_keys
+
+
+class TestFullSapSchema16xRouting:
+    """The `SAP-Schema-16.x` name covers two structurally different certs: the
+    reduced RdSAP shape (top-level `door_count`/`glazed_area`) AND full SAP (the
+    as-designed `LIG-*` new-builds — `assessment_type=SAP`, measured
+    `sap_opening_types` + structured `sap_building_parts`, doors lodged as
+    openings not a top-level count). Dispatch must route each to its own mapper;
+    sending a full-SAP cert through the reduced normaliser failed loud with
+    `RdSapSchema17_1: missing required field 'door_count'`."""
+
+    def test_full_sap_16_x_cert_maps_via_full_sap_path(self) -> None:
+        # Arrange — a real full-SAP cert lodged under SAP-Schema-16.0 (cert
+        # 0293-3880-6151-9872-8795, lodged 83) that previously raised the
+        # missing-door_count ValueError in the reduced normaliser.
+        data = load("sap_16_0_full.json")
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        # Assert — mapped from the real measured openings, not reduced-field
+        # defaults: the single lodged door opening (0.9 x 2.1 m) → door_count 1.
+        assert isinstance(epc, EpcPropertyData)
+        assert epc.door_count == 1
+        assert epc.dwelling_type == "Detached house"
+
+    def test_reduced_16_x_cert_unaffected_by_full_sap_routing(self) -> None:
+        # Arrange — a reduced 16.2 cert (assessment_type RdSAP) must stay on the
+        # RdSAP path, keeping its top-level property_type.
+        data = load("sap_16_2.json")
+
+        # Act
+        epc = EpcPropertyDataMapper.from_api_response(data)
+
+        # Assert
+        assert epc.property_type is not None
+
+    def test_broken_schema_type_is_recorded(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        # Arrange — a full-SAP cert mislabelled as SAP-Schema-16.0: the label
+        # disagrees with the assessment_type, so the mismatch must be surfaced
+        # (not silently rerouted).
+        data = load("sap_16_0_full.json")
+
+        # Act
+        with caplog.at_level(logging.WARNING, logger="datatypes.epc.domain.mapper"):
+            EpcPropertyDataMapper.from_api_response(data)
+
+        # Assert
+        assert any(
+            "broken schema_type" in r.message and "SAP-Schema-16.0" in r.message
+            for r in caplog.records
+        )
+
+    def test_correctly_labelled_full_sap_is_not_recorded_as_broken(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        # Arrange — a correctly-labelled full-SAP cert: assessment_type SAP AND a
+        # recognised full-SAP label, so no mismatch to record.
+        data = load("sap_17_1.json")
+
+        # Act
+        with caplog.at_level(logging.WARNING, logger="datatypes.epc.domain.mapper"):
+            EpcPropertyDataMapper.from_api_response(data)
+
+        # Assert
+        assert not any("broken schema_type" in r.message for r in caplog.records)
+
+
+class TestFullSapSchema16xNoFloorDimensions:
+    """SAP-Schema-16.0 certs with assessment_type=SAP that lodge no
+    sap_floor_dimensions inside sap_building_parts (uprn 10090783001,
+    cert lodged 88). The API only carries total_floor_area at the top level;
+    the building part has sap_walls but no per-storey dimension data.
+
+    This is the root cause of the `max() arg is an empty sequence` crash in
+    building_geometry.roof_area() — documented in
+    scripts/handover_max_empty_sequence_design_question.md.
+    """
+
+    @pytest.fixture
+    def epc(self) -> EpcPropertyData:
+        return EpcPropertyDataMapper.from_api_response(
+            load("sap_16_0_full_no_floor_dims.json")
+        )
+
+    def test_maps_successfully(self, epc: EpcPropertyData) -> None:
+        assert isinstance(epc, EpcPropertyData)
+
+    def test_uprn_and_total_floor_area(self, epc: EpcPropertyData) -> None:
+        assert epc.uprn == 10090783001
+        assert epc.total_floor_area_m2 == 72.0
+
+    def test_building_part_floor_dimensions_synthesised(
+        self, epc: EpcPropertyData
+    ) -> None:
+        # The API lodges no sap_floor_dimensions for this cert family, but the
+        # mapper synthesises one from the cert-level total_floor_area so that
+        # roof_area() and gross_heat_loss_wall_area() do not crash downstream.
+        main_part = epc.sap_building_parts[0]
+        assert len(main_part.sap_floor_dimensions) == 1
+
+    def test_total_floor_area_available_at_cert_level(
+        self, epc: EpcPropertyData
+    ) -> None:
+        # total_floor_area_m2 IS present at the EpcPropertyData level (72 m²).
+        # This is the candidate fallback value for roof_area() — accurate for
+        # single-storey single-part certs, an overestimate for multi-storey.
+        assert epc.total_floor_area_m2 == 72.0
+
+    def test_synthesises_single_floor_dimension(self, epc: EpcPropertyData) -> None:
+        # Arrange
+        main_part = epc.sap_building_parts[0]
+
+        # Act / Assert
+        assert len(main_part.sap_floor_dimensions) == 1
+
+    def test_synthesised_floor_dimension_total_area(self, epc: EpcPropertyData) -> None:
+        # Arrange
+        fd = epc.sap_building_parts[0].sap_floor_dimensions[0]
+
+        # Act / Assert
+        assert fd.total_floor_area_m2 == 72.0
+
+    def test_synthesised_floor_dimension_heat_loss_perimeter(
+        self, epc: EpcPropertyData
+    ) -> None:
+        # Arrange: both sap_walls are wall_type=2 (exposed): 43.2 + 3.88 = 47.08 m².
+        # total_height falls back to 1.0 (no storey heights lodged), so perimeter = 47.08 m.
+        fd = epc.sap_building_parts[0].sap_floor_dimensions[0]
+
+        # Act / Assert
+        assert fd.heat_loss_perimeter_m == pytest.approx(47.08)
+
+    def test_raises_when_floor_dims_absent_and_synthesis_not_possible(
+        self,
+    ) -> None:
+        # Arrange: strip sap_flat_details so is_flat=False — synthesis guard
+        # cannot fire, empty floor dims must raise rather than silently produce [].
+        data = load("sap_16_0_full_no_floor_dims.json")
+        data.pop("sap_flat_details", None)
+
+        # Act / Assert
+        with pytest.raises(ValueError, match="Wardalls Grove.*SE14 5FB.*sap_floor_dimensions"):
+            EpcPropertyDataMapper.from_api_response(data)

datatypes/epc/schema/rdsap_schema_17_0.py

@@ -120,6 +120,14 @@ class SapBuildingPart:
     wall_insulation_thickness: Optional[str] = None
     floor_insulation_thickness: Optional[str] = None
     flat_roof_insulation_thickness: Optional[Union[str, int]] = None
+    # RdSAP 10 §6.1 — non-separated conservatory glazed-BP fields
+    # ({double_glazed, floor_area, glazed_perimeter, room_height}); declared so
+    # `from_dict` keeps them and the mapper splits the BP into
+    # `EpcPropertyData.sap_conservatory`. Mirrors the 19.0 / 21.0.1 schema.
+    floor_area: Optional[Union[Measurement, int, float]] = None
+    room_height: Optional[Union[Measurement, int, float]] = None
+    double_glazed: Optional[str] = None
+    glazed_perimeter: Optional[Union[Measurement, int, float]] = None
 
 
 @dataclass

datatypes/epc/schema/rdsap_schema_17_1.py

@@ -106,6 +106,14 @@ class SapBuildingPart:
     # Can be a thickness string (e.g. "100mm") or 0 for uninsulated flat roofs
     roof_insulation_thickness: Optional[Union[str, int]] = None
     wall_insulation_thickness: Optional[str] = None
+    # RdSAP 10 §6.1 — non-separated conservatory glazed-BP fields
+    # ({double_glazed, floor_area, glazed_perimeter, room_height}); declared so
+    # `from_dict` keeps them and the mapper splits the BP into
+    # `EpcPropertyData.sap_conservatory`. Mirrors the 19.0 / 21.0.1 schema.
+    floor_area: Optional[Union[Measurement, int, float]] = None
+    room_height: Optional[Union[Measurement, int, float]] = None
+    double_glazed: Optional[str] = None
+    glazed_perimeter: Optional[Union[Measurement, int, float]] = None
 
 
 @dataclass

datatypes/epc/schema/rdsap_schema_18_0.py

@@ -121,6 +121,14 @@ class SapBuildingPart:
     wall_insulation_thickness: Optional[str] = None
     floor_insulation_thickness: Optional[str] = None
     flat_roof_insulation_thickness: Optional[Union[str, int]] = None
+    # RdSAP 10 §6.1 — non-separated conservatory glazed-BP fields
+    # ({double_glazed, floor_area, glazed_perimeter, room_height}); declared so
+    # `from_dict` keeps them and the mapper splits the BP into
+    # `EpcPropertyData.sap_conservatory`. Mirrors the 19.0 / 21.0.1 schema.
+    floor_area: Optional[Union[Measurement, int, float]] = None
+    room_height: Optional[Union[Measurement, int, float]] = None
+    double_glazed: Optional[str] = None
+    glazed_perimeter: Optional[Union[Measurement, int, float]] = None
 
 
 @dataclass

datatypes/epc/schema/rdsap_schema_19_0.py

@@ -122,6 +122,16 @@ class SapBuildingPart:
     wall_insulation_thickness: Optional[str] = None
     floor_insulation_thickness: Optional[str] = None
     flat_roof_insulation_thickness: Optional[Union[str, int]] = None
+    # RdSAP 10 §6.1 — a NON-SEPARATED conservatory (`conservatory_type == 4`) is
+    # lodged by the gov API as a glazed "building part" carrying ONLY these four
+    # fields (no fabric, no floor dimensions). Previously undeclared → dropped by
+    # `from_dict`, so the conservatory was silently lost on the 19.0 path. The
+    # mapper splits this BP out into `EpcPropertyData.sap_conservatory`. Mirrors
+    # the 21.0.1 schema declaration.
+    floor_area: Optional[Union[Measurement, int, float]] = None
+    room_height: Optional[Union[Measurement, int, float]] = None
+    double_glazed: Optional[str] = None
+    glazed_perimeter: Optional[Union[Measurement, int, float]] = None
 
 
 @dataclass

datatypes/epc/schema/rdsap_schema_20_0_0.py

@@ -131,9 +131,12 @@ class SapBuildingPart:
     # ADR-0027: 17/1000 certs lodge a CONSERVATORY-shaped building part carrying
     # only {double_glazed, floor_area, glazed_perimeter, room_height} — none of
     # the wall/roof/floor construction fields below. Following the 21.0.1
-    # precedent every field is Optional, so a conservatory part parses to an
-    # all-None SapBuildingPart; its thermal effect is carried separately by the
-    # cert-level conservatory_type, so the empty part flows through harmlessly.
+    # precedent every field is Optional. RdSAP 10 §6.1: the four glazed fields
+    # are declared so `from_dict` keeps them and the mapper can split this BP out
+    # into `EpcPropertyData.sap_conservatory` (a non-separated conservatory,
+    # `conservatory_type == 4`). Previously undeclared → dropped → the part
+    # mapped to an all-None fabric part that mis-scored AND violated the
+    # NOT-NULL `construction_age_band` column on persist (modelling_e2e triage).
     identifier: Optional[str] = None
     wall_dry_lined: Optional[str] = None
     floor_heat_loss: Optional[int] = None
@@ -153,6 +156,12 @@ class SapBuildingPart:
     wall_insulation_thickness: Optional[str] = None
     floor_insulation_thickness: Optional[str] = None
     flat_roof_insulation_thickness: Optional[Union[str, int]] = None
+    # RdSAP 10 §6.1 — non-separated conservatory glazed-BP fields (see the
+    # class docstring). Mirrors the 19.0 / 21.0.1 schema declaration.
+    floor_area: Optional[Union[Measurement, int, float]] = None
+    room_height: Optional[Union[Measurement, int, float]] = None
+    double_glazed: Optional[str] = None
+    glazed_perimeter: Optional[Union[Measurement, int, float]] = None
 
 
 @dataclass

datatypes/epc/schema/rdsap_schema_21_0_0.py

@@ -14,7 +14,7 @@ class EnergyElement:
 
 @dataclass
 class Addendum:
-    addendum_numbers: List[int]
+    addendum_numbers: Optional[List[int]] = None
     stone_walls: Optional[str] = None
     system_build: Optional[str] = None
 
@@ -109,7 +109,7 @@ class PhotovoltaicSupplyNoneOrNoDetails:
 
 @dataclass
 class PhotovoltaicSupply:
-    none_or_no_details: PhotovoltaicSupplyNoneOrNoDetails
+    none_or_no_details: Optional[PhotovoltaicSupplyNoneOrNoDetails] = None
 
 
 @dataclass
@@ -127,15 +127,15 @@ class PhotovoltaicArray:
     overshading: int
 
 
-@dataclass
+@dataclass(kw_only=True)
 class SapEnergySource:
     mains_gas: str
     meter_type: int
     pv_connection: int
-    pv_battery_count: int
+    pv_battery_count: Optional[int] = None
     photovoltaic_supply: Union[PhotovoltaicSupply, List[List[PhotovoltaicArray]]]
     wind_turbines_count: int
-    wind_turbine_details: WindTurbineDetails
+    wind_turbine_details: Optional[WindTurbineDetails] = None
     gas_smart_meter_present: str
     is_dwelling_export_capable: str
     wind_turbines_terrain_type: int
@@ -153,11 +153,8 @@ class WindowTransmissionDetails:
 
 @dataclass
 class SapWindow:
-    pvc_frame: str
-    glazing_gap: int
     orientation: int
     window_type: int
-    frame_factor: float
     glazing_type: int
     window_width: float
     window_height: float
@@ -165,8 +162,15 @@ class SapWindow:
     window_location: int
     window_wall_type: int
     permanent_shutters_present: str
-    window_transmission_details: WindowTransmissionDetails
     permanent_shutters_insulated: str
+    # Optional window refinements: real-API 21.0.0 certs routinely omit these
+    # (frame_factor → Table 6c fallback, transmission → SAP10 glazing lookup,
+    # glazing_gap → type-only U). Modelled required originally but defaulted in
+    # 21.0.1; mirror that so otherwise-complete cohort certs are not skipped.
+    pvc_frame: Optional[str] = None
+    glazing_gap: Optional[int] = None
+    frame_factor: Optional[float] = None
+    window_transmission_details: Optional[WindowTransmissionDetails] = None
 
 
 @dataclass
@@ -273,7 +277,7 @@ class SapAlternativeWall:
     sheltered_wall: Optional[str] = None
 
 
-@dataclass
+@dataclass(kw_only=True)
 class SapBuildingPart:
     identifier: str
     wall_dry_lined: str
@@ -287,7 +291,7 @@ class SapBuildingPart:
     party_wall_construction: Union[int, str]
     wall_thickness_measured: str
     roof_insulation_location: Union[int, str]
-    roof_insulation_thickness: Union[str, int]
+    roof_insulation_thickness: Optional[Union[str, int]] = None
     sap_room_in_roof: Optional[SapRoomInRoof] = None
     sap_alternative_wall_1: Optional[SapAlternativeWall] = None
     sap_alternative_wall_2: Optional[SapAlternativeWall] = None
@@ -382,7 +386,9 @@ class RenewableHeatIncentive:
     impact_of_solid_wall_insulation: Optional[int] = None
 
 
-@dataclass
+# kw_only so optional fields (aligned with 21.0.1) can be defaulted in place
+# without reordering; parsed only via `from_dict` (keyword construction).
+@dataclass(kw_only=True)
 class RdSapSchema21_0_0:
     uprn: int
     roofs: List[EnergyElement]
@@ -414,7 +420,7 @@ class RdSapSchema21_0_0:
     assessment_type: str
     completion_date: str
     inspection_date: str
-    wet_rooms_count: int
+    wet_rooms_count: Optional[int] = None
     extensions_count: int
     measurement_type: int
     total_floor_area: int
@@ -425,7 +431,7 @@ class RdSapSchema21_0_0:
     sap_energy_source: SapEnergySource
     secondary_heating: EnergyElement
     sap_building_parts: List[SapBuildingPart]
-    open_chimneys_count: int
+    open_chimneys_count: Optional[int] = None
     solar_water_heating: str
     habitable_room_count: int
     heating_cost_current: float
@@ -438,10 +444,10 @@ class RdSapSchema21_0_0:
     has_hot_water_cylinder: str
     heating_cost_potential: float
     hot_water_cost_current: float
-    insulated_door_u_value: float
+    insulated_door_u_value: Optional[float] = None
     mechanical_ventilation: int
     percent_draughtproofed: int
-    suggested_improvements: List[SuggestedImprovement]
+    suggested_improvements: Optional[List[SuggestedImprovement]] = None
     co2_emissions_potential: float
     energy_rating_potential: int
     lighting_cost_potential: float
@@ -449,27 +455,27 @@ class RdSapSchema21_0_0:
     hot_water_cost_potential: float
     renewable_heat_incentive: RenewableHeatIncentive
     draughtproofed_door_count: int
-    mechanical_vent_duct_type: int
-    windows_transmission_details: WindowsTransmissionDetails
-    cfl_fixed_lighting_bulbs_count: int
+    mechanical_vent_duct_type: Optional[int] = None
+    windows_transmission_details: Optional[WindowsTransmissionDetails] = None
+    cfl_fixed_lighting_bulbs_count: Optional[int] = None
     energy_consumption_current: int
     has_fixed_air_conditioning: str
     multiple_glazed_proportion: int
     calculation_software_version: str
     energy_consumption_potential: int
     environmental_impact_current: int
-    led_fixed_lighting_bulbs_count: int
-    mechanical_vent_duct_placement: int
-    mechanical_vent_duct_insulation: int
+    led_fixed_lighting_bulbs_count: Optional[int] = None
+    mechanical_vent_duct_placement: Optional[int] = None
+    mechanical_vent_duct_insulation: Optional[int] = None
     potential_energy_efficiency_band: str
-    pressure_test_certificate_number: int
-    mechanical_ventilation_index_number: int
+    pressure_test_certificate_number: Optional[int] = None
+    mechanical_ventilation_index_number: Optional[int] = None
     co2_emissions_current_per_floor_area: int
     current_energy_efficiency_band: str
     environmental_impact_potential: int
-    low_energy_fixed_lighting_bulbs_count: int
-    mechanical_vent_duct_insulation_level: int
-    mechanical_vent_measured_installation: str
+    low_energy_fixed_lighting_bulbs_count: Optional[int] = None
+    mechanical_vent_duct_insulation_level: Optional[int] = None
+    mechanical_vent_measured_installation: Optional[str] = None
     incandescent_fixed_lighting_bulbs_count: int
     sap_flat_details: Optional[SapFlatDetails] = None
     addendum: Optional[Addendum] = None

datatypes/epc/schema/sap_schema_17_1.py

@@ -171,6 +171,11 @@ class EnergyElement:
     environmental_efficiency_rating: int
 
 
+@dataclass
+class SapFlatDetails:
+    level: Optional[int] = None
+
+
 @dataclass
 class SapSchema17_1:
     uprn: int
@@ -184,7 +189,6 @@ class SapSchema17_1:
     postcode: str
     post_town: str
     inspection_date: str
-    has_hot_water_cylinder: str
     has_fixed_air_conditioning: str
     roofs: List[EnergyElement]
     walls: List[EnergyElement]
@@ -194,6 +198,37 @@ class SapSchema17_1:
     sap_heating: SapHeating
     sap_energy_source: SapEnergySource = field(default_factory=SapEnergySource)
     sap_ventilation: SapVentilation = field(default_factory=SapVentilation)
+    # SAP spec the cert was lodged under (~9.92 = SAP 2012 for this family).
+    # Drives Rebaselining trigger (a): sap_version < 10.2 → Effective = the
+    # SAP-10.2 calc, not the lodged figure (ADR-0037). Optional defensively.
+    sap_version: Optional[float] = None
+    # gov property_type code (0 house, 1 bungalow, 2 flat, 3 maisonette, 4 park
+    # home). Feeds the calculator's house/flat heat-transmission split; left
+    # absent the dwelling is classified as neither → mis-scored (ADR-0037).
+    property_type: Optional[int] = None
+    # gov built_form code (1 detached … 6 enclosed mid-terrace). Carried for the
+    # property-details panel and exposed-side handling (ADR-0037).
+    built_form: Optional[int] = None
+    # Lodged secondary heating element (description + ratings); "None" when the
+    # cert has no secondary. Read by the calculator's cert_to_inputs — dropping
+    # it under-counts a real secondary heater (ADR-0037).
+    secondary_heating: Optional[EnergyElement] = None
+    # "SAP" for this family (vs "RdSAP"); carried for provenance / the panel.
+    assessment_type: Optional[str] = None
+    # Human-facing display elements (description + ratings) for the property-
+    # details panel — distinct from the measured/structural fields the calculator
+    # reads. Dropped by the WIP mapper → all "Unknown" on the FE (ADR-0037).
+    # Full-SAP lodges windows under the plural key (a single element, not a list).
+    main_heating: List[EnergyElement] = field(default_factory=list)
+    main_heating_controls: List[EnergyElement] = field(default_factory=list)
+    windows: Optional[EnergyElement] = None
+    lighting: Optional[EnergyElement] = None
+    hot_water: Optional[EnergyElement] = None
     # measured living-room area (m²); the engine consumes it via a back-solved
     # habitable_rooms_count (Table 27). Optional — 100% present in the corpus.
     living_area: Optional[Union[int, float]] = None
+    # Some 17.0 full-SAP certs omit the top-level flag and lodge it only under
+    # sap_heating; the mapper falls back to sap_heating.has_hot_water_cylinder.
+    has_hot_water_cylinder: Optional[str] = None
+    # Present for flat-type dwellings; absence means not a flat.
+    sap_flat_details: Optional[SapFlatDetails] = None

datatypes/epc/schema/tests/fixtures/sap_16_0_full.json

@@ -0,0 +1,435 @@
+{
+  "uprn": 10008075674,
+  "roofs": [
+    {
+      "description": "Average thermal transmittance 0.11 W/m\u00b2K",
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "walls": [
+    {
+      "description": "Average thermal transmittance 0.21 W/m\u00b2K",
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "floors": [
+    {
+      "description": "Average thermal transmittance 0.12 W/m\u00b2K",
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "status": "entered",
+  "windows": {
+    "description": "High performance glazing",
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "lighting": {
+    "description": "Low energy lighting in all fixed outlets",
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "postcode": "PE2 9HD",
+  "data_type": 2,
+  "hot_water": {
+    "description": "From main system, plus solar",
+    "energy_efficiency_rating": 5,
+    "environmental_efficiency_rating": 5
+  },
+  "post_town": "PETERBOROUGH",
+  "built_form": 1,
+  "created_at": "2012-05-30 13:30:25.000000",
+  "living_area": 18.011,
+  "orientation": 7,
+  "region_code": 2,
+  "report_type": 3,
+  "sap_heating": {
+    "thermal_store": 1,
+    "has_solar_panel": "true",
+    "water_fuel_type": 1,
+    "solar_store_volume": 100,
+    "water_heating_code": 901,
+    "hot_water_store_size": 210,
+    "main_heating_details": [
+      {
+        "burner_control": 3,
+        "main_fuel_type": 1,
+        "efficiency_type": 3,
+        "heat_emitter_type": 1,
+        "main_heating_code": 102,
+        "is_flue_fan_present": "false",
+        "main_heating_number": 1,
+        "is_condensing_boiler": "true",
+        "main_heating_control": 2110,
+        "is_interlocked_system": "true",
+        "main_heating_category": 2,
+        "main_heating_fraction": 1,
+        "gas_or_oil_boiler_type": 1,
+        "main_heating_flue_type": 2,
+        "main_heating_efficiency": 89,
+        "main_heating_data_source": 2,
+        "has_delayed_start_thermostat": "false",
+        "load_or_weather_compensation": 0,
+        "is_central_heating_pump_in_heated_space": "true"
+      }
+    ],
+    "has_hot_water_cylinder": "true",
+    "has_solar_powered_pump": "false",
+    "has_cylinder_thermostat": "true",
+    "solar_panel_aperture_area": 4.51,
+    "has_fixed_air_conditioning": "false",
+    "secondary_heating_category": 1,
+    "solar_panel_collector_type": 2,
+    "is_cylinder_in_heated_space": "true",
+    "solar_panel_collector_pitch": 2,
+    "is_hot_water_separately_timed": "true",
+    "is_primary_pipework_insulated": "true",
+    "hot_water_store_insulation_type": 1,
+    "hot_water_store_heat_loss_source": 3,
+    "is_solar_store_combined_cylinder": "true",
+    "solar_panel_collector_data_source": 2,
+    "solar_panel_collector_orientation": 5,
+    "solar_panel_collector_overshading": 1,
+    "hot_water_store_insulation_thickness": 85,
+    "solar_panel_collector_heat_loss_rate": 3.681,
+    "solar_panel_collector_zero_loss_efficiency": 77
+  },
+  "sap_version": 9.9,
+  "schema_type": "SAP-Schema-16.0",
+  "uprn_source": "Energy Assessor",
+  "country_code": "EAW",
+  "main_heating": [
+    {
+      "description": "Boiler and radiators, mains gas",
+      "energy_efficiency_rating": 4,
+      "environmental_efficiency_rating": 4
+    }
+  ],
+  "air_tightness": {
+    "description": "Air permeability 4.9 m\u00b3/h.m\u00b2 (as tested)",
+    "energy_efficiency_rating": 4,
+    "environmental_efficiency_rating": 4
+  },
+  "dwelling_type": "Detached house",
+  "language_code": 1,
+  "property_type": 0,
+  "address_line_1": "43, New Road",
+  "address_line_2": "Woodston",
+  "schema_version": "LIG-15.0",
+  "assessment_date": "2012-05-30",
+  "assessment_type": "SAP",
+  "completion_date": "2012-05-30",
+  "inspection_date": "2012-05-30",
+  "sap_ventilation": {
+    "psv_count": 0,
+    "pressure_test": 1,
+    "air_permeability": 4.88,
+    "open_flues_count": 0,
+    "ventilation_type": 1,
+    "extract_fans_count": 3,
+    "open_fireplaces_count": 0,
+    "sheltered_sides_count": 2,
+    "flueless_gas_fires_count": 0
+  },
+  "design_water_use": 1,
+  "total_floor_area": 76,
+  "transaction_type": 6,
+  "conservatory_type": 1,
+  "registration_date": "2012-05-30",
+  "restricted_access": 0,
+  "sap_energy_source": {
+    "electricity_tariff": 1,
+    "wind_turbines_count": 0,
+    "wind_turbine_terrain_type": 2,
+    "fixed_lighting_outlets_count": 0,
+    "low_energy_fixed_lighting_outlets_count": 0,
+    "low_energy_fixed_lighting_outlets_percentage": 100
+  },
+  "sap_opening_types": [
+    {
+      "name": "Door (1)",
+      "type": 1,
+      "u_value": 1.8,
+      "data_source": 2,
+      "description": "Data from Manufacturer",
+      "glazing_type": 1
+    },
+    {
+      "name": "Windows (1)",
+      "type": 4,
+      "u_value": 1.5,
+      "data_source": 2,
+      "description": "Data from Manufacturer",
+      "frame_factor": 0.7,
+      "glazing_type": 7,
+      "solar_transmittance": 0.63
+    }
+  ],
+  "secondary_heating": {
+    "description": "None",
+    "energy_efficiency_rating": 0,
+    "environmental_efficiency_rating": 0
+  },
+  "lzc_energy_sources": [
+    10
+  ],
+  "sap_building_parts": [
+    {
+      "sap_roofs": [
+        {
+          "name": "main roof",
+          "u_value": 0.11,
+          "roof_type": 2,
+          "kappa_value": 9,
+          "total_roof_area": 38.65
+        },
+        {
+          "name": "flat roof",
+          "u_value": 0.19,
+          "roof_type": 2,
+          "kappa_value": 9,
+          "total_roof_area": 1.39
+        }
+      ],
+      "sap_walls": [
+        {
+          "name": "brick wall",
+          "u_value": 0.21,
+          "wall_type": 2,
+          "kappa_value": 150,
+          "total_wall_area": 115.98,
+          "is_curtain_walling": "false"
+        },
+        {
+          "name": "downstairs",
+          "u_value": 0,
+          "wall_type": 5,
+          "kappa_value": 9,
+          "total_wall_area": 44.7
+        },
+        {
+          "name": "upper",
+          "u_value": 0,
+          "wall_type": 5,
+          "kappa_value": 9,
+          "total_wall_area": 52.99
+        }
+      ],
+      "identifier": "Main Dwelling",
+      "overshading": 2,
+      "sap_openings": [
+        {
+          "name": 1,
+          "type": "Door (1)",
+          "width": 0.9,
+          "height": 2.1,
+          "location": "brick wall",
+          "orientation": 7
+        },
+        {
+          "name": 2,
+          "type": "Windows (1)",
+          "width": 0.665,
+          "height": 1.05,
+          "location": "brick wall",
+          "orientation": 1
+        },
+        {
+          "name": 3,
+          "type": "Windows (1)",
+          "width": 2.09,
+          "height": 1.05,
+          "location": "brick wall",
+          "orientation": 7
+        },
+        {
+          "name": 4,
+          "type": "Windows (1)",
+          "width": 0.665,
+          "height": 1.05,
+          "location": "brick wall",
+          "orientation": 5
+        },
+        {
+          "name": 5,
+          "type": "Windows (1)",
+          "width": 2.5,
+          "height": 2.1,
+          "location": "brick wall",
+          "orientation": 3
+        },
+        {
+          "name": 6,
+          "type": "Windows (1)",
+          "width": 1.05,
+          "height": 1.425,
+          "location": "brick wall",
+          "orientation": 7
+        },
+        {
+          "name": 7,
+          "type": "Windows (1)",
+          "width": 0.6,
+          "height": 1.2,
+          "location": "brick wall",
+          "orientation": 1
+        },
+        {
+          "name": 8,
+          "type": "Windows (1)",
+          "width": 0.8,
+          "height": 1.425,
+          "location": "brick wall",
+          "orientation": 3
+        },
+        {
+          "name": 9,
+          "type": "Windows (1)",
+          "width": 0.8,
+          "height": 1.425,
+          "location": "brick wall",
+          "orientation": 3
+        },
+        {
+          "name": 10,
+          "type": "Windows (1)",
+          "width": 0.6,
+          "height": 1.05,
+          "location": "brick wall",
+          "orientation": 5
+        }
+      ],
+      "construction_year": 2012,
+      "sap_thermal_bridges": {
+        "thermal_bridges": [
+          {
+            "length": 9.77,
+            "psi_value": 0.3,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 2
+          },
+          {
+            "length": 7.27,
+            "psi_value": 0.04,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 3
+          },
+          {
+            "length": 23.55,
+            "psi_value": 0.05,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 4
+          },
+          {
+            "length": 27.18,
+            "psi_value": 0.16,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 5
+          },
+          {
+            "length": 27.18,
+            "psi_value": 0.07,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 6
+          },
+          {
+            "length": 8.68,
+            "psi_value": 0.06,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 10
+          },
+          {
+            "length": 17.17,
+            "psi_value": 0.24,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 12
+          },
+          {
+            "length": 21.04,
+            "psi_value": 0.09,
+            "psi_value_source": 4,
+            "thermal_bridge_type": 16
+          }
+        ],
+        "thermal_bridge_code": 5
+      },
+      "building_part_number": 1,
+      "sap_floor_dimensions": [
+        {
+          "storey": 0,
+          "u_value": 0.12,
+          "floor_type": 2,
+          "storey_height": 2.55,
+          "heat_loss_area": 38.65,
+          "total_floor_area": 38.65,
+          "kappa_value_from_below": 18
+        },
+        {
+          "storey": 1,
+          "u_value": 0,
+          "floor_type": 3,
+          "kappa_value": 18,
+          "storey_height": 2.71,
+          "heat_loss_area": 1.3900032,
+          "total_floor_area": 37.26,
+          "kappa_value_from_below": 18
+        }
+      ]
+    }
+  ],
+  "bedf_revision_number": 323,
+  "heating_cost_current": 260,
+  "co2_emissions_current": 1.2,
+  "energy_rating_average": 60,
+  "energy_rating_current": 83,
+  "lighting_cost_current": 43,
+  "main_heating_controls": [
+    {
+      "description": "Time and temperature zone control",
+      "energy_efficiency_rating": 5,
+      "environmental_efficiency_rating": 5
+    }
+  ],
+  "has_hot_water_cylinder": "true",
+  "heating_cost_potential": 251,
+  "hot_water_cost_current": 37,
+  "suggested_improvements": [
+    {
+      "sequence": 1,
+      "typical_saving": 219,
+      "indicative_cost": "\u00a311,000 - \u00a320,000",
+      "improvement_type": "U",
+      "improvement_details": {
+        "improvement_number": 34
+      },
+      "improvement_category": 3,
+      "energy_performance_rating": 94,
+      "environmental_impact_rating": 97
+    }
+  ],
+  "co2_emissions_potential": 1.2,
+  "energy_rating_potential": 83,
+  "lighting_cost_potential": 43,
+  "hot_water_cost_potential": 47,
+  "is_in_smoke_control_area": "unknown",
+  "renewable_heat_incentive": {
+    "rhi_new_dwelling": {
+      "space_heating": 3010,
+      "water_heating": 2175
+    }
+  },
+  "seller_commission_report": "Y",
+  "energy_consumption_current": 84,
+  "has_fixed_air_conditioning": "false",
+  "calculation_software_version": "Version: 1.4.0.80",
+  "energy_consumption_potential": 84,
+  "environmental_impact_current": 87,
+  "current_energy_efficiency_band": "B",
+  "environmental_impact_potential": 87,
+  "has_heated_separate_conservatory": "false",
+  "potential_energy_efficiency_band": "B",
+  "co2_emissions_current_per_floor_area": 16
+}

deployment/terraform/lambda/hubspot_deal_etl/variables.tf

@@ -19,7 +19,7 @@ variable "image_digest" {
 
 variable "maximum_concurrency" {
   type        = number
-  default     = 2
+  default     = 20
   description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
 }
 

deployment/terraform/lambda/modelling_e2e/main.tf

@@ -0,0 +1,64 @@
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+  config = {
+    bucket = "assessment-model-terraform-state"
+    key    = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+
+data "aws_secretsmanager_secret_version" "db_credentials" {
+  secret_id = "${var.stage}/assessment_model/db_credentials"
+}
+
+locals {
+  db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
+
+  # Per-container minimum gap (seconds) between Google Solar API calls. The Solar
+  # API caps the whole GCP project at a hard 600 QPM shared across Building
+  # Insights + Data Layers. Each of the up-to-`maximum_concurrency` containers
+  # loops its ~50-property batch sequentially, so pacing each one to
+  # 0.8 (safety headroom) × 600 QPM ÷ 60 ÷ N keeps the fleet sum under the
+  # ceiling — eliminating the sustained 429 storm that per-process backoff alone
+  # can't ride out. Derived from `maximum_concurrency` so N has one source of
+  # truth. At N=32 → 4.0s. Consumed by the handler's throttle (see
+  # infrastructure/solar/google_solar_api_client.py).
+  solar_min_request_interval_seconds = var.maximum_concurrency / (0.8 * 600 / 60)
+}
+
+module "lambda" {
+  source = "../../modules/lambda_with_sqs"
+
+  name  = var.lambda_name
+  stage = var.stage
+
+  image_uri = local.image_uri
+
+  reserved_concurrent_executions = var.reserved_concurrent_executions
+
+  batch_size          = var.batch_size
+  maximum_concurrency = var.maximum_concurrency
+
+  timeout     = 900
+  memory_size = 3008
+
+  environment = {
+    STAGE                = var.stage
+    LOG_LEVEL            = "info"
+    POSTGRES_USERNAME    = local.db_credentials.db_assessment_model_username
+    POSTGRES_PASSWORD    = local.db_credentials.db_assessment_model_password
+    POSTGRES_HOST        = var.db_host
+    POSTGRES_DATABASE    = var.db_name
+    POSTGRES_PORT        = var.db_port
+    OPEN_EPC_API_TOKEN   = var.open_epc_api_token
+    GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
+    DATA_BUCKET          = "retrofit-data-${var.stage}"
+
+    SOLAR_MIN_REQUEST_INTERVAL_SECONDS = tostring(local.solar_min_request_interval_seconds)
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "modelling_e2e_s3_read" {
+  role       = module.lambda.role_name
+  policy_arn = data.terraform_remote_state.shared.outputs.modelling_e2e_s3_read_arn
+}

deployment/terraform/lambda/modelling_e2e/outputs.tf

@@ -0,0 +1,9 @@
+output "modelling_e2e_queue_url" {
+  value       = module.lambda.queue_url
+  description = "URL of the modelling-e2e SQS queue (pass to trigger_modelling_e2e_sqs.py --sqs-url)"
+}
+
+output "modelling_e2e_queue_arn" {
+  value       = module.lambda.queue_arn
+  description = "ARN of the modelling-e2e SQS queue"
+}

deployment/terraform/lambda/modelling_e2e/provider.tf

@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+  }
+
+  backend "s3" {
+    bucket = "modelling-e2e-terraform-state"
+    key    = "terraform.tfstate"
+    region = "eu-west-2"
+  }
+
+  required_version = ">= 1.2.0"
+}
+
+provider "aws" {
+  region = "eu-west-2"
+}

deployment/terraform/lambda/modelling_e2e/variables.tf

@@ -0,0 +1,65 @@
+variable "lambda_name" {
+  type        = string
+  description = "Logical name of the lambda"
+}
+
+variable "stage" {
+  description = "Deployment stage (e.g. dev, prod)"
+  type        = string
+}
+
+variable "ecr_repo_url" {
+  type        = string
+  description = "ECR repository URL (no tag, no digest)"
+}
+
+variable "image_digest" {
+  type        = string
+  description = "Image digest (sha256:...)"
+}
+
+variable "reserved_concurrent_executions" {
+  type        = number
+  default     = -1
+  description = "Reserved concurrency for the Lambda. -1 = unreserved."
+}
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = 12
+  description = "Maximum concurrent Lambda invocations from the SQS trigger."
+}
+
+variable "batch_size" {
+  type    = number
+  default = 1
+}
+
+variable "db_host" {
+  type      = string
+  sensitive = true
+}
+
+variable "db_name" {
+  type      = string
+  sensitive = true
+}
+
+variable "db_port" {
+  type      = string
+  sensitive = true
+}
+
+variable "open_epc_api_token" {
+  type      = string
+  sensitive = true
+}
+
+variable "google_solar_api_key" {
+  type      = string
+  sensitive = true
+}
+
+locals {
+  image_uri = "${var.ecr_repo_url}@${var.image_digest}"
+}

deployment/terraform/shared/main.tf

@@ -81,6 +81,10 @@ resource "aws_db_instance" "default" {
   publicly_accessible = true
   # Specify the CA certificate with the default RDS CA certificate
   ca_cert_identifier = "rds-ca-rsa2048-g1"
+
+  # Performance Insights (7-day retention is free)
+  performance_insights_enabled          = true
+  performance_insights_retention_period = 7
   # Temporary to enfore immediate change
   apply_immediately = true
   # Set up storage type to gp3 for better performance
@@ -858,3 +862,35 @@ module "sharepoint_renamer_registry" {
   stage  = var.stage
 }
 
+################################################
+# Modelling E2E – Lambda
+################################################
+module "modelling_e2e_state_bucket" {
+  source      = "../modules/tf_state_bucket"
+  bucket_name = "modelling-e2e-terraform-state"
+}
+
+module "modelling_e2e_registry" {
+  source = "../modules/container_registry"
+  name   = "modelling-e2e"
+  stage  = var.stage
+}
+
+module "modelling_e2e_s3_read" {
+  source = "../modules/s3_iam_policy"
+
+  policy_name        = "ModellingE2EReadS3"
+  policy_description = "Allow modelling-e2e Lambda to read spatial parquet from the data bucket"
+  bucket_arns        = ["arn:aws:s3:::retrofit-data-${var.stage}"]
+  actions            = ["s3:GetObject", "s3:ListBucket"]
+  resource_paths     = ["/*"]
+}
+
+output "modelling_e2e_s3_read_arn" {
+  value = module.modelling_e2e_s3_read.policy_arn
+}
+
+output "modelling_e2e_ecr_url" {
+  value = module.modelling_e2e_registry.repository_url
+}
+

docs/adr/0014-bill-derivation-from-real-fuel-rates.md

@@ -86,9 +86,7 @@ production migration is FE-owned (Drizzle); `docs/migrations/` updated.
   **stubs them at 0 kWh**, so the bill total currently understates by the unregulated electricity
   load. Khalim is adding the fields to `SapResult` directly; the adapter wires the
   `APPLIANCES`/`COOKING` sections in as soon as they land.
-- **Off-peak (Economy 7) day/night split** — the snapshot carries the E7 day/night rates, but
-  `FuelRates` exposes single-rate fuels only; the day/night accessor + the calculator's Table 12a
-  high/low-rate split land in a later slice.
+- ~~**Off-peak (Economy 7) day/night split**~~ — **resolved, see the 2026-06-24 amendment below.**
 - **Heat-network rate model** — heat-network certs raise `UnpricedFuel` for now (the one common gap).
 - **Regional rates + Ofgem-cap ETL** — national snapshot now; both are later refinements behind the
   same `FuelRatesRepository` port.
@@ -152,3 +150,17 @@ Bill Derivation is no longer Baseline-only — the **Modelling** stage now re-ru
 - **Plan-level first, per-measure savings next (telescoping cascade).** This slice fills the plan columns (`post_energy_bill`, `post_energy_consumption`, `energy_bill_savings`, `energy_consumption_savings`). Per-measure `recommendation.kwh_savings` / `energy_cost_savings` come from a **bill cascade over the role-3 best-practice order** (fabric → heating → renewables) — re-bill each cumulative prefix and diff, telescoping exactly to the plan totals (mirroring the SAP role-3 attribution; reuses the per-prefix `sap_result`s, no extra calls). Per-measure savings can be **negative** (ventilation increases energy) and still telescope. The legacy `recommendation.energy_savings` column is **vestigial** (legacy set it to `0`; the canonical delivered-energy field is `kwh_savings`) — left NULL.
 
 - **Limitation carried over.** The "Appliances + cooking kWh stubbed at 0" deferral above still applies — Modelling's post-package bill understates by the same unregulated-electricity load until those fields land on `SapResult`. Baseline and Modelling share the gap, so baseline-vs-post savings remain consistent.
+
+## Amendment (2026-06-24): off-peak is a whole-meter tariff, day/night split per end use from the calculator's Table 12a fractions
+
+Resolves the §4 / Deferred "off-peak day/night split" — forced by a `modelling_e2e` failure (`no rate for fuel ELECTRICITY_OFF_PEAK`) on an off-peak-metered dwelling, where the snapshot's E7 day/night entry was deliberately skipped (no single `unit_rate_p_per_kwh`) and `BillDerivation` raised `UnpricedFuel`. Decided in a `/grill-with-docs` session.
+
+- **Off-peak is a property of the METER, not of an end use.** An Economy-7-style dwelling has *one* dual-rate meter, and **every** electric end use — heating, hot water, *and* lighting / pumps-fans / appliances / cooking — bills on it (the calculator already prices the whole meter this way via Table 12a Grid 1 + Grid 2). The pre-amendment `EnergyBreakdown` modelled off-peak as a per-end-use *fuel* that only heating/HW could carry, hard-wiring the other electric lines to standard `ELECTRICITY` — a **latent under-pricing** of the daytime appliance/lighting load (standard 24.67p vs off-peak day 29.73p), the mirror of the heating over-pricing that folding-to-standard would cause. So the breakdown now models off-peak as a **whole-meter tariff**: on an off-peak meter, `from_sap_result` routes *all* electric lines to `Fuel.ELECTRICITY_OFF_PEAK`. This needs an **off-peak-meter signal on `SapResult`** (a fraction of 1.0 on the off-peak day rate is not the same as standard electricity).
+
+- **The day/night split is the calculator's existing Table 12a high-rate fraction, surfaced as a calculator output.** Each end use's **High-Rate Fraction** (fraction of its kWh at the day/high rate; remainder at night/low) is already computed inside `cert_to_inputs` (`_main_space_heating_high_rate_fraction`, `water_heating_high_rate_fraction`, the Table-13 immersion blend, the HP-DHW exception, `other_use_high_rate_fraction` for Grid 2) and folded into a blended *Table-32* cost the bill discards. Per the "fuel is a calculator output" amendment above, the **fraction is now also calculator output** — threaded `CalculatorInputs → SapResult` per end use — so the bill **reuses** it rather than re-deriving a second day/night model that would drift from the rating. Unregulated loads SAP does not rate (appliances, cooking) have no Table 12a fraction; they **inherit the `ALL_OTHER_USES` fraction** (same Grid-2 row + daytime-weighted profile as lighting) — a documented extension, since SAP itself never splits them.
+
+- **Representation: fraction-on-the-line; rates stay in `FuelRates`.** `EnergyLine` gains an optional `high_rate_fraction` (`None` = single-rate fuel). `FuelRates` gains a **day/night accessor** for `ELECTRICITY_OFF_PEAK` (the repository now reads the snapshot's existing `day_p_per_kwh` / `night_p_per_kwh`). `BillDerivation` prices an off-peak line at `kwh × (frac × day_rate + (1−frac) × night_rate)`. Rejected: (a) **pre-splitting kWh into day/night lines** upstream — leaks rate-tier structure into the calculator/adapter and doubles the section roll-up; (b) the **blended day/night rate handed over by the calculator** — that is the calculator pricing at real rates, the exact boundary §2 draws.
+
+- **Rejected shortcuts (both re-open closed decisions).** *Fold `ELECTRICITY_OFF_PEAK` → standard `ELECTRICITY`*: knowingly mis-prices in both directions (≈+45% on night-shifted storage heat, under-prices daytime loads). *A single hand-blended off-peak `unit_rate` in the snapshot*: reintroduces the legacy `AnnualBillSavings.py` **blended `PRICE_FACTOR`** anti-pattern this ADR's Context rewrote away from — the day/night weighting is per-end-use and per-dwelling, not a property of the rate.
+
+- **Safety: no-op for unaffected certs.** Standard-tariff dwellings are unchanged (every fraction is 1.0 and the carrier stays `ELECTRICITY`). Off-peak dwellings currently **hard-crash** (`UnpricedFuel` aborts the batch, §1), so there is no passing bill value to regress — they move from crash → priced. SAP scores are untouched: the bill is output-only and never feeds the rating.

docs/adr/0033-age-band-resolves-assumed-fabric-states.md

@@ -0,0 +1,136 @@
+# Age band resolves the "(assumed)"/"as built"/"Unknown" fabric states
+
+ADR-0032 dec-4 **deferred** the `WallType` "(assumed) insulated" / "partial
+insulation (assumed)" states (and, transitively, flat roofs): their
+`wall_insulation_type` is "age-inferred in RdSAP, so the code is ambiguous and
+must be pinned against the Elmhurst accuracy harness rather than guessed." This
+ADR closes that deferral. The harness has now run. Terms in CONTEXT.md (Landlord
+Overrides, Effective EPC, Validation Cohort); supersedes ADR-0032 dec-4.
+
+## Status
+
+Accepted. Supersedes the deferral in ADR-0032 dec-4. Evidence:
+`scripts/hyde/uvalue_probe_{walls,roofs}.csv` (full A–M Elmhurst sweep, produced
+by `scripts/hyde/probe_uvalues.py`).
+
+## Context: what the Elmhurst sweep showed
+
+The SAP 10.2 spec defers RdSAP to "a separate document" (Appendix S, p.122), so
+the authoritative source for "what U-value does enum X at age Y produce" is the
+accredited Elmhurst RdSAP-10 entry tool itself. We drove it across all 13
+construction-age bands (A–M) for the two components Khalim flagged and read the
+tool's recomputed **Default U-value** for every option:
+
+* **Cavity wall, As Built** `[A–M]`:
+  `1.5, 1.5, 1.5, 1.5, 1.5, 1.0, 0.6, 0.6, 0.45, 0.35, 0.3, 0.28, 0.26`
+* **Cavity wall, Filled** `[A–M]`:
+  `0.7, 0.7, 0.7, 0.7, 0.7, 0.4, 0.35, 0.35, 0.45, 0.35, 0.3, 0.28, 0.26`
+* **Flat roof, As Built / Unknown** `[A–M]`:
+  `2.3, 2.3, 2.3, 2.3, 1.5, 0.68, 0.4, 0.35, 0.35, 0.25, 0.25, 0.18, 0.15`
+
+Two facts fall out:
+
+1. **Every value already lives in our tables.** As Built ≡ `(WALL_CAVITY, 0)`,
+   Filled ≡ `_CAVITY_FILLED_ENG`, flat As Built/Unknown ≡ `_FLAT_ROOF_BY_AGE`
+   (`domain/sap10_ml/rdsap_uvalues.py`) — exact match, all 13 bands. The U-value
+   layer was never the gap.
+2. **"Unknown" computes identically to "As Built"** at every band — confirming
+   Khalim's read that Unknown is handled like As Built.
+
+## Decisions
+
+### 1. The "(assumed)" qualifier is *not* a distinct insulation code — age band is
+
+ADR-0032 dec-4 worried that "partial insulation (assumed)" needs a
+`wall_insulation_type` code we couldn't name. The sweep shows why none exists:
+Elmhurst has **no "partial" enum**. "Partial insulation (assumed)" is simply
+*As Built (code 4) evaluated at band F (1976–1982)* → 1.0 W/m²K. Likewise
+"no insulation (assumed)" is As Built at an old band (A–E → 1.5) and "insulated
+(assumed)" is As Built at a newer band (G+ → ≤0.60). All three resolve to
+`wall_insulation_type = 4`; the **construction-age-band overlay** (already a
+separate, wired override component) supplies the U-value. This matches Khalim's
+bands: cavity ≤1975 uninsulated / 1976–82 partial / later insulated is exactly
+the As-Built-by-age curve above.
+
+### 2. Resolve to int codes, not a computed U-value override
+
+Because int codes + the existing age-band tables reproduce Elmhurst exactly,
+overlays keep emitting `wall_construction` / `wall_insulation_type` (walls) and
+`roof_construction` / `roof_insulation_thickness` (roofs) — no new
+`wall_u_value` / `roof_u_value` override path. This keeps the landlord overlay on
+the same code path as every Measure (ADR-0032 dec-1) and avoids a second way to
+move the wall/roof U-value. (Considered and rejected: writing a computed U-value
+override. It would duplicate the RdSAP tables already in the calculator and
+diverge the moment those tables are revised.)
+
+### 3. Walls — extend the state map; no new machinery
+
+`wall_type_overlay._STATE_INSULATION` gains the three "as built" variants
+("as built", "as built, insulated (assumed)", "as built, partial insulation
+(assumed)") → code 4, alongside the existing "no insulation (assumed)" → 4.
+These previously produced **no overlay** (silently dropping the landlord's
+correction). The age-band overlay does the rest.
+
+### 4. Flat roofs — the overlay must set `roof_construction_type`
+
+The calculator's flat-roof age-band path (`u_roof`, `_FLAT_ROOF_BY_AGE`) fires
+only when the roof is flat, which `heat_transmission.py` reads from the **string**
+`part.roof_construction_type` (`"flat" in roof_type_lower`, line 999) — *not* the
+int `roof_construction`. The roof overlay today sets only
+`roof_insulation_thickness`, so a flat-roof override can't reach that path. This
+ADR adds `roof_construction_type` to `BuildingPartOverlay` and has
+`roof_type_overlay` set it to `"Flat"` for the "Flat, …" family, alongside the
+existing `roof_insulation_thickness` (set when the description carries an explicit
+mm depth).
+
+The same shape-flag mechanism extends to pitched roofs with no clean depth:
+
+* `"Pitched, Unknown loft insulation"` → `roof_construction_type="Pitched"`,
+  thickness left `None` → the pitched age-band default (`_ROOF_BY_AGE`).
+* `"Pitched, no insulation*"` → `roof_construction_type="Pitched"` **plus**
+  `roof_insulation_thickness=0` → the uninsulated 2.30 (Table 16 row 0). Pitched
+  "no insulation" text is load-bearing in `u_roof` (§5.11 note) — it does NOT take
+  the age-band default, unlike flat. Setting `0` (not `None`) also lets this case
+  override a lodged numeric thickness, which the "Unknown" cases cannot.
+
+Constraint: the applicator only folds *non-None* overlay fields, so it cannot
+*clear* a lodged `roof_insulation_thickness` to `None`. For As Built / Unknown
+(no depth) the age-band default fires only when the baseline part carries no
+numeric thickness — true on the Hyde flow, where the landlord override IS the roof
+source and genuine As Built/Unknown lodgements carry `AB`/`ND`/None, not a number.
+A flat/pitched-Unknown override layered on a part that already has a numeric
+thickness keeps that thickness; that case is out of scope here and surfaced by the
+coverage audit (below) for the verify step.
+
+## Consequences
+
+* Previously-dropped Hyde rows now move the score: ~1,366 flat-roof overrides
+  (`Flat: As Built` + `Flat: Unknown`) and every cavity "as built/partial/
+  insulated (assumed)" wall. This is the point — a more accurate Effective EPC —
+  and (per ADR-0032 dec-5) such overlaid Properties stay excluded from the
+  Validation Cohort on the divergence signal.
+* `BuildingPartOverlay` gains a `roof_construction` field; `apply_simulations`
+  must fold it like `wall_construction`.
+* The decision is pinned by tests mirroring the probe CSVs (one row per
+  component × band), so a future RdSAP table revision that breaks parity trips a
+  regression rather than silently drifting from Elmhurst.
+* `scripts/hyde/probe_uvalues.py` stays as the re-runnable oracle: re-sweep to
+  re-confirm parity whenever the RdSAP tables or Elmhurst change.
+* A **coverage-audit pair** guards against the silent-no-op class this ADR fixes,
+  and is the reusable gate for every future portfolio (Hyde is the template):
+  - `scripts/hyde/audit_override_coverage.py` — enum-level: runs every override
+    value through its overlay mapper and flags those that produce no overlay or
+    only an inert field. Run it whenever an override enum or overlay changes.
+  - `scripts/hyde/audit_hyde_rows.py` — row-level: weights the verdict by actual
+    portfolio row counts and cross-checks that every "another/same dwelling above"
+    party-ceiling no-op is genuinely a Flat/Maisonette (a House/Bungalow with a
+    dwelling above is a data error, not a safe no-op — flagged to CSV for review).
+  Contradictory properties (party-ceiling roof on a non-flat type — Hyde had 50,
+  possibly houses split into flats) are **skipped entirely** by the `write` step:
+  no overrides are written for them and their org_refs are recorded to
+  `skipped_contradictory_properties.csv` for joint human review, rather than
+  guessing which of the two conflicting landlord fields is correct.
+  Known residue logged, not silently dropped: `PitchedWithSlopingCeiling: *`
+  (~272 Hyde rows, needs the col-3 path), `Roof room(s)` (needs `sap_room_in_roof`),
+  the `Renewables` column (no ComponentSpec — PV unmapped), and `Timber frame,
+  with additional insulation` (wall, ~7 rows).