From 6026944fe5c116638415cee75ccc3cad6155a4ad Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 00:40:37 +0000 Subject: [PATCH 01/27] feat(epc): read new EPC graph for new-approach properties via resolver The new modelling pipeline no longer writes property_details_epc. Route all EPC reads through src/lib/services/epcSources.ts, which branches on the cutoff (property.updated_at >= 2026-06-01): legacy rows keep reading property_details_epc; new-approach rows read epc_property (+ children), property_baseline_performance, and plan/recommendation. - per-property: resolveConditionReport (building passport), resolveDetailsEpcMeta (/api/property-meta) - aggregates: shared carbon/bills/energy/estimated/expired/floor-area/ lodgement/mainfuel SQL fragments + newApproachJoins, wired into the portfolio list + filters, reporting metrics, and both scenario metrics routes estimated = predicted-only EPC. Gracefully omits gap fields for new properties (mainfuel, energyTariff, floorHeight, installed-measures bill adjustment, SAP-05 likely-downgrade). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario/[scenarioId]/metrics/route.ts | 14 +- .../scenario/default/metrics/route.ts | 14 +- .../api/property-meta/[propertyId]/route.ts | 22 +- .../reporting/databaseFunctions.ts | 69 +-- .../building-passport/[propertyId]/utils.ts | 27 +- src/app/portfolio/[slug]/utils.ts | 55 ++- src/lib/services/epcSources.ts | 419 ++++++++++++++++++ 7 files changed, 527 insertions(+), 93 deletions(-) create mode 100644 src/lib/services/epcSources.ts diff --git a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts index 8fc292b6..702ebeda 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts @@ -3,6 +3,7 @@ import { sql } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { sapToEpc } from "@/app/utils"; import type { PortfolioGoalType } from "@/app/db/schema/portfolio"; +import { newApproachJoins, carbonSql, billsSql } from "@/lib/services/epcSources"; /* ======================= Types @@ -208,26 +209,19 @@ export async function GET( /* ---------- Carbon ---------- */ CASE WHEN lp.id IS NOT NULL THEN lp.post_co2_emissions - ELSE e.co2_emissions + ELSE ${carbonSql(sql`e`)} END AS effective_carbon, /* ---------- Bills ---------- */ CASE WHEN lp.id IS NOT NULL THEN lp.post_energy_bill - ELSE ( - e.heating_cost_current + - e.hot_water_cost_current + - e.lighting_cost_current + - e.appliances_cost_current + - e.gas_standing_charge + - e.electricity_standing_charge - - COALESCE(e.installed_measures_total_energy_bill_adjustment, 0) - ) + ELSE ${billsSql(sql`e`)} END AS effective_bills FROM property p LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} LEFT JOIN LATERAL ( SELECT * diff --git a/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts index 6d5df36b..a1ed7257 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts @@ -3,6 +3,7 @@ import { sql } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { sapToEpc } from "@/app/utils"; import type { PortfolioGoalType } from "@/app/db/schema/portfolio"; +import { newApproachJoins, carbonSql, billsSql } from "@/lib/services/epcSources"; /* ======================= Types @@ -150,26 +151,19 @@ export async function GET( /* ---------- Carbon ---------- */ CASE WHEN lp.id IS NOT NULL THEN lp.post_co2_emissions - ELSE e.co2_emissions + ELSE ${carbonSql(sql`e`)} END AS effective_carbon, /* ---------- Bills ---------- */ CASE WHEN lp.id IS NOT NULL THEN lp.post_energy_bill - ELSE ( - e.heating_cost_current + - e.hot_water_cost_current + - e.lighting_cost_current + - e.appliances_cost_current + - e.gas_standing_charge + - e.electricity_standing_charge - - COALESCE(e.installed_measures_total_energy_bill_adjustment, 0) - ) + ELSE ${billsSql(sql`e`)} END AS effective_bills FROM property p LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} LEFT JOIN LATERAL ( SELECT * diff --git a/src/app/api/property-meta/[propertyId]/route.ts b/src/app/api/property-meta/[propertyId]/route.ts index 81f47ae7..f526544d 100644 --- a/src/app/api/property-meta/[propertyId]/route.ts +++ b/src/app/api/property-meta/[propertyId]/route.ts @@ -1,6 +1,7 @@ import { db } from "@/app/db/db"; import { property } from "@/app/db/schema/property"; import { serializeBigInt } from "@/app/utils"; +import { resolveDetailsEpcMeta } from "@/lib/services/epcSources"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; @@ -31,20 +32,21 @@ export async function GET(request: NextRequest, props: { params: Promise<{ prope currentValuation: true, }, where: eq(property.id, BigInt(propertyId)), - with: { - detailsEpc: { - columns: { - currentEnergyDemand: true, - co2Emissions: true, - estimated: true, - }, - }, - }, }); if (!propertyMeta) { return new NextResponse(null, { status: 404 }); } - return new NextResponse(JSON.stringify(propertyMeta, serializeBigInt)); + // `detailsEpc` is resolved via the cutoff: legacy properties read + // property_details_epc; new-approach ones read epc_property + + // property_baseline_performance. See src/lib/services/epcSources.ts. + const detailsEpc = await resolveDetailsEpcMeta( + propertyMeta.id, + propertyMeta.updatedAt, + ); + + return new NextResponse( + JSON.stringify({ ...propertyMeta, detailsEpc }, serializeBigInt), + ); } diff --git a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts index 7dbf1216..c1048dd5 100644 --- a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts +++ b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts @@ -1,6 +1,15 @@ import { db } from "@/app/db/db"; import { sql, eq } from "drizzle-orm"; import { scenario } from "@/app/db/schema/recommendations"; +import { + NEW_APPROACH_CUTOFF, + newApproachJoins, + carbonSql, + billsSql, + energyConsumptionSql, + estimatedSql, + isExpiredSql, +} from "@/lib/services/epcSources"; import type { AverageMetrics, @@ -28,19 +37,12 @@ export async function getAverages( const result = await db.execute(sql` SELECT AVG(p.current_sap_points)::float AS avg_sap, - AVG(e.co2_emissions)::float AS avg_carbon, - AVG( - e.heating_cost_current + - e.hot_water_cost_current + - e.lighting_cost_current + - e.appliances_cost_current + - e.gas_standing_charge + - e.electricity_standing_charge - - COALESCE(e.installed_measures_total_energy_bill_adjustment, 0) - )::float AS avg_bills, - AVG(e.primary_energy_consumption)::float AS avg_energy_consumption + AVG(${carbonSql(sql`e`)})::float AS avg_carbon, + AVG(${billsSql(sql`e`)})::float AS avg_bills, + AVG(${energyConsumptionSql(sql`e`)})::float AS avg_energy_consumption FROM property p LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} WHERE p.portfolio_id = ${portfolioId}; `); @@ -50,18 +52,11 @@ export async function getAverages( export async function getTotals(portfolioId: number): Promise { const result = await db.execute(sql` SELECT - SUM(e.co2_emissions)::float AS total_carbon, - SUM( - e.heating_cost_current + - e.hot_water_cost_current + - e.lighting_cost_current + - e.appliances_cost_current + - e.gas_standing_charge + - e.electricity_standing_charge - - COALESCE(e.installed_measures_total_energy_bill_adjustment, 0) - )::float AS total_bills + SUM(${carbonSql(sql`e`)})::float AS total_carbon, + SUM(${billsSql(sql`e`)})::float AS total_bills FROM property p LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} WHERE p.portfolio_id = ${portfolioId}; `); @@ -104,14 +99,15 @@ export async function getCountByEpcBand( SELECT COALESCE(p.current_epc_rating::text, 'Unknown') AS epc, COUNT(*) FILTER ( - WHERE e.estimated = false OR e.estimated IS NULL + WHERE ${estimatedSql(sql`e`)} = false )::int AS actual, COUNT(*) FILTER ( - WHERE e.estimated = true + WHERE ${estimatedSql(sql`e`)} = true )::int AS estimated FROM property p LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} WHERE p.portfolio_id = ${portfolioId} GROUP BY epc ) q @@ -136,10 +132,12 @@ export async function getEstimatedCounts( ): Promise { const result = await db.execute(sql` SELECT - SUM(CASE WHEN e.estimated = true THEN 1 ELSE 0 END)::int AS estimated, - SUM(CASE WHEN e.estimated = false THEN 1 ELSE 0 END)::int AS actual - FROM property_details_epc e - WHERE e.portfolio_id = ${portfolioId}; + SUM(CASE WHEN ${estimatedSql(sql`e`)} = true THEN 1 ELSE 0 END)::int AS estimated, + SUM(CASE WHEN ${estimatedSql(sql`e`)} = false THEN 1 ELSE 0 END)::int AS actual + FROM property p + LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} + WHERE p.portfolio_id = ${portfolioId}; `); return result.rows[0]; @@ -163,14 +161,16 @@ export async function getExpiredEpcCount(portfolioId: number): Promise { const result = await db.execute<{ expired: number }>(sql` SELECT SUM( - CASE - WHEN is_expired = true AND estimated = false - THEN 1 - ELSE 0 + CASE + WHEN ${isExpiredSql(sql`e`)} = true AND ${estimatedSql(sql`e`)} = false + THEN 1 + ELSE 0 END )::int AS expired - FROM property_details_epc - WHERE portfolio_id = ${portfolioId}; + FROM property p + LEFT JOIN property_details_epc e ON e.property_id = p.id + ${newApproachJoins} + WHERE p.portfolio_id = ${portfolioId}; `); return result.rows[0].expired; @@ -186,6 +186,9 @@ export async function getLikelyDowngrades( JOIN property_details_epc e ON e.property_id = p.id WHERE p.portfolio_id = ${portfolioId} + -- GAP: the SAP-05 "likely downgrade" signal has no equivalent in the new + -- pipeline (handled there via Rebaseline). Restrict to legacy properties. + AND p.updated_at < ${NEW_APPROACH_CUTOFF}::date AND e.sap_05_overwritten = true AND p.current_sap_points IS NOT NULL AND e.sap_05_score IS NOT NULL diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index c56b477c..0ab77e7b 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -13,14 +13,16 @@ import { surveyDB } from "@/app/db/surveyDB/connection"; import { Feature, GeneralFeature, - PropertyDetailsEpc, PropertyDetailsSpatial, PropertyMeta, - propertyDetailsEpc, propertyDetailsSpatial, NonIntrusiveSurveyData, nonInstrusiveSurvey, } from "@/app/db/schema/property"; +import { + ConditionReport, + resolveConditionReport, +} from "@/lib/services/epcSources"; import { getRating } from "@/app/utils"; import { eq, desc, and } from "drizzle-orm"; import { @@ -250,16 +252,11 @@ export async function getPropertyMeta( export async function getConditionReport( propertyId: string -): Promise { - const data = await db.query.propertyDetailsEpc.findFirst({ - where: eq(propertyDetailsEpc.propertyId, BigInt(propertyId)), - }); - - if (!data) { - throw new Error("Network response was not ok"); - } - - return data; +): Promise { + // Branches on the new-approach cutoff: legacy properties read + // property_details_epc; new-approach ones read the epc_property graph + + // property_baseline_performance. See src/lib/services/epcSources.ts. + return resolveConditionReport(propertyId); } export async function getSpatialData( @@ -298,7 +295,7 @@ export async function getNonIntrusiveSurvey( } export function formatGeneralFeatures( - conditionReportData: PropertyDetailsEpc, + conditionReportData: ConditionReport, propertyType: string ): GeneralFeature[] { // if a property is a flat/maisonette, we show heat loss coridoor information, otherwise we won't show it @@ -353,7 +350,7 @@ export function formatGeneralFeatures( } export function formatRetrofitFeatures( - conditionReportData: PropertyDetailsEpc + conditionReportData: ConditionReport ): Feature[] { // Helper function to convert rating number to string @@ -437,7 +434,7 @@ export function formatRetrofitFeatures( } export function formatHeatDemandFeatures( - conditionReportData: PropertyDetailsEpc + conditionReportData: ConditionReport ): GeneralFeature[] { return [ { diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index e3676c89..cf4ccfac 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -19,6 +19,14 @@ import { ScenarioSelect, } from "@/app/db/schema/recommendations"; import { sql } from "drizzle-orm"; +import { + newApproachJoins, + carbonSql, + totalFloorAreaSql, + lodgementDateSql, + isExpiredSql, + mainfuelSql, +} from "@/lib/services/epcSources"; import { FilterGroups, PropertyFilter, @@ -528,8 +536,11 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul } case "epcExpiryDate": { - const expiryExpr = sql`(epc.lodgement_date + INTERVAL '10 years')`; - const guard = sql`epc.lodgement_date IS NOT NULL`; + // Branches on the cutoff: new-approach properties derive the lodgement + // date from the lodged epc_property row. See epcSources.ts. + const lodgementExpr = lodgementDateSql(sql`epc`); + const expiryExpr = sql`(${lodgementExpr} + INTERVAL '10 years')`; + const guard = sql`${lodgementExpr} IS NOT NULL`; if (filter.operator === "date_before") { return sql`${guard} AND ${expiryExpr} < ${filter.value}::date`; @@ -543,7 +554,7 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul if (filter.operator === "date_preset") { switch (filter.value) { case "expired": - return sql`epc.is_expired = true`; + return sql`${isExpiredSql(sql`epc`)} = true`; case "expires_this_year": return sql`${guard} AND EXTRACT(YEAR FROM ${expiryExpr}) = EXTRACT(YEAR FROM CURRENT_DATE)`; case "expires_within_1_year": @@ -578,7 +589,9 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul builtForm: sql`p.built_form`, tenure: sql`p.tenure`, yearBuilt: sql`p.year_built`, - mainfuel: sql`epc.mainfuel`, + // GAP: new-approach properties have no main-fuel string → NULL, so they + // only match the "no value" filter branch. See epcSources.ts. + mainfuel: mainfuelSql(sql`epc`), }; const col = colMap[filter.field]; @@ -616,18 +629,20 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul case "floorArea": { const n = parseFloat(filter.value); if (isNaN(n)) return null; - if (filter.operator === "num_gte") return sql`epc.total_floor_area >= ${n}`; - if (filter.operator === "num_lte") return sql`epc.total_floor_area <= ${n}`; - if (filter.operator === "num_equals") return sql`epc.total_floor_area = ${n}`; + const area = totalFloorAreaSql(sql`epc`); + if (filter.operator === "num_gte") return sql`${area} >= ${n}`; + if (filter.operator === "num_lte") return sql`${area} <= ${n}`; + if (filter.operator === "num_equals") return sql`${area} = ${n}`; return null; } case "co2Emissions": { const n = parseFloat(filter.value); if (isNaN(n)) return null; - if (filter.operator === "num_gte") return sql`epc.co2_emissions >= ${n}`; - if (filter.operator === "num_lte") return sql`epc.co2_emissions <= ${n}`; - if (filter.operator === "num_equals") return sql`epc.co2_emissions = ${n}`; + const carbon = carbonSql(sql`epc`); + if (filter.operator === "num_gte") return sql`${carbon} >= ${n}`; + if (filter.operator === "num_lte") return sql`${carbon} <= ${n}`; + if (filter.operator === "num_equals") return sql`${carbon} = ${n}`; return null; } } @@ -666,6 +681,7 @@ export async function getPropertiesCount( SELECT COUNT(DISTINCT p.id)::int AS count FROM property p LEFT JOIN property_details_epc epc ON epc.property_id = p.id + ${newApproachJoins} LEFT JOIN plan pl ON pl.property_id = p.id AND pl.is_default = true WHERE p.portfolio_id = ${portfolioId} ${combinedWhere} @@ -706,11 +722,11 @@ export async function getProperties( p.built_form AS "builtForm", p.tenure AS tenure, p.year_built AS "yearBuilt", - epc.lodgement_date::text AS "epcLodgementDate", - epc.is_expired AS "epcIsExpired", - epc.total_floor_area AS "totalFloorArea", - epc.co2_emissions AS "co2Emissions", - epc.mainfuel AS mainfuel, + ${lodgementDateSql(sql`epc`)}::text AS "epcLodgementDate", + ${isExpiredSql(sql`epc`)} AS "epcIsExpired", + ${totalFloorAreaSql(sql`epc`)} AS "totalFloorArea", + ${carbonSql(sql`epc`)} AS "co2Emissions", + ${mainfuelSql(sql`epc`)} AS mainfuel, p.lexiscore AS lexiscore FROM property p LEFT JOIN property_targets t @@ -728,6 +744,7 @@ export async function getProperties( AND r.already_installed = false LEFT JOIN property_details_epc epc ON epc.property_id = p.id + ${newApproachJoins} WHERE p.portfolio_id = ${portfolioId} ${combinedWhere} GROUP BY @@ -748,11 +765,19 @@ export async function getProperties( p.built_form, p.tenure, p.year_built, + -- legacy + new-source columns referenced by the branched EPC expressions + p.updated_at, epc.lodgement_date, epc.is_expired, epc.total_floor_area, epc.co2_emissions, epc.mainfuel, + epl.id, + epp.id, + epl.registration_date, + epl.inspection_date, + epl.total_floor_area_m2, + bp.effective_co2_emissions_t_per_yr, p.lexiscore LIMIT ${limit} OFFSET ${offset}; `); diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts new file mode 100644 index 00000000..d57a9bae --- /dev/null +++ b/src/lib/services/epcSources.ts @@ -0,0 +1,419 @@ +/** + * EPC read-model resolver. + * + * The legacy plan engine wrote a flat per-property table `property_details_epc`. + * The new modelling pipeline (Hestia-Homes/Model) does NOT write that table — + * it writes the normalized EPC graph (`epc_property` + children), the modelling + * output (`plan` / `recommendation`) and `property_baseline_performance`. + * + * A property is "new approach" when `property.updated_at >= 2026-06-01` (the + * cutoff — the old pipeline predates it). For such properties the app must read + * the new sources and must NOT touch `property_details_epc`. Legacy properties + * keep reading the old table unchanged. + * + * This module is the single place that knows the cutoff and the field→source + * mapping, so callers (building-passport, property-meta, portfolio list, + * reporting + scenario metrics) branch consistently. + * + * Mapping applied (see the handover report for the full table and flagged gaps): + * - EPC fabric descriptions + 1–5 ratings → `epc_energy_element` + * - whole-dwelling counts / flags → `epc_property` (+ `epc_flat_details`) + * - baseline carbon / energy / bills → `property_baseline_performance` + * - post-retrofit data → `plan` / `recommendation` + * - "estimated" flag → property has only a `source='predicted'` + * EPC (no `source='lodged'` row) + * + * Gracefully-omitted gaps for new properties (no clean equivalent — reported to + * the owner): `mainfuel`, `energyTariff`, `floorHeight`, the installed-measures + * bill adjustment, and the SAP-05 "likely downgrade" metric. + */ +import { db } from "@/app/db/db"; +import { and, eq, sql } from "drizzle-orm"; +import { + epcProperty, + epcEnergyElement, + epcFlatDetails, + propertyBaselinePerformance, + propertyDetailsEpc, + property, +} from "@/app/db/schema/property"; + +/** The discriminator: properties updated on/after this date use the new sources. */ +export const NEW_APPROACH_CUTOFF = "2026-06-01"; + +export function isNewApproach(updatedAt: Date | string | null | undefined): boolean { + if (!updatedAt) return false; + const d = updatedAt instanceof Date ? updatedAt : new Date(updatedAt); + return d >= new Date(`${NEW_APPROACH_CUTOFF}T00:00:00.000Z`); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Shared SQL fragments for set-based aggregate queries. +// +// These assume the standard aliases: +// p = property +// bp = property_baseline_performance (new — baseline carbon/energy/bills) +// epl = epc_property WHERE source='lodged' +// epp = epc_property WHERE source='predicted' +// and take the legacy `property_details_epc` alias as a fragment so each query +// can keep its own (`e` vs `epc`). +// +// Add `newApproachJoins` to any query that uses one of the *Sql expressions. +// ───────────────────────────────────────────────────────────────────────────── + +type Alias = ReturnType; + +/** TRUE for new-approach rows. */ +export const isNewApproachSql = sql`p.updated_at >= ${NEW_APPROACH_CUTOFF}::date`; + +/** LEFT JOINs that expose the new sources alongside the legacy table. */ +export const newApproachJoins = sql` + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id + LEFT JOIN epc_property epl ON epl.property_id = p.id AND epl.source = 'lodged' + LEFT JOIN epc_property epp ON epp.property_id = p.id AND epp.source = 'predicted' +`; + +/** Baseline carbon (t CO₂/yr). New: effective performance. Legacy: e.co2_emissions. */ +export const carbonSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_co2_emissions_t_per_yr ELSE ${e}.co2_emissions END`; + +/** Primary energy intensity (kWh/m²/yr). New: effective performance. Legacy: e.primary_energy_consumption. */ +export const energyConsumptionSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_primary_energy_intensity_kwh_per_m2_yr ELSE ${e}.primary_energy_consumption END`; + +/** + * Annual energy bill (£). New: property_baseline_performance.total_annual_bill_gbp + * (NULL when the bill block hasn't been derived yet — excluded from AVG/SUM). + * Legacy: the old component-sum formula incl. standing charges and the + * installed-measures adjustment. + */ +export const billsSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN bp.total_annual_bill_gbp ELSE ( + ${e}.heating_cost_current + + ${e}.hot_water_cost_current + + ${e}.lighting_cost_current + + ${e}.appliances_cost_current + + ${e}.gas_standing_charge + + ${e}.electricity_standing_charge - + COALESCE(${e}.installed_measures_total_energy_bill_adjustment, 0) + ) END`; + +/** Total floor area (m²). New: lodged epc_property. Legacy: e.total_floor_area. */ +export const totalFloorAreaSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN epl.total_floor_area_m2 ELSE ${e}.total_floor_area END`; + +/** EPC lodgement date. New: lodged epc_property registration/inspection date. Legacy: e.lodgement_date. */ +export const lodgementDateSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN COALESCE(epl.registration_date, epl.inspection_date) ELSE ${e}.lodgement_date END`; + +/** + * "estimated" boolean. New: the property only has a predicted EPC (no lodged + * row). Legacy: e.estimated. + */ +export const estimatedSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN (epl.id IS NULL AND epp.id IS NOT NULL) ELSE COALESCE(${e}.estimated, false) END`; + +/** + * "expired" boolean. New: derived from the lodged EPC age (>10 years), since the + * new graph stores no is_expired flag. Legacy: e.is_expired. + */ +export const isExpiredSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN (epl.registration_date IS NOT NULL AND epl.registration_date < (CURRENT_DATE - INTERVAL '10 years')) ELSE ${e}.is_expired END`; + +/** Main fuel. GAP for new properties (no clean equivalent) → NULL. Legacy: e.mainfuel. */ +export const mainfuelSql = (e: Alias) => + sql`CASE WHEN ${isNewApproachSql} THEN NULL ELSE ${e}.mainfuel END`; + +// ───────────────────────────────────────────────────────────────────────────── +// Per-property resolvers (TypeScript object building). +// ───────────────────────────────────────────────────────────────────────────── + +/** + * The subset of the old `PropertyDetailsEpc` shape that the building-passport + * screens actually consume. Both the legacy row and the new graph map into this. + */ +export interface ConditionReport { + totalFloorArea: number | null; + // fabric descriptions + 1–5 ratings + walls: string | null; + wallsRating: number | null; + roof: string | null; + roofRating: number | null; + windows: string | null; + windowsRating: number | null; + heating: string | null; + heatingRating: number | null; + heatingControls: string | null; + heatingControlsRating: number | null; + hotWater: string | null; + hotWaterRating: number | null; + lighting: string | null; + lightingRating: number | null; + floor: string | null; + floorRating: number | null; + ventilation: string | null; + mainfuel: string | null; + solarPv: number | null; + solarHotWater: boolean | null; + windTurbine: number | null; + // general features + heatLossCorridor: boolean | null; + unheatedCorridorLength: number | null; + numberStoreys: number | null; + floorHeight: number | null; + numberHeatedRooms: number | null; + numberOpenFireplaces: number | null; + numberExtensions: number | null; + mainsGas: boolean | null; + energyTariff: string | null; + // heat demand + costs + currentEnergyDemand: number | null; + primaryEnergyConsumption: number | null; + co2Emissions: number | null; + heatingEnergyCostCurrent: number | null; + hotWaterEnergyCostCurrent: number | null; + lightingEnergyCostCurrent: number | null; + appliancesEnergyCostCurrent: number | null; + // estimated-data banner + estimated: boolean | null; +} + +/** Map an `epc_energy_element.energy_efficiency_rating` to a safe 1–5 (or null). */ +function safeRating(r: number | null | undefined): number | null { + return r != null && r >= 1 && r <= 5 ? r : null; +} + +/** + * Resolve a property's condition report from the new EPC graph. Prefers the + * lodged EPC; falls back to predicted (in which case `estimated = true`). + */ +async function resolveNewConditionReport( + propertyId: bigint, +): Promise { + const lodged = await db.query.epcProperty.findFirst({ + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "lodged"), + ), + }); + const chosen = + lodged ?? + (await db.query.epcProperty.findFirst({ + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "predicted"), + ), + })); + + if (!chosen) return null; + + const [elements, flat, baseline] = await Promise.all([ + db.query.epcEnergyElement.findMany({ + where: eq(epcEnergyElement.epcPropertyId, chosen.id), + }), + db.query.epcFlatDetails.findFirst({ + where: eq(epcFlatDetails.epcPropertyId, chosen.id), + }), + db.query.propertyBaselinePerformance.findFirst({ + where: eq(propertyBaselinePerformance.propertyId, propertyId), + }), + ]); + + const byType = (t: string) => elements.find((el) => el.elementType === t); + const desc = (t: string) => byType(t)?.description ?? null; + const rating = (t: string) => safeRating(byType(t)?.energyEfficiencyRating); + + const sumKwh = baseline + ? (baseline.heatingKwh ?? baseline.spaceHeatingKwh ?? 0) + + (baseline.hotWaterKwh ?? baseline.waterHeatingKwh ?? 0) + + (baseline.lightingKwh ?? 0) + + (baseline.appliancesKwh ?? 0) + + (baseline.cookingKwh ?? 0) + + (baseline.pumpsFansKwh ?? 0) + + (baseline.coolingKwh ?? 0) + : null; + + return { + totalFloorArea: chosen.totalFloorAreaM2 ?? null, + walls: desc("wall"), + wallsRating: rating("wall"), + roof: desc("roof"), + roofRating: rating("roof"), + windows: desc("window"), + windowsRating: rating("window"), + heating: desc("main_heating"), + heatingRating: rating("main_heating"), + heatingControls: desc("main_heating_controls"), + heatingControlsRating: rating("main_heating_controls"), + hotWater: desc("hot_water"), + hotWaterRating: rating("hot_water"), + lighting: desc("lighting"), + lightingRating: rating("lighting"), + floor: desc("floor"), + floorRating: rating("floor"), + ventilation: chosen.ventilationType ?? null, + // GAP: no clean main-fuel string in the new graph (jsonb on + // epc_main_heating_detail) — omitted for new properties. + mainfuel: null, + solarPv: chosen.photovoltaicArray ? 1 : 0, + solarHotWater: chosen.solarWaterHeating ?? null, + windTurbine: chosen.energyWindTurbinesCount ?? null, + heatLossCorridor: + flat?.heatLossCorridor != null ? flat.heatLossCorridor > 0 : null, + unheatedCorridorLength: flat?.unheatedCorridorLengthM ?? null, + numberStoreys: chosen.numberOfStoreys ?? null, + // GAP: no single dwelling floor height (only per-building-part) — omitted. + floorHeight: null, + numberHeatedRooms: chosen.heatedRoomsCount ?? null, + numberOpenFireplaces: chosen.openChimneysCount ?? null, + numberExtensions: chosen.extensionsCount ?? null, + mainsGas: chosen.energyMainsGas ?? null, + // GAP: no energy tariff in the new graph — omitted. + energyTariff: null, + currentEnergyDemand: sumKwh, + primaryEnergyConsumption: + baseline?.effectivePrimaryEnergyIntensityKwhPerM2Yr ?? null, + co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, + heatingEnergyCostCurrent: baseline?.heatingCostGbp ?? null, + hotWaterEnergyCostCurrent: baseline?.hotWaterCostGbp ?? null, + lightingEnergyCostCurrent: baseline?.lightingCostGbp ?? null, + appliancesEnergyCostCurrent: baseline?.appliancesCostGbp ?? null, + estimated: !lodged, + }; +} + +/** Map a legacy `property_details_epc` row into the `ConditionReport` shape. */ +function legacyConditionReport(row: Record): ConditionReport { + const r = row as Partial; + return { + totalFloorArea: r.totalFloorArea ?? null, + walls: r.walls ?? null, + wallsRating: r.wallsRating ?? null, + roof: r.roof ?? null, + roofRating: r.roofRating ?? null, + windows: r.windows ?? null, + windowsRating: r.windowsRating ?? null, + heating: r.heating ?? null, + heatingRating: r.heatingRating ?? null, + heatingControls: r.heatingControls ?? null, + heatingControlsRating: r.heatingControlsRating ?? null, + hotWater: r.hotWater ?? null, + hotWaterRating: r.hotWaterRating ?? null, + lighting: r.lighting ?? null, + lightingRating: r.lightingRating ?? null, + floor: r.floor ?? null, + floorRating: r.floorRating ?? null, + ventilation: r.ventilation ?? null, + mainfuel: r.mainfuel ?? null, + solarPv: r.solarPv ?? null, + solarHotWater: r.solarHotWater ?? null, + windTurbine: r.windTurbine ?? null, + heatLossCorridor: r.heatLossCorridor ?? null, + unheatedCorridorLength: r.unheatedCorridorLength ?? null, + numberStoreys: r.numberStoreys ?? null, + floorHeight: r.floorHeight ?? null, + numberHeatedRooms: r.numberHeatedRooms ?? null, + numberOpenFireplaces: r.numberOpenFireplaces ?? null, + numberExtensions: r.numberExtensions ?? null, + mainsGas: r.mainsGas ?? null, + energyTariff: r.energyTariff ?? null, + currentEnergyDemand: r.currentEnergyDemand ?? null, + primaryEnergyConsumption: r.primaryEnergyConsumption ?? null, + co2Emissions: r.co2Emissions ?? null, + heatingEnergyCostCurrent: r.heatingEnergyCostCurrent ?? null, + hotWaterEnergyCostCurrent: r.hotWaterEnergyCostCurrent ?? null, + lightingEnergyCostCurrent: r.lightingEnergyCostCurrent ?? null, + appliancesEnergyCostCurrent: r.appliancesEnergyCostCurrent ?? null, + estimated: r.estimated ?? null, + }; +} + +/** + * Resolve a property's condition report, branching on the cutoff. New-approach + * properties never touch `property_details_epc`. + */ +export async function resolveConditionReport( + propertyId: string, +): Promise { + const id = BigInt(propertyId); + const prop = await db.query.property.findFirst({ + columns: { updatedAt: true }, + where: eq(property.id, id), + }); + + if (prop && isNewApproach(prop.updatedAt)) { + const resolved = await resolveNewConditionReport(id); + if (!resolved) { + throw new Error( + `No epc_property graph found for new-approach property ${propertyId}`, + ); + } + return resolved; + } + + const legacy = await db.query.propertyDetailsEpc.findFirst({ + where: eq(propertyDetailsEpc.propertyId, id), + }); + if (!legacy) { + throw new Error(`No property_details_epc row for property ${propertyId}`); + } + return legacyConditionReport(legacy as Record); +} + +/** The minimal EPC meta the `/api/property-meta` route embeds as `detailsEpc`. */ +export interface DetailsEpcMeta { + currentEnergyDemand: number | null; + co2Emissions: number | null; + estimated: boolean; +} + +/** Resolve the `detailsEpc` meta block, branching on the cutoff. */ +export async function resolveDetailsEpcMeta( + propertyId: bigint, + updatedAt: Date | string | null | undefined, +): Promise { + if (isNewApproach(updatedAt)) { + const [lodged, predicted, baseline] = await Promise.all([ + db.query.epcProperty.findFirst({ + columns: { id: true }, + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "lodged"), + ), + }), + db.query.epcProperty.findFirst({ + columns: { id: true }, + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "predicted"), + ), + }), + db.query.propertyBaselinePerformance.findFirst({ + where: eq(propertyBaselinePerformance.propertyId, propertyId), + }), + ]); + + const currentEnergyDemand = baseline + ? (baseline.heatingKwh ?? baseline.spaceHeatingKwh ?? 0) + + (baseline.hotWaterKwh ?? baseline.waterHeatingKwh ?? 0) + + (baseline.lightingKwh ?? 0) + + (baseline.appliancesKwh ?? 0) + : null; + + return { + currentEnergyDemand, + co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, + estimated: !lodged && !!predicted, + }; + } + + const legacy = await db.query.propertyDetailsEpc.findFirst({ + columns: { currentEnergyDemand: true, co2Emissions: true, estimated: true }, + where: eq(propertyDetailsEpc.propertyId, propertyId), + }); + return { + currentEnergyDemand: legacy?.currentEnergyDemand ?? null, + co2Emissions: legacy?.co2Emissions ?? null, + estimated: legacy?.estimated ?? false, + }; +} From ef55c49e43595d1504cdf022b22194ddc5a9d21a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 01:00:31 +0000 Subject: [PATCH 02/27] fix(epc): tolerate absent EPC data + read recommendations via plan_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For new-approach properties the modelling pipeline currently writes only epc_property + epc_energy_element + plan/recommendation — property_baseline_ performance, epc_property_energy_performance metrics and the property-row headline columns are not yet populated. - layout: guard propertyMeta.detailsEpc?.estimated (detailsEpc can be absent) - resolveConditionReport: degrade to an all-Unknown report instead of throwing when the EPC graph/legacy row is missing (no more 500s) - getProperties: join recommendation via recommendation.plan_id instead of the retired plan_recommendations table, so Plan Cost / recommendation SAP populate Co-Authored-By: Claude Opus 4.8 (1M context) --- .../building-passport/[propertyId]/layout.tsx | 2 +- src/app/portfolio/[slug]/utils.ts | 7 ++++--- src/lib/services/epcSources.ts | 15 ++++----------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/layout.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/layout.tsx index a4c68a42..e60c2f85 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/layout.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/layout.tsx @@ -65,7 +65,7 @@ export default async function DashboardLayout(props: { decentHomes={decentHomes} /> - {propertyMeta.detailsEpc.estimated && } + {propertyMeta.detailsEpc?.estimated && } {children} diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index cf4ccfac..6025ebfd 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -736,10 +736,11 @@ export async function getProperties( AND pl.is_default = true LEFT JOIN funding_package fp ON fp.plan_id = pl.id - LEFT JOIN plan_recommendations pr - ON pr.plan_id = pl.id + -- Recommendations link to their plan via recommendation.plan_id (the + -- plan_recommendations join table is retired). Using plan_id is what makes + -- Plan Cost / recommendation SAP populate for new-approach properties. LEFT JOIN recommendation r - ON r.id = pr.recommendation_id + ON r.plan_id = pl.id AND r.default = true AND r.already_installed = false LEFT JOIN property_details_epc epc diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index d57a9bae..540f073c 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -342,22 +342,15 @@ export async function resolveConditionReport( }); if (prop && isNewApproach(prop.updatedAt)) { - const resolved = await resolveNewConditionReport(id); - if (!resolved) { - throw new Error( - `No epc_property graph found for new-approach property ${propertyId}`, - ); - } - return resolved; + // Degrade gracefully when the epc_property graph is missing — the screens + // render "Unknown" rather than 500ing. + return (await resolveNewConditionReport(id)) ?? legacyConditionReport({}); } const legacy = await db.query.propertyDetailsEpc.findFirst({ where: eq(propertyDetailsEpc.propertyId, id), }); - if (!legacy) { - throw new Error(`No property_details_epc row for property ${propertyId}`); - } - return legacyConditionReport(legacy as Record); + return legacyConditionReport((legacy ?? {}) as Record); } /** The minimal EPC meta the `/api/property-meta` route embeds as `detailsEpc`. */ From f3e00429f997fcfbab1b92f1b737cc21bfa4dae4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 01:07:00 +0000 Subject: [PATCH 03/27] fix(epc): source baseline carbon/energy/bills from lodged columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per domain semantics: lodged_* on property_baseline_performance is the gov-registry (actual) figure, effective_* is the modelling-adjusted baseline. Use the lodged columns for the "current" carbon/primary-energy display and the individual cost columns (summed) for bills. Bills return NULL when no baseline row exists, so they're excluded from AVG/SUM rather than shown as £0. (property_baseline_performance is still unwritten DB-wide; this makes the reads correct for when the pipeline populates it.) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/portfolio/[slug]/utils.ts | 2 +- src/lib/services/epcSources.ts | 41 ++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index 6025ebfd..316f67a8 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -778,7 +778,7 @@ export async function getProperties( epl.registration_date, epl.inspection_date, epl.total_floor_area_m2, - bp.effective_co2_emissions_t_per_yr, + bp.lodged_co2_emissions_t_per_yr, p.lexiscore LIMIT ${limit} OFFSET ${offset}; `); diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index 540f073c..b5c3d78a 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -73,22 +73,39 @@ export const newApproachJoins = sql` LEFT JOIN epc_property epp ON epp.property_id = p.id AND epp.source = 'predicted' `; -/** Baseline carbon (t CO₂/yr). New: effective performance. Legacy: e.co2_emissions. */ +/** + * Baseline carbon (t CO₂/yr). New: the lodged (gov-registry) value. Legacy: + * e.co2_emissions. Swap to `effective_co2_emissions_t_per_yr` if the + * modelling-adjusted baseline is wanted instead. + */ export const carbonSql = (e: Alias) => - sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_co2_emissions_t_per_yr ELSE ${e}.co2_emissions END`; + sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_co2_emissions_t_per_yr ELSE ${e}.co2_emissions END`; -/** Primary energy intensity (kWh/m²/yr). New: effective performance. Legacy: e.primary_energy_consumption. */ +/** Primary energy intensity (kWh/m²/yr). New: lodged value. Legacy: e.primary_energy_consumption. */ export const energyConsumptionSql = (e: Alias) => - sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_primary_energy_intensity_kwh_per_m2_yr ELSE ${e}.primary_energy_consumption END`; + sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_primary_energy_intensity_kwh_per_m2_yr ELSE ${e}.primary_energy_consumption END`; /** - * Annual energy bill (£). New: property_baseline_performance.total_annual_bill_gbp - * (NULL when the bill block hasn't been derived yet — excluded from AVG/SUM). - * Legacy: the old component-sum formula incl. standing charges and the - * installed-measures adjustment. + * Annual energy bill (£). New: sum of the individual cost columns on + * property_baseline_performance (NULL when no baseline row exists, so it's + * excluded from AVG/SUM rather than counted as £0). Legacy: the old + * component-sum formula incl. standing charges and the installed-measures + * adjustment. */ export const billsSql = (e: Alias) => - sql`CASE WHEN ${isNewApproachSql} THEN bp.total_annual_bill_gbp ELSE ( + sql`CASE WHEN ${isNewApproachSql} THEN ( + CASE WHEN bp.id IS NULL THEN NULL ELSE ( + COALESCE(bp.heating_cost_gbp, 0) + + COALESCE(bp.hot_water_cost_gbp, 0) + + COALESCE(bp.lighting_cost_gbp, 0) + + COALESCE(bp.appliances_cost_gbp, 0) + + COALESCE(bp.cooking_cost_gbp, 0) + + COALESCE(bp.pumps_fans_cost_gbp, 0) + + COALESCE(bp.cooling_cost_gbp, 0) + + COALESCE(bp.standing_charges_gbp, 0) - + COALESCE(bp.seg_credit_gbp, 0) + ) END + ) ELSE ( ${e}.heating_cost_current + ${e}.hot_water_cost_current + ${e}.lighting_cost_current + @@ -272,8 +289,8 @@ async function resolveNewConditionReport( energyTariff: null, currentEnergyDemand: sumKwh, primaryEnergyConsumption: - baseline?.effectivePrimaryEnergyIntensityKwhPerM2Yr ?? null, - co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, + baseline?.lodgedPrimaryEnergyIntensityKwhPerM2Yr ?? null, + co2Emissions: baseline?.lodgedCo2EmissionsTPerYr ?? null, heatingEnergyCostCurrent: baseline?.heatingCostGbp ?? null, hotWaterEnergyCostCurrent: baseline?.hotWaterCostGbp ?? null, lightingEnergyCostCurrent: baseline?.lightingCostGbp ?? null, @@ -395,7 +412,7 @@ export async function resolveDetailsEpcMeta( return { currentEnergyDemand, - co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, + co2Emissions: baseline?.lodgedCo2EmissionsTPerYr ?? null, estimated: !lodged && !!predicted, }; } From 14f06abb6deea76b196c37ef06339503ff2e3a65 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 14:07:09 +0000 Subject: [PATCH 04/27] perf(properties): LATERAL lookups + recommendation by property_id /api/properties timed out (15s) on just 12 properties: joining the 26M-row recommendation table on the unindexed plan_id forced a full-table hash, and the multi-million-row plan/funding joins compounded it. Rewrite getProperties/getPropertiesCount to read the denormalised model (no plan_recommendations / recommendation_materials): one-row LATERAL lookups for target/plan/funding and a recommendation aggregate keyed by the INDEXED recommendation.property_id, dropping the GROUP BY entirely. ~1s -> instant for a 12-property portfolio. Also fix the reporting page for new-approach properties (current_sap_points / current_epc_rating on the property row are unwritten): source avg SAP and the EPC band distribution from property_baseline_performance (lodged_sap_score / lodged_epc_band) via new sapSql / epcBandSql helpers. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../reporting/databaseFunctions.ts | 6 +- src/app/portfolio/[slug]/utils.ts | 95 +++++++++---------- src/lib/services/epcSources.ts | 11 +++ 3 files changed, 61 insertions(+), 51 deletions(-) diff --git a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts index c1048dd5..db61d9df 100644 --- a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts +++ b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts @@ -9,6 +9,8 @@ import { energyConsumptionSql, estimatedSql, isExpiredSql, + sapSql, + epcBandSql, } from "@/lib/services/epcSources"; import type { @@ -36,7 +38,7 @@ export async function getAverages( ): Promise { const result = await db.execute(sql` SELECT - AVG(p.current_sap_points)::float AS avg_sap, + AVG(${sapSql})::float AS avg_sap, AVG(${carbonSql(sql`e`)})::float AS avg_carbon, AVG(${billsSql(sql`e`)})::float AS avg_bills, AVG(${energyConsumptionSql(sql`e`)})::float AS avg_energy_consumption @@ -97,7 +99,7 @@ export async function getCountByEpcBand( SELECT * FROM ( SELECT - COALESCE(p.current_epc_rating::text, 'Unknown') AS epc, + COALESCE((${epcBandSql})::text, 'Unknown') AS epc, COUNT(*) FILTER ( WHERE ${estimatedSql(sql`e`)} = false )::int AS actual, diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index 316f67a8..971a700b 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -682,7 +682,14 @@ export async function getPropertiesCount( FROM property p LEFT JOIN property_details_epc epc ON epc.property_id = p.id ${newApproachJoins} - LEFT JOIN plan pl ON pl.property_id = p.id AND pl.is_default = true + LEFT JOIN LATERAL ( + SELECT id, post_sap_points FROM plan + WHERE property_id = p.id + AND portfolio_id = p.portfolio_id + AND is_default = true + ORDER BY created_at DESC + LIMIT 1 + ) pl ON true WHERE p.portfolio_id = ${portfolioId} ${combinedWhere} `); @@ -714,8 +721,8 @@ export async function getProperties( t.epc AS "targetEpc", pl.id AS "planId", fp.scheme AS "fundingScheme", - COALESCE(SUM(r.sap_points), 0) AS "totalRecommendationSapPoints", - COALESCE(SUM(r.estimated_cost), 0) AS "totalRecommendationCost", + COALESCE(rec.sap_points, 0) AS "totalRecommendationSapPoints", + COALESCE(rec.cost, 0) AS "totalRecommendationCost", p.landlord_property_id AS "landlordPropertyId", p.original_sap_points AS "originalSapPoints", p.property_type AS "propertyType", @@ -729,57 +736,47 @@ export async function getProperties( ${mainfuelSql(sql`epc`)} AS mainfuel, p.lexiscore AS lexiscore FROM property p - LEFT JOIN property_targets t - ON t.property_id = p.id - LEFT JOIN plan pl - ON pl.property_id = p.id - AND pl.is_default = true - LEFT JOIN funding_package fp - ON fp.plan_id = pl.id - -- Recommendations link to their plan via recommendation.plan_id (the - -- plan_recommendations join table is retired). Using plan_id is what makes - -- Plan Cost / recommendation SAP populate for new-approach properties. - LEFT JOIN recommendation r - ON r.plan_id = pl.id - AND r.default = true - AND r.already_installed = false + -- LATERAL one-row lookups keep this query at "12 properties × index probe" + -- instead of hash-joining the multi-million-row plan/recommendation tables. + LEFT JOIN LATERAL ( + SELECT epc FROM property_targets + WHERE property_id = p.id + ORDER BY created_at DESC + LIMIT 1 + ) t ON true + LEFT JOIN LATERAL ( + SELECT * FROM plan + WHERE property_id = p.id + AND portfolio_id = p.portfolio_id + AND is_default = true + ORDER BY created_at DESC + LIMIT 1 + ) pl ON true + LEFT JOIN LATERAL ( + SELECT scheme FROM funding_package + WHERE plan_id = pl.id + LIMIT 1 + ) fp ON true + -- Active (default, not-yet-installed) recommendations for the property, + -- read denormalised. We aggregate by recommendation.property_id — the only + -- indexed access path (there is NO index on recommendation.plan_id, and the + -- table has tens of millions of rows) — and rely on the partial index on + -- (default, already_installed). The retired plan_recommendations join table + -- is intentionally not used. + LEFT JOIN LATERAL ( + SELECT + COALESCE(SUM(sap_points), 0) AS sap_points, + COALESCE(SUM(estimated_cost), 0) AS cost + FROM recommendation + WHERE property_id = p.id + AND "default" = true + AND already_installed = false + ) rec ON true LEFT JOIN property_details_epc epc ON epc.property_id = p.id ${newApproachJoins} WHERE p.portfolio_id = ${portfolioId} ${combinedWhere} - GROUP BY - p.id, - p.portfolio_id, - p.address, - p.postcode, - p.status, - p.creation_status, - p.current_epc_rating, - p.current_sap_points, - t.epc, - pl.id, - fp.scheme, - p.landlord_property_id, - p.original_sap_points, - p.property_type, - p.built_form, - p.tenure, - p.year_built, - -- legacy + new-source columns referenced by the branched EPC expressions - p.updated_at, - epc.lodgement_date, - epc.is_expired, - epc.total_floor_area, - epc.co2_emissions, - epc.mainfuel, - epl.id, - epp.id, - epl.registration_date, - epl.inspection_date, - epl.total_floor_area_m2, - bp.lodged_co2_emissions_t_per_yr, - p.lexiscore LIMIT ${limit} OFFSET ${offset}; `); diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index b5c3d78a..c7377f34 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -85,6 +85,17 @@ export const carbonSql = (e: Alias) => export const energyConsumptionSql = (e: Alias) => sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_primary_energy_intensity_kwh_per_m2_yr ELSE ${e}.primary_energy_consumption END`; +/** + * Baseline SAP score. New: the lodged (gov-registry) score from + * property_baseline_performance (the property-row current_sap_points isn't + * written for new-approach properties). Legacy: property.current_sap_points. + * Swap to `effective_sap_score` for the modelling-adjusted baseline. + */ +export const sapSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_sap_score ELSE p.current_sap_points END`; + +/** Baseline EPC band. New: lodged band from property_baseline_performance. Legacy: property.current_epc_rating. */ +export const epcBandSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_epc_band ELSE p.current_epc_rating END`; + /** * Annual energy bill (£). New: sum of the individual cost columns on * property_baseline_performance (NULL when no baseline row exists, so it's From 5d87481b6bddcdac450591c426ec4d98a4e253da Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 14:20:49 +0000 Subject: [PATCH 05/27] fix(plans): source headline SAP/EPC from baseline for new properties The plans page computed the plan uplift as postSapPoints - currentSapPoints, but current_sap_points is unwritten on the property row for new-approach properties (null -> 0), so the uplift showed the full post score (e.g. +71.39) and no current rating. property-meta now backfills currentSapPoints / currentEpcRating from property_baseline_performance (lodged_sap_score / lodged_epc_band) for new-approach properties, fixing every propertyMeta consumer (plans page, hero cards, building passport). e.g. 709634: current 60 (D), uplift now +11.39. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../api/property-meta/[propertyId]/route.ts | 32 +++++++++++++------ src/lib/services/epcSources.ts | 26 +++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/app/api/property-meta/[propertyId]/route.ts b/src/app/api/property-meta/[propertyId]/route.ts index f526544d..73281f6b 100644 --- a/src/app/api/property-meta/[propertyId]/route.ts +++ b/src/app/api/property-meta/[propertyId]/route.ts @@ -1,7 +1,10 @@ import { db } from "@/app/db/db"; import { property } from "@/app/db/schema/property"; import { serializeBigInt } from "@/app/utils"; -import { resolveDetailsEpcMeta } from "@/lib/services/epcSources"; +import { + resolveDetailsEpcMeta, + resolvePropertyHeadline, +} from "@/lib/services/epcSources"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; @@ -38,15 +41,26 @@ export async function GET(request: NextRequest, props: { params: Promise<{ prope return new NextResponse(null, { status: 404 }); } - // `detailsEpc` is resolved via the cutoff: legacy properties read - // property_details_epc; new-approach ones read epc_property + - // property_baseline_performance. See src/lib/services/epcSources.ts. - const detailsEpc = await resolveDetailsEpcMeta( - propertyMeta.id, - propertyMeta.updatedAt, - ); + // `detailsEpc` and the headline SAP/EPC are resolved via the cutoff: legacy + // properties read property_details_epc / the property row; new-approach ones + // read property_baseline_performance (the property-row headline columns aren't + // written yet). See src/lib/services/epcSources.ts. + const [detailsEpc, headline] = await Promise.all([ + resolveDetailsEpcMeta(propertyMeta.id, propertyMeta.updatedAt), + resolvePropertyHeadline(propertyMeta.id, propertyMeta.updatedAt), + ]); return new NextResponse( - JSON.stringify({ ...propertyMeta, detailsEpc }, serializeBigInt), + JSON.stringify( + { + ...propertyMeta, + currentSapPoints: + headline.currentSapPoints ?? propertyMeta.currentSapPoints, + currentEpcRating: + headline.currentEpcRating ?? propertyMeta.currentEpcRating, + detailsEpc, + }, + serializeBigInt, + ), ); } diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index c7377f34..efec6984 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -381,6 +381,32 @@ export async function resolveConditionReport( return legacyConditionReport((legacy ?? {}) as Record); } +/** + * Headline SAP score + EPC band for the property meta. New-approach properties + * don't have `current_sap_points` / `current_epc_rating` written on the row yet, + * so we fall back to the lodged baseline (property_baseline_performance). Returns + * nulls for legacy properties, signalling the caller to keep the row values. + * + * Uses lodged_* (gov-registry actual) to match the rest of the EPC reads; swap to + * effective_* for the modelling-adjusted baseline. + */ +export async function resolvePropertyHeadline( + propertyId: bigint, + updatedAt: Date | string | null | undefined, +): Promise<{ currentSapPoints: number | null; currentEpcRating: string | null }> { + if (!isNewApproach(updatedAt)) { + return { currentSapPoints: null, currentEpcRating: null }; + } + const baseline = await db.query.propertyBaselinePerformance.findFirst({ + columns: { lodgedSapScore: true, lodgedEpcBand: true }, + where: eq(propertyBaselinePerformance.propertyId, propertyId), + }); + return { + currentSapPoints: baseline?.lodgedSapScore ?? null, + currentEpcRating: baseline?.lodgedEpcBand ?? null, + }; +} + /** The minimal EPC meta the `/api/property-meta` route embeds as `detailsEpc`. */ export interface DetailsEpcMeta { currentEnergyDemand: number | null; From a184127cd47dbf97d3bcf56e3e07037243965f44 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 14:25:41 +0000 Subject: [PATCH 06/27] fix(plan-detail): read recommendations denormalised, not via plan_recommendations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getRecommendations read the retired plan_recommendations join table, which is empty for new-approach plans, so the plan detail page showed no recommendations. Read the recommendation table directly. Because recommendation.plan_id is unindexed (26M rows), key off the plan's property_id (indexed) and filter plan_id in memory — fast for any plan, default or not. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../building-passport/[propertyId]/utils.ts | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index 0ab77e7b..7ab6bece 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -1,6 +1,7 @@ import S3 from "aws-sdk/clients/s3"; import { Recommendation, + recommendation, planRecommendations, plan, Plan, @@ -163,23 +164,26 @@ export async function getEnergyAssessmentDocuments( export async function getRecommendations( planId: string ): Promise { - const data = (await db.query.planRecommendations.findMany({ - where: eq(planRecommendations.planId, BigInt(planId)), - columns: {}, - with: { - recommendation: true, - }, - })) as RecommendationList; - - if (!data) { - throw new Error("Network response was not ok"); + // Recommendations link to their plan via recommendation.plan_id (the + // plan_recommendations join table is retired). recommendation.plan_id is NOT + // indexed and the table has tens of millions of rows, so we key off the plan's + // property_id (indexed) and filter plan_id in memory. We drop already-installed + // measures, matching the previous behaviour. + const planRow = await db.query.plan.findFirst({ + columns: { propertyId: true }, + where: eq(plan.id, BigInt(planId)), + }); + if (!planRow?.propertyId) { + return []; } - // unnest the recommendations - // We drop measures that are already installed - const recommendations = data - .map((item) => item.recommendation) - .filter((rec) => !rec.alreadyInstalled); + const recommendations = await db.query.recommendation.findMany({ + where: and( + eq(recommendation.propertyId, planRow.propertyId), + eq(recommendation.planId, BigInt(planId)), + eq(recommendation.alreadyInstalled, false) + ), + }); return recommendations; } From e1d56cc7738fbf89ec2490181e2c53fb9a56a443 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 14:31:38 +0000 Subject: [PATCH 07/27] fix(building-passport): don't cache property-meta fetch getPropertyMeta self-fetched /api/property-meta with revalidate:60 (despite a comment saying it shouldn't cache). Vercel's Data Cache served a stale response from before the headline backfill, so the current EPC rating / SAP rendered blank on the plans and plan-detail pages even though the API now returns them. Use cache:"no-store" so backfilled headline values propagate immediately. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[slug]/building-passport/[propertyId]/utils.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index 7ab6bece..6728e09b 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -242,11 +242,13 @@ export async function getPropertyMeta( ): Promise { const url = process.env.URL + `/api/property-meta/${propertyId}`; - // Note, for the moment, we don't cache the data + // Don't cache: property meta (incl. headline SAP/EPC, which is now backfilled + // from the baseline table) changes as the modelling pipeline writes data, and + // a stale Data Cache entry was causing the current rating to render blank. const response = await fetch(url, { method: "GET", headers: { "Content-Type": "application/json" }, - next: { revalidate: 60 }, + cache: "no-store", }); if (!response.ok) { throw new Error("Network response was not ok"); From fec714a4064a52bd99af12a308dde6834ff779ae Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 17 Jun 2026 14:40:17 +0000 Subject: [PATCH 08/27] fix(property-meta): resolve in-process, drop self-fetch to process.env.URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getPropertyMeta self-fetched process.env.URL + /api/property-meta. That URL can resolve to a different deployment (e.g. production), so the building-passport pages were reading property meta from OLD code — the backfilled headline SAP/EPC never arrived and the current rating rendered blank, regardless of caching. Extract resolvePropertyMeta() and call it directly from both the route and getPropertyMeta, so the page runs identical in-process logic. JSON round-trip through serializeBigInt preserves the previous shape (bigints/dates as strings). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../api/property-meta/[propertyId]/route.ts | 60 +++---------------- .../building-passport/[propertyId]/utils.ts | 24 ++++---- src/lib/services/epcSources.ts | 51 ++++++++++++++++ 3 files changed, 69 insertions(+), 66 deletions(-) diff --git a/src/app/api/property-meta/[propertyId]/route.ts b/src/app/api/property-meta/[propertyId]/route.ts index 73281f6b..84ccc2c9 100644 --- a/src/app/api/property-meta/[propertyId]/route.ts +++ b/src/app/api/property-meta/[propertyId]/route.ts @@ -1,66 +1,20 @@ -import { db } from "@/app/db/db"; -import { property } from "@/app/db/schema/property"; import { serializeBigInt } from "@/app/utils"; -import { - resolveDetailsEpcMeta, - resolvePropertyHeadline, -} from "@/lib/services/epcSources"; -import { eq } from "drizzle-orm"; +import { resolvePropertyMeta } from "@/lib/services/epcSources"; import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest, props: { params: Promise<{ propertyId: string }> }) { const params = await props.params; const propertyId = params.propertyId; - const propertyMeta = await db.query.property.findFirst({ - columns: { - id: true, - uprn: true, - address: true, - postcode: true, - hasPreConditionReport: true, - hasRecommendations: true, - createdAt: true, - propertyType: true, - builtForm: true, - localAuthority: true, - constituency: true, - numberOfRooms: true, - yearBuilt: true, - tenure: true, - currentEpcRating: true, - currentSapPoints: true, - originalSapPoints: true, - updatedAt: true, - currentValuation: true, - }, - where: eq(property.id, BigInt(propertyId)), - }); + // `detailsEpc` and the headline SAP/EPC are resolved via the cutoff: legacy + // properties read property_details_epc / the property row; new-approach ones + // read property_baseline_performance (the property-row headline columns aren't + // written yet). See src/lib/services/epcSources.ts. + const propertyMeta = await resolvePropertyMeta(propertyId); if (!propertyMeta) { return new NextResponse(null, { status: 404 }); } - // `detailsEpc` and the headline SAP/EPC are resolved via the cutoff: legacy - // properties read property_details_epc / the property row; new-approach ones - // read property_baseline_performance (the property-row headline columns aren't - // written yet). See src/lib/services/epcSources.ts. - const [detailsEpc, headline] = await Promise.all([ - resolveDetailsEpcMeta(propertyMeta.id, propertyMeta.updatedAt), - resolvePropertyHeadline(propertyMeta.id, propertyMeta.updatedAt), - ]); - - return new NextResponse( - JSON.stringify( - { - ...propertyMeta, - currentSapPoints: - headline.currentSapPoints ?? propertyMeta.currentSapPoints, - currentEpcRating: - headline.currentEpcRating ?? propertyMeta.currentEpcRating, - detailsEpc, - }, - serializeBigInt, - ), - ); + return new NextResponse(JSON.stringify(propertyMeta, serializeBigInt)); } diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index 6728e09b..c2533cea 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -23,8 +23,9 @@ import { import { ConditionReport, resolveConditionReport, + resolvePropertyMeta, } from "@/lib/services/epcSources"; -import { getRating } from "@/app/utils"; +import { getRating, serializeBigInt } from "@/app/utils"; import { eq, desc, and } from "drizzle-orm"; import { energyAssessment, @@ -240,20 +241,17 @@ export async function getPlans(propertyId: string): Promise { export async function getPropertyMeta( propertyId: string ): Promise { - const url = process.env.URL + `/api/property-meta/${propertyId}`; - - // Don't cache: property meta (incl. headline SAP/EPC, which is now backfilled - // from the baseline table) changes as the modelling pipeline writes data, and - // a stale Data Cache entry was causing the current rating to render blank. - const response = await fetch(url, { - method: "GET", - headers: { "Content-Type": "application/json" }, - cache: "no-store", - }); - if (!response.ok) { + // Resolve in-process instead of self-fetching /api/property-meta. The HTTP + // round-trip went to process.env.URL, which can point at a different + // deployment (e.g. production) and was serving stale/old-code responses — so + // the backfilled headline SAP/EPC never reached the page. We JSON round-trip + // through serializeBigInt to keep the exact shape the fetch used to return + // (bigints as strings, dates as ISO strings). + const meta = await resolvePropertyMeta(propertyId); + if (!meta) { throw new Error("Network response was not ok"); } - return response.json(); + return JSON.parse(JSON.stringify(meta, serializeBigInt)) as PropertyMeta; } export async function getConditionReport( diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index efec6984..0b376957 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -464,3 +464,54 @@ export async function resolveDetailsEpcMeta( estimated: legacy?.estimated ?? false, }; } + +/** + * Resolve the full property-meta object (the property row + branched + * `detailsEpc` + headline SAP/EPC). Shared by the `/api/property-meta` route and + * the building-passport `getPropertyMeta` so both run identical, in-process + * logic — no self-fetch to `process.env.URL` (which could hit a different + * deployment) and no Data Cache. Returns null when the property doesn't exist. + * BigInt fields are left raw; the caller serialises with `serializeBigInt`. + */ +export async function resolvePropertyMeta(propertyId: string) { + const propertyMeta = await db.query.property.findFirst({ + columns: { + id: true, + uprn: true, + address: true, + postcode: true, + hasPreConditionReport: true, + hasRecommendations: true, + createdAt: true, + propertyType: true, + builtForm: true, + localAuthority: true, + constituency: true, + numberOfRooms: true, + yearBuilt: true, + tenure: true, + currentEpcRating: true, + currentSapPoints: true, + originalSapPoints: true, + updatedAt: true, + currentValuation: true, + }, + where: eq(property.id, BigInt(propertyId)), + }); + + if (!propertyMeta) return null; + + const [detailsEpc, headline] = await Promise.all([ + resolveDetailsEpcMeta(propertyMeta.id, propertyMeta.updatedAt), + resolvePropertyHeadline(propertyMeta.id, propertyMeta.updatedAt), + ]); + + return { + ...propertyMeta, + currentSapPoints: + headline.currentSapPoints ?? propertyMeta.currentSapPoints, + currentEpcRating: + headline.currentEpcRating ?? propertyMeta.currentEpcRating, + detailsEpc, + }; +} From 04d9b45f0a1b5f063f1148fc9d2d7e0f0fd237cf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 08:21:16 +0000 Subject: [PATCH 09/27] perf(properties): stop reading property_targets and funding_package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were per-property LATERAL lookups against tables with no index on the join column (property_targets.property_id, funding_package.plan_id), so each seq-scanned 150k-180k rows once per property — ~3.4s of the load for a 291-property portfolio came from property_targets alone. Neither value is actually used by the portfolio table: the "Expected EPC" column derives from currentSapPoints + recommendation SAP, and fundingScheme is fetched separately by the proposals table. Drop both reads (kept as NULL to preserve the PropertyWithRelations shape) and trim the plan LATERAL to the columns used. 291-property portfolio: ~3.5s -> ~85ms. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/portfolio/[slug]/utils.ts | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index 971a700b..c6947ccf 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -718,9 +718,14 @@ export async function getProperties( p.creation_status AS "creationStatus", p.current_epc_rating AS "currentEpcRating", p.current_sap_points AS "currentSapPoints", - t.epc AS "targetEpc", + -- property_targets is no longer read (the "Expected EPC" column derives + -- from currentSapPoints + recommendation SAP). Kept as NULL to preserve + -- the PropertyWithRelations shape. + NULL AS "targetEpc", pl.id AS "planId", - fp.scheme AS "fundingScheme", + -- funding_package is no longer read here (the proposals table fetches it + -- separately). Kept as NULL to preserve the PropertyWithRelations shape. + NULL AS "fundingScheme", COALESCE(rec.sap_points, 0) AS "totalRecommendationSapPoints", COALESCE(rec.cost, 0) AS "totalRecommendationCost", p.landlord_property_id AS "landlordPropertyId", @@ -736,27 +741,16 @@ export async function getProperties( ${mainfuelSql(sql`epc`)} AS mainfuel, p.lexiscore AS lexiscore FROM property p - -- LATERAL one-row lookups keep this query at "12 properties × index probe" + -- LATERAL one-row lookups keep this query at "N properties × index probe" -- instead of hash-joining the multi-million-row plan/recommendation tables. LEFT JOIN LATERAL ( - SELECT epc FROM property_targets - WHERE property_id = p.id - ORDER BY created_at DESC - LIMIT 1 - ) t ON true - LEFT JOIN LATERAL ( - SELECT * FROM plan + SELECT id, post_sap_points FROM plan WHERE property_id = p.id AND portfolio_id = p.portfolio_id AND is_default = true ORDER BY created_at DESC LIMIT 1 ) pl ON true - LEFT JOIN LATERAL ( - SELECT scheme FROM funding_package - WHERE plan_id = pl.id - LIMIT 1 - ) fp ON true -- Active (default, not-yet-installed) recommendations for the property, -- read denormalised. We aggregate by recommendation.property_id — the only -- indexed access path (there is NO index on recommendation.plan_id, and the From 5dd2179befd92679bafdbceca7e0a167485c6a5e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 08:37:30 +0000 Subject: [PATCH 10/27] fix(properties): source current SAP/EPC from baseline in the portfolio table getProperties selected p.current_sap_points / p.current_epc_rating directly, which are null for new-approach properties (headline columns unwritten on the row). That made the Current EPC column blank and, since the Expected EPC column is currentSapPoints + recommendation SAP, made Expected EPC show 0. Use the sapSql / epcBandSql baseline helpers (lodged_sap_score / lodged_epc_band for new-approach properties, property row for legacy), matching the reporting page. Verified for properties that have a baseline row; the rest populate once the baseline backfill completes. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/portfolio/[slug]/utils.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index c6947ccf..40e612a5 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -26,6 +26,8 @@ import { lodgementDateSql, isExpiredSql, mainfuelSql, + sapSql, + epcBandSql, } from "@/lib/services/epcSources"; import { FilterGroups, @@ -716,8 +718,11 @@ export async function getProperties( p.postcode AS postcode, p.status AS status, p.creation_status AS "creationStatus", - p.current_epc_rating AS "currentEpcRating", - p.current_sap_points AS "currentSapPoints", + -- Current EPC/SAP: new-approach properties have null headline columns on + -- the property row, so source from the lodged baseline. The "Expected EPC" + -- column is currentSapPoints + recommendation SAP, so this also fixes it. + ${epcBandSql} AS "currentEpcRating", + ${sapSql} AS "currentSapPoints", -- property_targets is no longer read (the "Expected EPC" column derives -- from currentSapPoints + recommendation SAP). Kept as NULL to preserve -- the PropertyWithRelations shape. From b7cbd5c03f42925a8f092073bf30b554888bf652 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 10:02:53 +0000 Subject: [PATCH 11/27] fix(assessment): fill property type / built form / age / floor height from EPC graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For new-approach properties the property-row text columns (property_type, built_form, year_built) are null, and the condition report had no floor height, so the assessment page showed Unknown for several general features. - property-meta: backfill property type / built form / age from epc_property (+ epc_building_part construction age band), mapping RdSAP codes to labels (property_type 2 -> Flat, built_form 3 -> End-Terrace, age band F -> 1976–1982), falling back to the raw value for unmapped codes. - condition report: source floor height from the main building part's epc_floor_dimension.room_height_m. - formatGeneralFeatures: use ?? instead of || so a real 0 (open fireplaces / extensions) shows as 0 rather than "Unknown". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../building-passport/[propertyId]/utils.ts | 11 +- src/lib/services/epcSources.ts | 116 +++++++++++++++++- 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index c2533cea..89875a9b 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -316,26 +316,27 @@ export function formatGeneralFeatures( }, { feature: "Number of storeys", - description: conditionReportData.numberStoreys || "Unknown", + description: conditionReportData.numberStoreys ?? "Unknown", }, ]; const generealFeatures: GeneralFeature[] = [ { feature: "Floor Height", - description: conditionReportData.floorHeight || "Unknown", + description: conditionReportData.floorHeight ?? "Unknown", }, { feature: "Number of heated rooms", - description: conditionReportData.numberHeatedRooms || "Unknown", + description: conditionReportData.numberHeatedRooms ?? "Unknown", }, { feature: "Number of open fire places", - description: conditionReportData.numberOpenFireplaces || "Unknown", + // ?? so a genuine 0 (no fireplaces) shows as 0, not "Unknown". + description: conditionReportData.numberOpenFireplaces ?? "Unknown", }, { feature: "Number of extensions", - description: conditionReportData.numberExtensions || "Unknown", + description: conditionReportData.numberExtensions ?? "Unknown", }, { feature: "Mains gas", diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index 0b376957..e8071093 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -33,6 +33,8 @@ import { epcProperty, epcEnergyElement, epcFlatDetails, + epcBuildingPart, + epcFloorDimension, propertyBaselinePerformance, propertyDetailsEpc, property, @@ -235,7 +237,7 @@ async function resolveNewConditionReport( if (!chosen) return null; - const [elements, flat, baseline] = await Promise.all([ + const [elements, flat, baseline, buildingParts] = await Promise.all([ db.query.epcEnergyElement.findMany({ where: eq(epcEnergyElement.epcPropertyId, chosen.id), }), @@ -245,8 +247,22 @@ async function resolveNewConditionReport( db.query.propertyBaselinePerformance.findFirst({ where: eq(propertyBaselinePerformance.propertyId, propertyId), }), + db.query.epcBuildingPart.findMany({ + columns: { id: true, identifier: true }, + where: eq(epcBuildingPart.epcPropertyId, chosen.id), + }), ]); + // Floor height: room height of the main building part (fall back to any part). + const mainPart = + buildingParts.find((b) => b.identifier === "main") ?? buildingParts[0]; + const mainFloor = mainPart + ? await db.query.epcFloorDimension.findFirst({ + columns: { roomHeightM: true }, + where: eq(epcFloorDimension.epcBuildingPartId, mainPart.id), + }) + : null; + const byType = (t: string) => elements.find((el) => el.elementType === t); const desc = (t: string) => byType(t)?.description ?? null; const rating = (t: string) => safeRating(byType(t)?.energyEfficiencyRating); @@ -290,8 +306,9 @@ async function resolveNewConditionReport( flat?.heatLossCorridor != null ? flat.heatLossCorridor > 0 : null, unheatedCorridorLength: flat?.unheatedCorridorLengthM ?? null, numberStoreys: chosen.numberOfStoreys ?? null, - // GAP: no single dwelling floor height (only per-building-part) — omitted. - floorHeight: null, + // Room height of the main building part (the new graph stores height per + // building part rather than one dwelling value). + floorHeight: mainFloor?.roomHeightM ?? null, numberHeatedRooms: chosen.heatedRoomsCount ?? null, numberOpenFireplaces: chosen.openChimneysCount ?? null, numberExtensions: chosen.extensionsCount ?? null, @@ -465,6 +482,90 @@ export async function resolveDetailsEpcMeta( }; } +// RdSAP code → label maps for the descriptive fields the new pipeline stores as +// codes on epc_property / epc_building_part (the property-row text columns aren't +// written for new-approach properties). Unknown codes fall back to the raw value. +const PROPERTY_TYPE_LABELS: Record = { + "0": "House", + "1": "Bungalow", + "2": "Flat", + "3": "Maisonette", + "4": "Park home", +}; +const BUILT_FORM_LABELS: Record = { + "1": "Detached", + "2": "Semi-Detached", + "3": "End-Terrace", + "4": "Mid-Terrace", + "5": "Enclosed End-Terrace", + "6": "Enclosed Mid-Terrace", +}; +const AGE_BAND_YEARS: Record = { + A: "before 1900", + B: "1900–1929", + C: "1930–1949", + D: "1950–1966", + E: "1967–1975", + F: "1976–1982", + G: "1983–1990", + H: "1991–1995", + I: "1996–2002", + J: "2003–2006", + K: "2007–2011", + L: "2012 onwards", +}; + +/** + * Resolve the descriptive property fields (type / built form / age) from the + * epc_property graph for new-approach properties, since the property-row text + * columns aren't written. Returns nulls when there's no EPC graph. + */ +async function resolvePropertyDescriptors( + propertyId: bigint, +): Promise<{ + propertyType: string | null; + builtForm: string | null; + yearBuilt: string | null; +}> { + const ep = + (await db.query.epcProperty.findFirst({ + columns: { id: true, propertyType: true, builtForm: true, dwellingType: true }, + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "lodged"), + ), + })) ?? + (await db.query.epcProperty.findFirst({ + columns: { id: true, propertyType: true, builtForm: true, dwellingType: true }, + where: and( + eq(epcProperty.propertyId, propertyId), + eq(epcProperty.source, "predicted"), + ), + })); + + if (!ep) { + return { propertyType: null, builtForm: null, yearBuilt: null }; + } + + const mainPart = await db.query.epcBuildingPart.findFirst({ + columns: { constructionAgeBand: true }, + where: eq(epcBuildingPart.epcPropertyId, ep.id), + }); + + const propertyType = + (ep.propertyType ? PROPERTY_TYPE_LABELS[ep.propertyType] : null) ?? + ep.dwellingType ?? + ep.propertyType ?? + null; + const builtForm = ep.builtForm + ? BUILT_FORM_LABELS[ep.builtForm] ?? ep.builtForm + : null; + const ageBand = mainPart?.constructionAgeBand ?? null; + const yearBuilt = ageBand ? AGE_BAND_YEARS[ageBand] ?? ageBand : null; + + return { propertyType, builtForm, yearBuilt }; +} + /** * Resolve the full property-meta object (the property row + branched * `detailsEpc` + headline SAP/EPC). Shared by the `/api/property-meta` route and @@ -501,13 +602,20 @@ export async function resolvePropertyMeta(propertyId: string) { if (!propertyMeta) return null; - const [detailsEpc, headline] = await Promise.all([ + const newApproach = isNewApproach(propertyMeta.updatedAt); + const [detailsEpc, headline, descriptors] = await Promise.all([ resolveDetailsEpcMeta(propertyMeta.id, propertyMeta.updatedAt), resolvePropertyHeadline(propertyMeta.id, propertyMeta.updatedAt), + newApproach + ? resolvePropertyDescriptors(propertyMeta.id) + : Promise.resolve(null), ]); return { ...propertyMeta, + propertyType: descriptors?.propertyType ?? propertyMeta.propertyType, + builtForm: descriptors?.builtForm ?? propertyMeta.builtForm, + yearBuilt: descriptors?.yearBuilt ?? propertyMeta.yearBuilt, currentSapPoints: headline.currentSapPoints ?? propertyMeta.currentSapPoints, currentEpcRating: From a1ba5b1e0d740602630ad0e252f17514114929dd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 10:23:00 +0000 Subject: [PATCH 12/27] fix(assessment): handle text + numeric property_type/built_form; correct age bands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit epc_property stores property_type/built_form either as numeric RdSAP codes (newer rows) or plain text ("House", "Detached" — most rows). The previous code only mapped codes and fell through to dwelling_type for text rows, showing "Semi-Detached house" instead of "House". Now: map numeric codes, pass text through as-is, fall back to dwelling_type only when absent. Also align AGE_BAND_YEARS with the backend ConstructionAgeBand enum: L is 2012–2022 (not "2012 onwards") and add M = 2023 onwards. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/services/epcSources.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index e8071093..5fdea8b0 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -500,6 +500,7 @@ const BUILT_FORM_LABELS: Record = { "5": "Enclosed End-Terrace", "6": "Enclosed Mid-Terrace", }; +// Matches the backend ConstructionAgeBand enum (RdSAP England & Wales letters). const AGE_BAND_YEARS: Record = { A: "before 1900", B: "1900–1929", @@ -512,7 +513,8 @@ const AGE_BAND_YEARS: Record = { I: "1996–2002", J: "2003–2006", K: "2007–2011", - L: "2012 onwards", + L: "2012–2022", + M: "2023 onwards", }; /** @@ -552,11 +554,12 @@ async function resolvePropertyDescriptors( where: eq(epcBuildingPart.epcPropertyId, ep.id), }); - const propertyType = - (ep.propertyType ? PROPERTY_TYPE_LABELS[ep.propertyType] : null) ?? - ep.dwellingType ?? - ep.propertyType ?? - null; + // property_type / built_form are stored either as numeric RdSAP codes (newer + // rows) or as plain text ("House", "Detached" — most rows). Map numeric codes; + // pass text through unchanged; fall back to dwelling_type for property type. + const propertyType = ep.propertyType + ? PROPERTY_TYPE_LABELS[ep.propertyType] ?? ep.propertyType + : ep.dwellingType ?? null; const builtForm = ep.builtForm ? BUILT_FORM_LABELS[ep.builtForm] ?? ep.builtForm : null; From 9750770c7cf7225cc8b3736e0c475cd96f12d220 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 11:24:25 +0000 Subject: [PATCH 13/27] fix(building-passport): fill storeys and habitable rooms from EPC graph On the overview page, Storeys read epc_property.number_of_storeys (null for some properties) and Habitable rooms read the property-row number_of_rooms (null for new-approach properties). - Habitable rooms: backfill propertyMeta.numberOfRooms from epc_property.habitable_rooms_count (always populated). - Storeys: fall back to the main building part's floor-dimension count when epc_property.number_of_storeys is null. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/services/epcSources.ts | 43 ++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index 5fdea8b0..e4987ab7 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -253,15 +253,18 @@ async function resolveNewConditionReport( }), ]); - // Floor height: room height of the main building part (fall back to any part). + // Floor height + storey count from the main building part's floor dimensions + // (fall back to any part). Each floor dimension is one storey of that part. const mainPart = buildingParts.find((b) => b.identifier === "main") ?? buildingParts[0]; - const mainFloor = mainPart - ? await db.query.epcFloorDimension.findFirst({ + const mainFloors = mainPart + ? await db.query.epcFloorDimension.findMany({ columns: { roomHeightM: true }, where: eq(epcFloorDimension.epcBuildingPartId, mainPart.id), }) - : null; + : []; + const floorHeight = mainFloors[0]?.roomHeightM ?? null; + const storeyCountFromFloors = mainFloors.length || null; const byType = (t: string) => elements.find((el) => el.elementType === t); const desc = (t: string) => byType(t)?.description ?? null; @@ -305,10 +308,11 @@ async function resolveNewConditionReport( heatLossCorridor: flat?.heatLossCorridor != null ? flat.heatLossCorridor > 0 : null, unheatedCorridorLength: flat?.unheatedCorridorLengthM ?? null, - numberStoreys: chosen.numberOfStoreys ?? null, + // Storeys: epc_property value if present, else the main part's floor count. + numberStoreys: chosen.numberOfStoreys ?? storeyCountFromFloors, // Room height of the main building part (the new graph stores height per // building part rather than one dwelling value). - floorHeight: mainFloor?.roomHeightM ?? null, + floorHeight, numberHeatedRooms: chosen.heatedRoomsCount ?? null, numberOpenFireplaces: chosen.openChimneysCount ?? null, numberExtensions: chosen.extensionsCount ?? null, @@ -528,17 +532,25 @@ async function resolvePropertyDescriptors( propertyType: string | null; builtForm: string | null; yearBuilt: string | null; + numberOfRooms: number | null; }> { + const cols = { + id: true, + propertyType: true, + builtForm: true, + dwellingType: true, + habitableRoomsCount: true, + } as const; const ep = (await db.query.epcProperty.findFirst({ - columns: { id: true, propertyType: true, builtForm: true, dwellingType: true }, + columns: cols, where: and( eq(epcProperty.propertyId, propertyId), eq(epcProperty.source, "lodged"), ), })) ?? (await db.query.epcProperty.findFirst({ - columns: { id: true, propertyType: true, builtForm: true, dwellingType: true }, + columns: cols, where: and( eq(epcProperty.propertyId, propertyId), eq(epcProperty.source, "predicted"), @@ -546,7 +558,12 @@ async function resolvePropertyDescriptors( })); if (!ep) { - return { propertyType: null, builtForm: null, yearBuilt: null }; + return { + propertyType: null, + builtForm: null, + yearBuilt: null, + numberOfRooms: null, + }; } const mainPart = await db.query.epcBuildingPart.findFirst({ @@ -566,7 +583,12 @@ async function resolvePropertyDescriptors( const ageBand = mainPart?.constructionAgeBand ?? null; const yearBuilt = ageBand ? AGE_BAND_YEARS[ageBand] ?? ageBand : null; - return { propertyType, builtForm, yearBuilt }; + return { + propertyType, + builtForm, + yearBuilt, + numberOfRooms: ep.habitableRoomsCount ?? null, + }; } /** @@ -619,6 +641,7 @@ export async function resolvePropertyMeta(propertyId: string) { propertyType: descriptors?.propertyType ?? propertyMeta.propertyType, builtForm: descriptors?.builtForm ?? propertyMeta.builtForm, yearBuilt: descriptors?.yearBuilt ?? propertyMeta.yearBuilt, + numberOfRooms: descriptors?.numberOfRooms ?? propertyMeta.numberOfRooms, currentSapPoints: headline.currentSapPoints ?? propertyMeta.currentSapPoints, currentEpcRating: From 1aeb33762e8993fa71ee59a0ea6d6e404eb15f55 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 11:30:46 +0000 Subject: [PATCH 14/27] fix(building-passport): fill tenure from EPC graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tenure read the property-row tenure column, null for new-approach properties. Backfill propertyMeta.tenure from epc_property.tenure (mostly text labels like "Rented Social"; numeric SAP codes 1/2/3 mapped to the same labels). Local authority / constituency have no source in the new pipeline (not on epc_property, null on the property row, energy_assessments doesn't cover these properties) — left as-is pending the pipeline writing them to the property row. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/services/epcSources.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index e4987ab7..62db4a70 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -504,6 +504,14 @@ const BUILT_FORM_LABELS: Record = { "5": "Enclosed End-Terrace", "6": "Enclosed Mid-Terrace", }; +// SAP tenure codes → the text labels already used in the data. NOTE: best-effort +// (standard SAP 1/2/3); most rows store the text label directly, only a few use +// codes. Confirm against the backend tenure enum. +const TENURE_LABELS: Record = { + "1": "Owner Occupied", + "2": "Rented Social", + "3": "Rented Private", +}; // Matches the backend ConstructionAgeBand enum (RdSAP England & Wales letters). const AGE_BAND_YEARS: Record = { A: "before 1900", @@ -533,6 +541,7 @@ async function resolvePropertyDescriptors( builtForm: string | null; yearBuilt: string | null; numberOfRooms: number | null; + tenure: string | null; }> { const cols = { id: true, @@ -540,6 +549,7 @@ async function resolvePropertyDescriptors( builtForm: true, dwellingType: true, habitableRoomsCount: true, + tenure: true, } as const; const ep = (await db.query.epcProperty.findFirst({ @@ -563,6 +573,7 @@ async function resolvePropertyDescriptors( builtForm: null, yearBuilt: null, numberOfRooms: null, + tenure: null, }; } @@ -583,11 +594,16 @@ async function resolvePropertyDescriptors( const ageBand = mainPart?.constructionAgeBand ?? null; const yearBuilt = ageBand ? AGE_BAND_YEARS[ageBand] ?? ageBand : null; + const tenure = ep.tenure + ? TENURE_LABELS[ep.tenure] ?? ep.tenure + : null; + return { propertyType, builtForm, yearBuilt, numberOfRooms: ep.habitableRoomsCount ?? null, + tenure, }; } @@ -642,6 +658,7 @@ export async function resolvePropertyMeta(propertyId: string) { builtForm: descriptors?.builtForm ?? propertyMeta.builtForm, yearBuilt: descriptors?.yearBuilt ?? propertyMeta.yearBuilt, numberOfRooms: descriptors?.numberOfRooms ?? propertyMeta.numberOfRooms, + tenure: descriptors?.tenure ?? propertyMeta.tenure, currentSapPoints: headline.currentSapPoints ?? propertyMeta.currentSapPoints, currentEpcRating: From a89e6bfe30f7c4ff5c20040b1c5f8f90664a66c6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 11:44:57 +0000 Subject: [PATCH 15/27] feat(building-passport): show longitude/latitude in Location & Status Local authority and constituency have no data source for new-approach properties (not on epc_property; null on the property row). Replace them with longitude/latitude from property_details_spatial (already fetched on the page), keeping Tenure. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[slug]/building-passport/[propertyId]/page.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx index 1035edfe..174f274b 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx @@ -218,9 +218,15 @@ export default async function BuildingPassportHome(props: { {/* Location & Status */}

Location & Status

- - + +
{/* Annual Energy Costs */} From 65db15d3c7796c7a69786c83eac3422ff44a7238 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 20:15:37 +0000 Subject: [PATCH 16/27] fix(plans): show effective (modelling) baseline as plan "current" SAP/EPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plans are scored against the effective baseline on property_baseline_performance, which re-baselining can move away from the lodged EPC shown everywhere else. Using lodged as a plan's "current" misrepresented the uplift — e.g. lodged C/71 vs effective D/64 with a post-retrofit C/69.6 read as a downgrade (71 → 69.6) instead of the real D → C improvement. Add resolveEffectiveHeadline + getPlanPropertyMeta and switch both plan pages to it. Legacy properties and backfill gaps fall back to the lodged/row values. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[propertyId]/plans/[planId]/page.tsx | 4 +-- .../[propertyId]/plans/page.tsx | 4 +-- .../building-passport/[propertyId]/utils.ts | 22 ++++++++++++++ src/lib/services/epcSources.ts | 30 +++++++++++++++++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx index 320f6685..c3cc05d2 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx @@ -1,6 +1,6 @@ import RecommendationContainer from "@/app/components/building-passport/RecommendationContainer"; import { - getPropertyMeta, + getPlanPropertyMeta, getRecommendations, getPlanMeta, getInstalledMeasuresByUprn, @@ -59,7 +59,7 @@ export default async function PlanDetail(props: { params: Promise<{ slug: string; propertyId: string; planId: string }>; }) { const params = await props.params; - const propertyMeta = await getPropertyMeta(params.propertyId); + const propertyMeta = await getPlanPropertyMeta(params.propertyId); const [recommendations, planMeta, installedMeasures, scenarioData] = await Promise.all([ getRecommendations(params.planId), diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx index 735c5638..47a1d5ea 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx @@ -1,4 +1,4 @@ -import { getPlans, getPropertyMeta } from "../utils"; +import { getPlans, getPlanPropertyMeta } from "../utils"; import { sapToEpc } from "@/app/utils"; import PlanCard from "./PlanCard"; import PlanHeroCard from "./PlanHeroCard"; @@ -9,7 +9,7 @@ export default async function RecommendationPlans(props: { }) { const params = await props.params; const [propertyMeta, plans] = await Promise.all([ - getPropertyMeta(params.propertyId), + getPlanPropertyMeta(params.propertyId), getPlans(params.propertyId), ]); diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index 89875a9b..022d76a9 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -24,6 +24,7 @@ import { ConditionReport, resolveConditionReport, resolvePropertyMeta, + resolveEffectiveHeadline, } from "@/lib/services/epcSources"; import { getRating, serializeBigInt } from "@/app/utils"; import { eq, desc, and } from "drizzle-orm"; @@ -254,6 +255,27 @@ export async function getPropertyMeta( return JSON.parse(JSON.stringify(meta, serializeBigInt)) as PropertyMeta; } +/** + * Property meta for the plan pages. Identical to `getPropertyMeta` except the + * "current" SAP/EPC is the effective (modelling) baseline the plans were scored + * against, not the lodged EPC — see `resolveEffectiveHeadline`. Legacy + * properties and backfill gaps fall back to the lodged/row values. + */ +export async function getPlanPropertyMeta( + propertyId: string +): Promise { + const meta = await getPropertyMeta(propertyId); + const effective = await resolveEffectiveHeadline( + BigInt(propertyId), + meta.updatedAt + ); + return { + ...meta, + currentSapPoints: effective.currentSapPoints ?? meta.currentSapPoints, + currentEpcRating: effective.currentEpcRating ?? meta.currentEpcRating, + }; +} + export async function getConditionReport( propertyId: string ): Promise { diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index 62db4a70..42d50410 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -428,6 +428,36 @@ export async function resolvePropertyHeadline( }; } +/** + * Headline SAP score + EPC band for the *plan* pages. Plans are scored against + * the modelling baseline (`effective_*` on property_baseline_performance), which + * re-baselining can move away from the lodged (gov-registry) EPC we show + * everywhere else. Using the lodged figure as a plan's "current" would + * misrepresent the uplift — e.g. lodged C/71 vs an effective baseline of D/64 + * with a post-retrofit C/69.6 reads as a *downgrade* (71 → 69.6) instead of the + * real D → C improvement. So plan pages take the effective baseline as "current". + * + * Returns nulls for legacy properties (no effective/lodged split — keep the + * property-row values) and when no baseline row exists yet (backfill gap — fall + * back to the lodged headline). + */ +export async function resolveEffectiveHeadline( + propertyId: bigint, + updatedAt: Date | string | null | undefined, +): Promise<{ currentSapPoints: number | null; currentEpcRating: string | null }> { + if (!isNewApproach(updatedAt)) { + return { currentSapPoints: null, currentEpcRating: null }; + } + const baseline = await db.query.propertyBaselinePerformance.findFirst({ + columns: { effectiveSapScore: true, effectiveEpcBand: true }, + where: eq(propertyBaselinePerformance.propertyId, propertyId), + }); + return { + currentSapPoints: baseline?.effectiveSapScore ?? null, + currentEpcRating: baseline?.effectiveEpcBand ?? null, + }; +} + /** The minimal EPC meta the `/api/property-meta` route embeds as `detailsEpc`. */ export interface DetailsEpcMeta { currentEnergyDemand: number | null; From e51bf1d23fb6005645499ef9034bf276a78d97c1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 20:47:06 +0000 Subject: [PATCH 17/27] fix(building-passport): use effective (modelling) baseline for current SAP/EPC/CO2 Widen the lodged->effective switch from plan-only to all per-property building-passport views (overview, assessment, plans). New-approach properties' "current" SAP/EPC/CO2/primary-energy now come from the effective (re-baselined) figure the modelling and plans were scored against, so a property's current state agrees with the plan maths -- re-baselining (e.g. pre_sap10) otherwise made plans read as flat or as a downgrade against the lodged gov-register EPC. Done by flipping the shared object resolvers (resolvePropertyHeadline, resolveDetailsEpcMeta, resolveNewConditionReport) to effective_*, which removes the plan-only getPlanPropertyMeta/resolveEffectiveHeadline indirection added in the previous commit. The set-based reporting fragments (sapSql/epcBandSql/carbonSql/...) stay on lodged_* so portfolio and reporting totals remain matched to the gov register. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[propertyId]/plans/[planId]/page.tsx | 4 +- .../[propertyId]/plans/page.tsx | 4 +- .../building-passport/[propertyId]/utils.ts | 22 -------- src/lib/services/epcSources.ts | 54 ++++++------------- 4 files changed, 21 insertions(+), 63 deletions(-) diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx index c3cc05d2..320f6685 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/[planId]/page.tsx @@ -1,6 +1,6 @@ import RecommendationContainer from "@/app/components/building-passport/RecommendationContainer"; import { - getPlanPropertyMeta, + getPropertyMeta, getRecommendations, getPlanMeta, getInstalledMeasuresByUprn, @@ -59,7 +59,7 @@ export default async function PlanDetail(props: { params: Promise<{ slug: string; propertyId: string; planId: string }>; }) { const params = await props.params; - const propertyMeta = await getPlanPropertyMeta(params.propertyId); + const propertyMeta = await getPropertyMeta(params.propertyId); const [recommendations, planMeta, installedMeasures, scenarioData] = await Promise.all([ getRecommendations(params.planId), diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx index 47a1d5ea..735c5638 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/plans/page.tsx @@ -1,4 +1,4 @@ -import { getPlans, getPlanPropertyMeta } from "../utils"; +import { getPlans, getPropertyMeta } from "../utils"; import { sapToEpc } from "@/app/utils"; import PlanCard from "./PlanCard"; import PlanHeroCard from "./PlanHeroCard"; @@ -9,7 +9,7 @@ export default async function RecommendationPlans(props: { }) { const params = await props.params; const [propertyMeta, plans] = await Promise.all([ - getPlanPropertyMeta(params.propertyId), + getPropertyMeta(params.propertyId), getPlans(params.propertyId), ]); diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts index 022d76a9..89875a9b 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/utils.ts @@ -24,7 +24,6 @@ import { ConditionReport, resolveConditionReport, resolvePropertyMeta, - resolveEffectiveHeadline, } from "@/lib/services/epcSources"; import { getRating, serializeBigInt } from "@/app/utils"; import { eq, desc, and } from "drizzle-orm"; @@ -255,27 +254,6 @@ export async function getPropertyMeta( return JSON.parse(JSON.stringify(meta, serializeBigInt)) as PropertyMeta; } -/** - * Property meta for the plan pages. Identical to `getPropertyMeta` except the - * "current" SAP/EPC is the effective (modelling) baseline the plans were scored - * against, not the lodged EPC — see `resolveEffectiveHeadline`. Legacy - * properties and backfill gaps fall back to the lodged/row values. - */ -export async function getPlanPropertyMeta( - propertyId: string -): Promise { - const meta = await getPropertyMeta(propertyId); - const effective = await resolveEffectiveHeadline( - BigInt(propertyId), - meta.updatedAt - ); - return { - ...meta, - currentSapPoints: effective.currentSapPoints ?? meta.currentSapPoints, - currentEpcRating: effective.currentEpcRating ?? meta.currentEpcRating, - }; -} - export async function getConditionReport( propertyId: string ): Promise { diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index 42d50410..cbec24b5 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -320,9 +320,11 @@ async function resolveNewConditionReport( // GAP: no energy tariff in the new graph — omitted. energyTariff: null, currentEnergyDemand: sumKwh, + // Effective (re-baselined) baseline so the assessment view agrees with the + // headline SAP/EPC and the plan maths — see resolvePropertyHeadline. primaryEnergyConsumption: - baseline?.lodgedPrimaryEnergyIntensityKwhPerM2Yr ?? null, - co2Emissions: baseline?.lodgedCo2EmissionsTPerYr ?? null, + baseline?.effectivePrimaryEnergyIntensityKwhPerM2Yr ?? null, + co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, heatingEnergyCostCurrent: baseline?.heatingCostGbp ?? null, hotWaterEnergyCostCurrent: baseline?.hotWaterCostGbp ?? null, lightingEnergyCostCurrent: baseline?.lightingCostGbp ?? null, @@ -405,45 +407,23 @@ export async function resolveConditionReport( /** * Headline SAP score + EPC band for the property meta. New-approach properties * don't have `current_sap_points` / `current_epc_rating` written on the row yet, - * so we fall back to the lodged baseline (property_baseline_performance). Returns - * nulls for legacy properties, signalling the caller to keep the row values. + * so we fall back to the modelling baseline (property_baseline_performance). + * Returns nulls for legacy properties, signalling the caller to keep the row + * values. * - * Uses lodged_* (gov-registry actual) to match the rest of the EPC reads; swap to - * effective_* for the modelling-adjusted baseline. + * Uses `effective_*` — the re-baselined figure the modelling (and therefore the + * plans) was scored against. The per-property building-passport views (overview, + * assessment, plans) must agree with the plan maths: re-baselining can move the + * effective score away from the lodged (gov-registry) EPC, so showing lodged as + * "current" misrepresents a plan's uplift — e.g. lodged C/78 vs an effective + * baseline of C/71 with a post-retrofit C/71.4 reads as flat instead of the real + * improvement. The set-based reporting fragments (`sapSql`, `epcBandSql`, + * `carbonSql`, …) deliberately stay on `lodged_*` to keep portfolio/reporting + * totals matched to the gov register. */ export async function resolvePropertyHeadline( propertyId: bigint, updatedAt: Date | string | null | undefined, -): Promise<{ currentSapPoints: number | null; currentEpcRating: string | null }> { - if (!isNewApproach(updatedAt)) { - return { currentSapPoints: null, currentEpcRating: null }; - } - const baseline = await db.query.propertyBaselinePerformance.findFirst({ - columns: { lodgedSapScore: true, lodgedEpcBand: true }, - where: eq(propertyBaselinePerformance.propertyId, propertyId), - }); - return { - currentSapPoints: baseline?.lodgedSapScore ?? null, - currentEpcRating: baseline?.lodgedEpcBand ?? null, - }; -} - -/** - * Headline SAP score + EPC band for the *plan* pages. Plans are scored against - * the modelling baseline (`effective_*` on property_baseline_performance), which - * re-baselining can move away from the lodged (gov-registry) EPC we show - * everywhere else. Using the lodged figure as a plan's "current" would - * misrepresent the uplift — e.g. lodged C/71 vs an effective baseline of D/64 - * with a post-retrofit C/69.6 reads as a *downgrade* (71 → 69.6) instead of the - * real D → C improvement. So plan pages take the effective baseline as "current". - * - * Returns nulls for legacy properties (no effective/lodged split — keep the - * property-row values) and when no baseline row exists yet (backfill gap — fall - * back to the lodged headline). - */ -export async function resolveEffectiveHeadline( - propertyId: bigint, - updatedAt: Date | string | null | undefined, ): Promise<{ currentSapPoints: number | null; currentEpcRating: string | null }> { if (!isNewApproach(updatedAt)) { return { currentSapPoints: null, currentEpcRating: null }; @@ -500,7 +480,7 @@ export async function resolveDetailsEpcMeta( return { currentEnergyDemand, - co2Emissions: baseline?.lodgedCo2EmissionsTPerYr ?? null, + co2Emissions: baseline?.effectiveCo2EmissionsTPerYr ?? null, estimated: !lodged && !!predicted, }; } From ec88dadae2b02d9ea284eaea36331771fbcb2b78 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 22:15:08 +0000 Subject: [PATCH 18/27] docs: record effective-as-canonical-current decision + provenance terms ADR-0002: Effective performance is the canonical "current" for display and reporting; lodged shown alongside (supersedes the mid-migration "reporting stays lodged" note). CONTEXT.md gains EPC-provenance and provenance-signal terms, and flags that the UI "Current EPC" surfaces show Effective. Co-Authored-By: Claude Opus 4.8 (1M context) --- CONTEXT.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CONTEXT.md b/CONTEXT.md index c2073354..cdd69ab7 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -124,6 +124,14 @@ _Avoid_: current performance, adjusted performance The act of substituting a corrected set of performance metrics in place of the Lodged values. Recorded on `property_baseline_performance` with a `rebaseline_reason` enum value: `none`, `pre_sap10`, `physical_state_changed`, or `both`. _Avoid_: override, adjustment, correction +**EPC provenance** (`epc_property.source`): +Whether a property's EPC picture is a real certificate or a gap-fill. `lodged` = a real public/landlord EPC exists; `predicted` = no certificate, so the EPC was estimated from nearby properties (EPC Prediction gap-fill). Independent of Rebaseline: a `lodged` property may still be rebaselined, and a `predicted` property still carries Lodged-performance figures (mirrored estimates), so the presence of `lodged_*` columns does **not** imply a real certificate — only `source = lodged` does. +_Avoid_: estimated EPC (reserve "estimated" for the UI signal), source EPC + +**Provenance signal** (UI): +What the user is told about an EPC's trustworthiness, derived from EPC provenance and Rebaseline together: **Estimated** (`source = predicted` — dominant; "estimated based on nearby homes") › **Re-modelled** (`source = lodged AND rebaseline_reason != none` — effective diverged from the lodged certificate under SAP 10) › **none** (`source = lodged AND rebaseline_reason = none` — effective equals lodged). Estimated always wins when both could apply. +_Avoid_: estimation notification, banner (those are component names) + ## Example dialogue > **Dev:** "If the **Combiner** finishes but the user hasn't clicked Finalise, what does the user see?" @@ -136,5 +144,6 @@ _Avoid_: override, adjustment, correction ## Flagged ambiguities +- The UI surfaces labelled **"Current EPC"** and **"Current Efficiency State"** (property table, building-passport card) show **Effective performance**, not Lodged — despite the glossary advising against "current performance" for Effective. The label is a product choice ("current" = the property's true present-day modelled state); the underlying datum is still **Effective performance**. The **"Lodged EPC"** column/badge is the only surface showing **Lodged performance**, and only when a real certificate exists (`source = lodged`). - "Upload" is used in the codebase to mean both the file-on-S3 and the BulkUpload row. We standardise on **BulkUpload** for the row; the file is just "the source file." - "Onboarding" appears in some route paths (`bulk_onboarding_inputs/...`) but isn't part of this glossary — we use **BulkUpload** end-to-end. From ea988cfe529d93ecc3d142d3d0b27dc76fe80da3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 22:16:01 +0000 Subject: [PATCH 19/27] feat(epc): provenance signals + effective-baseline portfolio table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theme 1 — EPC card provenance. New resolveProvenanceSignal (pure, unit- tested) + resolveProvenance expose provenanceSignal ("estimated" / "remodelled" / "none") and the lodged headline on property-meta. The building-passport card shows a "Lodged EPC" badge on re-modelled properties (even within-band), never on predicted ones; the existing "estimated from nearby homes" banner already covers predicted. Theme 2 — portfolio table. "Current EPC" + reporting now read the effective baseline (effectiveSapSql/effectiveEpcBandSql); a dedicated "Lodged EPC" column reads the source-gated lodged value (blank for predicted); a "Predicted" pill marks estimated rows; a provenance filter is added and the broken currentEpc/lodgedEpc filters (on null row columns) point at the effective/lodged baselines. Expected EPC now uses the default plan's post rating (expectedEpcRating, unit-tested) instead of current + Σ rec SAP, which double-counted across plans (732385: A→C); the same rec aggregate is scoped to the default plan, fixing the Cost column's 2x double-count. See ADR-0002. Pure logic covered by provenance.test.ts + expectedEpc.test.ts; SQL/query/UI verified against live properties 732385 and 729529. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CurrentEfficiencyCard.tsx | 24 ++++- src/app/db/schema/property.ts | 15 +++ .../reporting/databaseFunctions.ts | 8 +- .../[propertyId]/assessment/page.tsx | 3 + .../building-passport/[propertyId]/page.tsx | 3 + .../[slug]/components/PropertyFilters.tsx | 3 + .../[slug]/components/expectedEpc.test.ts | 20 ++++ .../[slug]/components/expectedEpc.ts | 19 ++++ .../components/propertyTableColumns.tsx | 53 ++++++---- src/app/portfolio/[slug]/utils.ts | 44 ++++++--- src/app/utils/propertyFilters.ts | 9 ++ src/lib/services/epcSources.ts | 96 ++++++++++++++++++- src/lib/services/provenance.test.ts | 59 ++++++++++++ src/lib/services/provenance.ts | 36 +++++++ 14 files changed, 353 insertions(+), 39 deletions(-) create mode 100644 src/app/portfolio/[slug]/components/expectedEpc.test.ts create mode 100644 src/app/portfolio/[slug]/components/expectedEpc.ts create mode 100644 src/lib/services/provenance.test.ts create mode 100644 src/lib/services/provenance.ts diff --git a/src/app/components/building-passport/CurrentEfficiencyCard.tsx b/src/app/components/building-passport/CurrentEfficiencyCard.tsx index 723cb630..7aad49fc 100644 --- a/src/app/components/building-passport/CurrentEfficiencyCard.tsx +++ b/src/app/components/building-passport/CurrentEfficiencyCard.tsx @@ -5,6 +5,9 @@ import { LodgedEpcTooltip } from "./LodgedEpcTooltip"; interface CurrentEfficiencyCardProps { epcRating: string | null; sapPoints: number; + provenanceSignal?: "estimated" | "remodelled" | "none"; + lodgedEpcRating?: string | null; + lodgedSapPoints?: number | null; originalSapPoints?: number | null; } @@ -58,11 +61,24 @@ function getEpcDescription(letter: string | null | undefined): string { export function CurrentEfficiencyCard({ epcRating, sapPoints, + provenanceSignal = "none", + lodgedEpcRating, + lodgedSapPoints, originalSapPoints, }: CurrentEfficiencyCardProps) { const epcHex = getEpcHex(epcRating); - const lodgedLetter = originalSapPoints ? sapToEpc(originalSapPoints) : null; - const showLodgedBadge = lodgedLetter !== null && lodgedLetter !== epcRating; + // New-approach: show the lodged certificate whenever the property was + // re-modelled (effective diverged from lodged), even within the same band. + // Legacy: fall back to the row's originalSapPoints, only when the band differs. + const legacyLodgedLetter = originalSapPoints ? sapToEpc(originalSapPoints) : null; + const remodelled = provenanceSignal === "remodelled"; + const lodgedLetter = remodelled + ? lodgedEpcRating ?? null + : legacyLodgedLetter !== epcRating + ? legacyLodgedLetter + : null; + const lodgedBadgeSap = remodelled ? lodgedSapPoints : originalSapPoints; + const showLodgedBadge = lodgedLetter !== null; const lodgedHex = getEpcHex(lodgedLetter); return ( @@ -87,9 +103,9 @@ export function CurrentEfficiencyCard({ > {lodgedLetter} - {originalSapPoints != null && ( + {lodgedBadgeSap != null && ( - / {Math.round(originalSapPoints)} + / {Math.round(lodgedBadgeSap)} )} diff --git a/src/app/db/schema/property.ts b/src/app/db/schema/property.ts index 3ae12aeb..8cb86c42 100644 --- a/src/app/db/schema/property.ts +++ b/src/app/db/schema/property.ts @@ -38,6 +38,14 @@ export interface PropertyMeta { currentEpcRating: string; currentSapPoints: number; originalSapPoints: number | null; + // Provenance of the displayed "current" figures (see ADR-0002 / provenance.ts). + // "estimated" = no certificate (predicted); "remodelled" = lodged but + // rebaselined; "none" = lodged == effective, or legacy. + provenanceSignal: "estimated" | "remodelled" | "none"; + // Lodged (gov-register) headline, only when a real certificate exists; feeds + // the "Lodged EPC" badge alongside the effective "current". + lodgedEpcRating: string | null; + lodgedSapPoints: number | null; updatedAt: string; currentValuation: number | null; detailsEpc: { @@ -378,6 +386,13 @@ export interface PropertyWithRelations extends Record { // New fields landlordPropertyId: string | null; originalSapPoints: number | null; + // Effective "current" (ADR-0002) is in currentEpcRating/currentSapPoints; these + // carry the separate lodged certificate + the default plan's post-retrofit + // rating + the provenance signal that drives the "Predicted" pill. + lodgedEpcRating: string | null; + postEpcRating: string | null; + postSapPoints: number | null; + provenanceSignal: "estimated" | "remodelled" | "none"; epcLodgementDate: string | null; epcIsExpired: boolean | null; // Optional columns (hidden by default) diff --git a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts index db61d9df..4fbe71d1 100644 --- a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts +++ b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts @@ -9,8 +9,8 @@ import { energyConsumptionSql, estimatedSql, isExpiredSql, - sapSql, - epcBandSql, + effectiveSapSql, + effectiveEpcBandSql, } from "@/lib/services/epcSources"; import type { @@ -38,7 +38,7 @@ export async function getAverages( ): Promise { const result = await db.execute(sql` SELECT - AVG(${sapSql})::float AS avg_sap, + AVG(${effectiveSapSql})::float AS avg_sap, AVG(${carbonSql(sql`e`)})::float AS avg_carbon, AVG(${billsSql(sql`e`)})::float AS avg_bills, AVG(${energyConsumptionSql(sql`e`)})::float AS avg_energy_consumption @@ -99,7 +99,7 @@ export async function getCountByEpcBand( SELECT * FROM ( SELECT - COALESCE((${epcBandSql})::text, 'Unknown') AS epc, + COALESCE((${effectiveEpcBandSql})::text, 'Unknown') AS epc, COUNT(*) FILTER ( WHERE ${estimatedSql(sql`e`)} = false )::int AS actual, diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/assessment/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/assessment/page.tsx index a2123cb9..dacb6205 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/assessment/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/assessment/page.tsx @@ -238,6 +238,9 @@ export default async function PreAssessmentReport(props: { diff --git a/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx b/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx index 174f274b..92c12eed 100644 --- a/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx +++ b/src/app/portfolio/[slug]/building-passport/[propertyId]/page.tsx @@ -86,6 +86,9 @@ export default async function BuildingPassportHome(props: { diff --git a/src/app/portfolio/[slug]/components/PropertyFilters.tsx b/src/app/portfolio/[slug]/components/PropertyFilters.tsx index 546b2b75..7add0920 100644 --- a/src/app/portfolio/[slug]/components/PropertyFilters.tsx +++ b/src/app/portfolio/[slug]/components/PropertyFilters.tsx @@ -16,6 +16,7 @@ import { TENURE_OPTIONS, YEAR_BUILT_OPTIONS, MAINFUEL_OPTIONS, + PROVENANCE_OPTIONS, } from "@/app/utils/propertyFilters"; /* ----------------------------------------------------------------------- @@ -33,6 +34,7 @@ const FIELD_OPTIONS: { value: FilterField; label: string }[] = [ { value: "builtForm", label: "Built Form" }, { value: "tenure", label: "Tenure" }, { value: "yearBuilt", label: "Year Built" }, + { value: "provenance", label: "Provenance" }, { value: "floorArea", label: "Floor Area (m²)" }, { value: "co2Emissions", label: "CO₂ Emissions (kg/m²/yr)" }, { value: "mainfuel", label: "Main Fuel" }, @@ -74,6 +76,7 @@ const ENUM_FIELD_OPTIONS: Record = { builtForm: BUILT_FORM_OPTIONS, tenure: TENURE_OPTIONS, yearBuilt: YEAR_BUILT_OPTIONS, + provenance: PROVENANCE_OPTIONS, mainfuel: MAINFUEL_OPTIONS, }; diff --git a/src/app/portfolio/[slug]/components/expectedEpc.test.ts b/src/app/portfolio/[slug]/components/expectedEpc.test.ts new file mode 100644 index 00000000..50a09aac --- /dev/null +++ b/src/app/portfolio/[slug]/components/expectedEpc.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { expectedEpcRating } from "./expectedEpc"; + +describe("expectedEpcRating", () => { + it("uses the default plan's post EPC rating when present", () => { + expect( + expectedEpcRating({ postEpcRating: "C", postSapPoints: 71.4 }), + ).toBe("C"); + }); + + it("derives the band from post SAP when the plan has no stored rating", () => { + expect( + expectedEpcRating({ postEpcRating: null, postSapPoints: 64 }), + ).toBe("D"); + }); + + it("is blank when the property has no default plan", () => { + expect(expectedEpcRating(null)).toBe(""); + }); +}); diff --git a/src/app/portfolio/[slug]/components/expectedEpc.ts b/src/app/portfolio/[slug]/components/expectedEpc.ts new file mode 100644 index 00000000..686e5cf1 --- /dev/null +++ b/src/app/portfolio/[slug]/components/expectedEpc.ts @@ -0,0 +1,19 @@ +import { sapToEpc } from "@/app/utils"; + +/** + * The portfolio "Expected EPC" — the post-retrofit band of a property's default + * plan. The plan engine already models interactions, SAP 10 and the effective + * baseline, so we trust `plan.post_epc_rating` (falling back to deriving it from + * `post_sap_points`) rather than adding raw recommendation SAP onto the current + * score, which double-counts across plans and overshoots. See ADR-0002 and the + * grilling notes for property 732385 (read A, should read C). + * + * Returns "" when there is no default plan — the column renders blank, not the + * current rating. + */ +export function expectedEpcRating( + plan: { postEpcRating: string | null; postSapPoints: number | null } | null, +): string { + if (!plan) return ""; + return plan.postEpcRating ?? sapToEpc(plan.postSapPoints); +} diff --git a/src/app/portfolio/[slug]/components/propertyTableColumns.tsx b/src/app/portfolio/[slug]/components/propertyTableColumns.tsx index 7ef022eb..9d2955d5 100644 --- a/src/app/portfolio/[slug]/components/propertyTableColumns.tsx +++ b/src/app/portfolio/[slug]/components/propertyTableColumns.tsx @@ -15,6 +15,7 @@ import { FunnelIcon } from "@heroicons/react/24/outline"; import { formatNumber, getEpcColorClass, sapToEpc } from "@/app/utils"; import { cn } from "@/lib/utils"; import { PropertyWithRelations } from "@/app/db/schema/property"; +import { expectedEpcRating } from "./expectedEpc"; import { X } from "lucide-react"; import { EnumOption, @@ -72,6 +73,20 @@ const EpcLetterBubble = ({ letter }: { letter: string }) => { ); }; +/** + * Marks a property whose EPC was estimated from nearby homes (no certificate) — + * the "trust this less" signal. Re-modelled properties aren't marked (the + * Lodged-EPC column differing from Current already conveys that). See ADR-0002. + */ +const PredictedPill = () => ( + + Predicted + +); + /* ----------------------------------------------------------------------- Column header with dropdown filter ------------------------------------------------------------------------ */ @@ -229,19 +244,27 @@ const coreColumns: ColumnDef[] = [ ), cell: ({ row }) => ( -
+
+ {row.original.provenanceSignal === "estimated" && }
), }, { - accessorKey: "originalSapPoints", + accessorKey: "lodgedEpcRating", header: () => (
Lodged EPC
), cell: ({ row }) => { - const originalSap = row.original.originalSapPoints; - const letter = originalSap ? sapToEpc(originalSap) : null; + // Effective is the "current"; this column shows the real lodged + // certificate, blank when there is none (predicted properties). For legacy + // rows the query returns the row's rating; new-approach returns lodged or + // NULL. originalSapPoints remains a fallback for legacy rows lacking a band. + const letter = + row.original.lodgedEpcRating ?? + (row.original.originalSapPoints + ? sapToEpc(row.original.originalSapPoints) + : null); return (
@@ -255,21 +278,17 @@ const coreColumns: ColumnDef[] = [
Expected EPC
), cell: ({ row }) => { - const currentSapPoints = row.original.currentSapPoints || 0; - const expectedSapPoints = row.original.totalRecommendationSapPoints || 0; - - if (currentSapPoints + expectedSapPoints === 0) { - return ( -
- -
- ); - } - const expectedEpc = sapToEpc(currentSapPoints + expectedSapPoints); - + const expectedEpc = expectedEpcRating( + row.original.postEpcRating != null || row.original.postSapPoints != null + ? { + postEpcRating: row.original.postEpcRating, + postSapPoints: row.original.postSapPoints, + } + : null, + ); return (
- +
); }, diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index 40e612a5..b429f6c8 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -26,8 +26,11 @@ import { lodgementDateSql, isExpiredSql, mainfuelSql, - sapSql, - epcBandSql, + effectiveSapSql, + effectiveEpcBandSql, + lodgedEpcBandSql, + lodgedSapSql, + provenanceSignalSql, } from "@/lib/services/epcSources"; import { FilterGroups, @@ -37,6 +40,7 @@ import { TENURE_OPTIONS, YEAR_BUILT_OPTIONS, MAINFUEL_OPTIONS, + PROVENANCE_OPTIONS, EnumOption, } from "@/app/utils/propertyFilters"; import { EPC_TO_SAP_MIN, EPC_TO_SAP_MAX } from "@/app/utils/epc"; @@ -46,6 +50,7 @@ const ENUM_FIELD_DB_OPTIONS: Record = { builtForm: BUILT_FORM_OPTIONS, tenure: TENURE_OPTIONS, yearBuilt: YEAR_BUILT_OPTIONS, + provenance: PROVENANCE_OPTIONS, mainfuel: MAINFUEL_OPTIONS, }; @@ -523,10 +528,11 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul return null; case "currentEpc": - return buildEpcSapCondition(sql`p.current_sap_points`, filter.operator, filter.value); + // "Current" is the effective baseline (ADR-0002), not the null row column. + return buildEpcSapCondition(effectiveSapSql, filter.operator, filter.value); case "lodgedEpc": - return buildEpcSapCondition(sql`p.original_sap_points`, filter.operator, filter.value); + return buildEpcSapCondition(lodgedSapSql, filter.operator, filter.value); case "expectedEpc": { if (filter.operator === "epc_at_least") { @@ -574,6 +580,7 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul case "builtForm": case "tenure": case "yearBuilt": + case "provenance": case "mainfuel": { if (filter.operator !== "enum_one_of") return null; @@ -591,6 +598,7 @@ function buildConditionSql(filter: PropertyFilter): ReturnType | nul builtForm: sql`p.built_form`, tenure: sql`p.tenure`, yearBuilt: sql`p.year_built`, + provenance: provenanceSignalSql, // GAP: new-approach properties have no main-fuel string → NULL, so they // only match the "no value" filter branch. See epcSources.ts. mainfuel: mainfuelSql(sql`epc`), @@ -718,15 +726,20 @@ export async function getProperties( p.postcode AS postcode, p.status AS status, p.creation_status AS "creationStatus", - -- Current EPC/SAP: new-approach properties have null headline columns on - -- the property row, so source from the lodged baseline. The "Expected EPC" - -- column is currentSapPoints + recommendation SAP, so this also fixes it. - ${epcBandSql} AS "currentEpcRating", - ${sapSql} AS "currentSapPoints", - -- property_targets is no longer read (the "Expected EPC" column derives - -- from currentSapPoints + recommendation SAP). Kept as NULL to preserve - -- the PropertyWithRelations shape. + -- "Current" is the effective (re-baselined) figure — the canonical current + -- per ADR-0002. The lodged certificate is a separate column. New-approach + -- properties have null headline columns on the property row, so both source + -- from property_baseline_performance. + ${effectiveEpcBandSql} AS "currentEpcRating", + ${effectiveSapSql} AS "currentSapPoints", + ${lodgedEpcBandSql} AS "lodgedEpcRating", + ${provenanceSignalSql} AS "provenanceSignal", + -- "Expected EPC" is the default plan's modelled post-retrofit rating, not + -- current + Σ recommendation SAP (which double-counts across plans and + -- overshoots). See expectedEpc.ts / ADR-0002. NULL AS "targetEpc", + pl.post_epc_rating AS "postEpcRating", + pl.post_sap_points AS "postSapPoints", pl.id AS "planId", -- funding_package is no longer read here (the proposals table fetches it -- separately). Kept as NULL to preserve the PropertyWithRelations shape. @@ -749,7 +762,7 @@ export async function getProperties( -- LATERAL one-row lookups keep this query at "N properties × index probe" -- instead of hash-joining the multi-million-row plan/recommendation tables. LEFT JOIN LATERAL ( - SELECT id, post_sap_points FROM plan + SELECT id, post_sap_points, post_epc_rating FROM plan WHERE property_id = p.id AND portfolio_id = p.portfolio_id AND is_default = true @@ -768,6 +781,11 @@ export async function getProperties( COALESCE(SUM(estimated_cost), 0) AS cost FROM recommendation WHERE property_id = p.id + -- Scope to the default plan. property_id is the indexed access path, so + -- this only filters the property's own (tiny) rec set in memory — no + -- unindexed plan_id scan — while stopping the SUM double-counting a + -- measure once per plan when a property has several plans. + AND plan_id = pl.id AND "default" = true AND already_installed = false ) rec ON true diff --git a/src/app/utils/propertyFilters.ts b/src/app/utils/propertyFilters.ts index e67309ae..1f85675e 100644 --- a/src/app/utils/propertyFilters.ts +++ b/src/app/utils/propertyFilters.ts @@ -10,6 +10,7 @@ export type FilterField = | "builtForm" | "tenure" | "yearBuilt" + | "provenance" | "floorArea" | "co2Emissions" | "mainfuel"; @@ -89,6 +90,14 @@ export const TENURE_OPTIONS: EnumOption[] = [ { label: "Not Recorded", dbValues: ["__null__"] }, ]; +// Provenance signal (ADR-0002). dbValues are the raw signal strings emitted by +// provenanceSignalSql / resolveProvenanceSignal. +export const PROVENANCE_OPTIONS: EnumOption[] = [ + { label: "Predicted", dbValues: ["estimated"] }, + { label: "Re-modelled", dbValues: ["remodelled"] }, + { label: "Standard", dbValues: ["none"] }, +]; + export const YEAR_BUILT_OPTIONS: EnumOption[] = [ "1900","1930","1950","1967","1976","1983","1991","1996", "2003","2007","2008","2009","2010","2011","2012","2013", diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index cbec24b5..d1a77dc0 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -29,6 +29,7 @@ */ import { db } from "@/app/db/db"; import { and, eq, sql } from "drizzle-orm"; +import { resolveProvenanceSignal, type ProvenanceSignal } from "./provenance"; import { epcProperty, epcEnergyElement, @@ -98,6 +99,44 @@ export const sapSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_sap_score /** Baseline EPC band. New: lodged band from property_baseline_performance. Legacy: property.current_epc_rating. */ export const epcBandSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.lodged_epc_band ELSE p.current_epc_rating END`; +/** + * Effective (re-baselined) SAP score / EPC band — the canonical "current" for + * display and reporting (ADR-0002). New: bp.effective_*; legacy: the property + * row. These replaced the lodged `sapSql`/`epcBandSql` above as the table + + * reporting source; those lodged fragments are intentionally retained (unused + * for now) for a possible future register-fidelity reporting view (ADR-0002). + * The "Lodged EPC" column itself uses the source-gated `lodgedEpcBandSql` below. + */ +export const effectiveSapSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_sap_score ELSE p.current_sap_points END`; +export const effectiveEpcBandSql = sql`CASE WHEN ${isNewApproachSql} THEN bp.effective_epc_band ELSE p.current_epc_rating END`; + +/** + * Lodged EPC band for the "Lodged EPC" column — only when a real certificate + * exists (`source = lodged`, i.e. the lodged epc_property row `epl`). A + * predicted property carries lodged_* (mirrored estimates) that must not + * surface, so it renders NULL. Legacy: derive from the property row's + * original_sap_points (handled in the column, kept as the row value here). + */ +export const lodgedEpcBandSql = sql`CASE WHEN ${isNewApproachSql} THEN (CASE WHEN epl.id IS NOT NULL THEN bp.lodged_epc_band ELSE NULL END) ELSE p.current_epc_rating END`; + +/** Lodged SAP score for the "Lodged EPC" filter — same source/gate as lodgedEpcBandSql. Legacy: the row's original_sap_points. */ +export const lodgedSapSql = sql`CASE WHEN ${isNewApproachSql} THEN (CASE WHEN epl.id IS NOT NULL THEN bp.lodged_sap_score ELSE NULL END) ELSE p.original_sap_points END`; + +/** + * Provenance signal for the portfolio table (mirrors resolveProvenanceSignal / + * provenance.ts). New-approach only; legacy → 'none'. + */ +export const provenanceSignalSql = sql`CASE + WHEN ${isNewApproachSql} THEN ( + CASE + WHEN epp.id IS NOT NULL AND epl.id IS NULL THEN 'estimated' + WHEN epl.id IS NOT NULL AND bp.rebaseline_reason IS NOT NULL AND bp.rebaseline_reason <> 'none' THEN 'remodelled' + ELSE 'none' + END + ) + ELSE 'none' +END`; + /** * Annual energy bill (£). New: sum of the individual cost columns on * property_baseline_performance (NULL when no baseline row exists, so it's @@ -438,6 +477,57 @@ export async function resolvePropertyHeadline( }; } +/** + * Provenance signal + the lodged (gov-register) headline for the EPC card. + * `provenanceSignal` drives the "estimated"/"re-modelled" UI signals; the lodged + * band/SAP feed the "Lodged EPC" badge and are only meaningful when a real + * certificate exists (`source = lodged`). Legacy properties get `none` and null + * lodged figures — the card falls back to the row's `originalSapPoints`. + * See provenance.ts (pure precedence), CONTEXT.md and ADR-0002. + */ +export async function resolveProvenance( + propertyId: bigint, + updatedAt: Date | string | null | undefined, +): Promise<{ + provenanceSignal: ProvenanceSignal; + lodgedEpcRating: string | null; + lodgedSapPoints: number | null; +}> { + if (!isNewApproach(updatedAt)) { + return { provenanceSignal: "none", lodgedEpcRating: null, lodgedSapPoints: null }; + } + const [lodged, predicted, baseline] = await Promise.all([ + db.query.epcProperty.findFirst({ + columns: { id: true }, + where: and(eq(epcProperty.propertyId, propertyId), eq(epcProperty.source, "lodged")), + }), + db.query.epcProperty.findFirst({ + columns: { id: true }, + where: and(eq(epcProperty.propertyId, propertyId), eq(epcProperty.source, "predicted")), + }), + db.query.propertyBaselinePerformance.findFirst({ + columns: { rebaselineReason: true, lodgedSapScore: true, lodgedEpcBand: true }, + where: eq(propertyBaselinePerformance.propertyId, propertyId), + }), + ]); + + const hasLodgedEpc = !!lodged; + const provenanceSignal = resolveProvenanceSignal({ + isNewApproach: true, + hasLodgedEpc, + hasPredictedEpc: !!predicted, + rebaselineReason: baseline?.rebaselineReason, + }); + + // Lodged figures only when a real certificate exists — a predicted property + // still carries lodged_* (mirrored estimates) which must not surface. + return { + provenanceSignal, + lodgedEpcRating: hasLodgedEpc ? baseline?.lodgedEpcBand ?? null : null, + lodgedSapPoints: hasLodgedEpc ? baseline?.lodgedSapScore ?? null : null, + }; +} + /** The minimal EPC meta the `/api/property-meta` route embeds as `detailsEpc`. */ export interface DetailsEpcMeta { currentEnergyDemand: number | null; @@ -654,16 +744,20 @@ export async function resolvePropertyMeta(propertyId: string) { if (!propertyMeta) return null; const newApproach = isNewApproach(propertyMeta.updatedAt); - const [detailsEpc, headline, descriptors] = await Promise.all([ + const [detailsEpc, headline, descriptors, provenance] = await Promise.all([ resolveDetailsEpcMeta(propertyMeta.id, propertyMeta.updatedAt), resolvePropertyHeadline(propertyMeta.id, propertyMeta.updatedAt), newApproach ? resolvePropertyDescriptors(propertyMeta.id) : Promise.resolve(null), + resolveProvenance(propertyMeta.id, propertyMeta.updatedAt), ]); return { ...propertyMeta, + provenanceSignal: provenance.provenanceSignal, + lodgedEpcRating: provenance.lodgedEpcRating, + lodgedSapPoints: provenance.lodgedSapPoints, propertyType: descriptors?.propertyType ?? propertyMeta.propertyType, builtForm: descriptors?.builtForm ?? propertyMeta.builtForm, yearBuilt: descriptors?.yearBuilt ?? propertyMeta.yearBuilt, diff --git a/src/lib/services/provenance.test.ts b/src/lib/services/provenance.test.ts new file mode 100644 index 00000000..77c95423 --- /dev/null +++ b/src/lib/services/provenance.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { resolveProvenanceSignal } from "./provenance"; + +describe("resolveProvenanceSignal", () => { + it("flags a predicted-only property as estimated, even when rebaselined", () => { + expect( + resolveProvenanceSignal({ + isNewApproach: true, + hasLodgedEpc: false, + hasPredictedEpc: true, + rebaselineReason: "pre_sap10", + }), + ).toBe("estimated"); + }); + + it("flags a lodged, rebaselined property as remodelled", () => { + expect( + resolveProvenanceSignal({ + isNewApproach: true, + hasLodgedEpc: true, + hasPredictedEpc: false, + rebaselineReason: "pre_sap10", + }), + ).toBe("remodelled"); + }); + + it("returns none for a lodged property that was not rebaselined", () => { + expect( + resolveProvenanceSignal({ + isNewApproach: true, + hasLodgedEpc: true, + hasPredictedEpc: false, + rebaselineReason: "none", + }), + ).toBe("none"); + }); + + it("prefers the real certificate when both lodged and predicted EPCs exist", () => { + expect( + resolveProvenanceSignal({ + isNewApproach: true, + hasLodgedEpc: true, + hasPredictedEpc: true, + rebaselineReason: "pre_sap10", + }), + ).toBe("remodelled"); + }); + + it("returns none for legacy properties regardless of EPC inputs", () => { + expect( + resolveProvenanceSignal({ + isNewApproach: false, + hasLodgedEpc: false, + hasPredictedEpc: true, + rebaselineReason: "pre_sap10", + }), + ).toBe("none"); + }); +}); diff --git a/src/lib/services/provenance.ts b/src/lib/services/provenance.ts new file mode 100644 index 00000000..b16e49a6 --- /dev/null +++ b/src/lib/services/provenance.ts @@ -0,0 +1,36 @@ +/** + * Provenance signal — what the user is told about an EPC's trustworthiness. + * See CONTEXT.md ("EPC provenance", "Provenance signal") and ADR-0002. + * + * Pure decision over a property's EPC provenance (`epc_property.source`) and its + * Rebaseline (`property_baseline_performance.rebaseline_reason`). The DB-reading + * resolver gathers the inputs; this function owns the precedence. + */ +export type ProvenanceSignal = "estimated" | "remodelled" | "none"; + +export function resolveProvenanceSignal(input: { + isNewApproach: boolean; + hasLodgedEpc: boolean; + hasPredictedEpc: boolean; + rebaselineReason: string | null | undefined; +}): ProvenanceSignal { + // Legacy (pre-cutoff) properties have no effective/lodged split and no + // provenance concept — they read the property row directly. + if (!input.isNewApproach) { + return "none"; + } + // No EPC certificate exists — the picture was estimated from nearby homes. + // Dominant: wins even when the baseline was also rebaselined. + if (input.hasPredictedEpc && !input.hasLodgedEpc) { + return "estimated"; + } + // A real certificate exists but the modelling baseline diverged from it. + if ( + input.hasLodgedEpc && + input.rebaselineReason != null && + input.rebaselineReason !== "none" + ) { + return "remodelled"; + } + return "none"; +} From 7bb2b093e5730be11b4da1349b4338e37f554a97 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 24 Jun 2026 18:51:54 +0000 Subject: [PATCH 20/27] perf(portfolio): only join EPC graph / plan LATERAL in count when a filter needs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getPropertiesCount returns the portfolio's total property count for pagination, but it dragged the whole read model through the COUNT: the property_details_epc + property_baseline_performance + two epc_property joins, plus a correlated default-plan LATERAL that ran once per property (31k+ plan lookups for a large portfolio). None of those joins change a COUNT (none multiply rows), so for an unfiltered load they were pure cost — pushing the query to ~14.7s, past Vercel's 15s limit (intermittent timeout on /api/properties). Join only what an active filter references: no filters -> plain COUNT over property; EPC/provenance filter -> add the epc-graph joins; Expected-EPC filter -> add the plan LATERAL. Unfiltered count 14,667ms -> 93ms (portfolio 796); provenance-filtered 156ms. getProperties is unchanged (its LIMIT 1000 already bounds the LATERALs). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/portfolio/[slug]/utils.ts | 52 ++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/src/app/portfolio/[slug]/utils.ts b/src/app/portfolio/[slug]/utils.ts index b429f6c8..6014cc25 100644 --- a/src/app/portfolio/[slug]/utils.ts +++ b/src/app/portfolio/[slug]/utils.ts @@ -34,6 +34,7 @@ import { } from "@/lib/services/epcSources"; import { FilterGroups, + FilterField, PropertyFilter, PROPERTY_TYPE_OPTIONS, BUILT_FORM_OPTIONS, @@ -681,25 +682,62 @@ function buildWhereClause(filterGroups: FilterGroups): ReturnType { : sql``; } +// Filter fields whose SQL references the EPC graph (epc/bp/epl/epp) vs the +// default-plan LATERAL. The count query only needs a join when an active filter +// references it — otherwise joining is pure cost. The `pl` LATERAL in particular +// runs once per property (31k+ correlated plan lookups for a large portfolio), +// which alone pushed the unfiltered count past Vercel's 15s limit. +const EPC_JOIN_FILTER_FIELDS = new Set([ + "currentEpc", + "lodgedEpc", + "provenance", + "co2Emissions", + "floorArea", + "epcExpiryDate", + "mainfuel", +]); +const PLAN_JOIN_FILTER_FIELDS = new Set(["expectedEpc"]); + +function filterFieldsInUse(filterGroups: FilterGroups): Set { + const fields = new Set(); + for (const group of filterGroups) { + for (const cond of group.conditions) fields.add(cond.field); + } + return fields; +} + export async function getPropertiesCount( portfolioId: string, filterGroups: FilterGroups = [] ): Promise { const combinedWhere = buildWhereClause(filterGroups); + const fields = filterFieldsInUse(filterGroups); - const result = await db.execute<{ count: string }>(sql` - SELECT COUNT(DISTINCT p.id)::int AS count - FROM property p - LEFT JOIN property_details_epc epc ON epc.property_id = p.id - ${newApproachJoins} - LEFT JOIN LATERAL ( + // Only join what an active filter needs. COUNT is unaffected by the LEFT JOINs + // otherwise (none multiply rows), so omitting them is purely a speed-up. + const needsEpcJoins = [...fields].some((f) => EPC_JOIN_FILTER_FIELDS.has(f)); + const needsPlanJoin = [...fields].some((f) => PLAN_JOIN_FILTER_FIELDS.has(f)); + + const epcJoins = needsEpcJoins + ? sql`LEFT JOIN property_details_epc epc ON epc.property_id = p.id + ${newApproachJoins}` + : sql``; + const planJoin = needsPlanJoin + ? sql`LEFT JOIN LATERAL ( SELECT id, post_sap_points FROM plan WHERE property_id = p.id AND portfolio_id = p.portfolio_id AND is_default = true ORDER BY created_at DESC LIMIT 1 - ) pl ON true + ) pl ON true` + : sql``; + + const result = await db.execute<{ count: string }>(sql` + SELECT COUNT(DISTINCT p.id)::int AS count + FROM property p + ${epcJoins} + ${planJoin} WHERE p.portfolio_id = ${portfolioId} ${combinedWhere} `); From 7de48448c00c2c20b683e7ebfe4d3c877ce9eeec Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 25 Jun 2026 07:44:22 +0000 Subject: [PATCH 21/27] fix(reporting): temp guards for sub-baseline scenario post_sap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The modelling writes a target-level post_sap (e.g. ~C) even for homes already above the scenario target, so plans can carry cost while modelling post_sap BELOW the effective baseline. That skewed three reporting surfaces. Three TEMP (demo) guards, all keyed on the effective baseline (ADR-0002); revert once the Model team fixes the sub-baseline plans: 1. EPC band chart: post-scenario SAP clamped to GREATEST(baseline, post) so already-compliant properties aren't shown "improving" down a band (portfolio 796: EPC B 4,244 -> 1,660 wrongly, now 4,479). 2. n_units_upgraded + cost: exclude plans whose post_sap is below the effective baseline (not real upgrades) -- 796: 10,283 -> 9,765, -£1.28M. 3. total_sap_uplift / £-per-SAP: baseline is the effective SAP, not the null-for-new-approach current_sap_points, and counts genuine gains only -- uplift 0 -> 89,724, so £/SAP £0 -> £536. Also fixes the no-plan branch to use the effective baseline instead of the null current_sap_points (Unknown-band leakage). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario/[scenarioId]/metrics/route.ts | 32 ++++++++++++--- .../scenario/default/metrics/route.ts | 40 +++++++++++++++---- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts index 702ebeda..d56943e8 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/metrics/route.ts @@ -3,7 +3,12 @@ import { sql } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { sapToEpc } from "@/app/utils"; import type { PortfolioGoalType } from "@/app/db/schema/portfolio"; -import { newApproachJoins, carbonSql, billsSql } from "@/lib/services/epcSources"; +import { + newApproachJoins, + carbonSql, + billsSql, + effectiveSapSql, +} from "@/lib/services/epcSources"; /* ======================= Types @@ -125,14 +130,18 @@ export async function GET( SUM(post_energy_bill)::float AS total_bills, SUM( CASE + -- TEMP (demo): baseline is the effective SAP, not p.current_sap_points + -- (NULL for new-approach → uplift summed to 0 → £/SAP showed 0). Count + -- genuine gains only (post > baseline); sub-baseline plans excluded. WHEN cost_of_works > 0 - AND post_sap_points IS NOT NULL - THEN post_sap_points - p.current_sap_points + AND post_sap_points > (${effectiveSapSql}) + THEN post_sap_points - (${effectiveSapSql}) ELSE 0 END )::float AS total_sap_uplift FROM latest_plans lp JOIN property p ON p.id = lp.property_id + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id -- Conditional filter: only restrict by original_sap_points when the toggle is on -- AND the scenario has an EPC target. Written as an OR chain so Postgres evaluates -- it as a single WHERE clause — avoiding the need to dynamically build the query @@ -177,8 +186,13 @@ export async function GET( )::float AS total_funding FROM latest_plans lp JOIN property p ON p.id = lp.property_id + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id LEFT JOIN funding_package fp ON fp.plan_id = lp.id WHERE lp.cost_of_works > 0 + -- TEMP (demo): exclude plans whose post SAP is below the effective baseline + -- (target-level post_sap on already-compliant homes) — not real upgrades. + -- COALESCE keeps rows we can't compare. See ADR-0002. + AND COALESCE(lp.post_sap_points >= (${effectiveSapSql}), true) AND ( ${useOriginalBaseline} = false OR ${minSap}::float IS NULL @@ -257,10 +271,18 @@ export async function GET( const epcRows = await db.execute(sql` SELECT CASE - WHEN lp.id IS NOT NULL THEN lp.post_sap_points - ELSE p.current_sap_points + -- A retrofit scenario can't make a property worse. The engine writes a + -- target-level post_sap (e.g. ~C) even for properties already above the + -- target, so post_sap can sit BELOW the baseline — which made the chart + -- show e.g. B properties "improving" down to C. Clamp to the baseline. + WHEN lp.id IS NOT NULL THEN GREATEST(${effectiveSapSql}, lp.post_sap_points) + -- No qualifying plan → unchanged property. Use the effective + -- (re-baselined) baseline to match the "before" distribution; NOT + -- p.current_sap_points (NULL for new-approach → "Unknown"). See ADR-0002. + ELSE ${effectiveSapSql} END AS effective_sap FROM property p + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id LEFT JOIN LATERAL ( SELECT * FROM plan diff --git a/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts index a1ed7257..b5f56284 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/default/metrics/route.ts @@ -3,7 +3,12 @@ import { sql } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { sapToEpc } from "@/app/utils"; import type { PortfolioGoalType } from "@/app/db/schema/portfolio"; -import { newApproachJoins, carbonSql, billsSql } from "@/lib/services/epcSources"; +import { + newApproachJoins, + carbonSql, + billsSql, + effectiveSapSql, +} from "@/lib/services/epcSources"; /* ======================= Types @@ -91,14 +96,19 @@ export async function GET( SUM(post_energy_bill)::float AS total_bills, SUM( CASE + -- TEMP (demo): baseline is the effective SAP, not p.current_sap_points + -- (NULL for new-approach → uplift summed to 0 → £/SAP showed 0). Count + -- genuine gains only (post > baseline); sub-baseline "downgrade" plans + -- are excluded rather than dragging the uplift negative. See ADR-0002. WHEN cost_of_works > 0 - AND post_sap_points IS NOT NULL - THEN post_sap_points - p.current_sap_points + AND post_sap_points > (${effectiveSapSql}) + THEN post_sap_points - (${effectiveSapSql}) ELSE 0 END )::float AS total_sap_uplift FROM latest_plans lp - JOIN property p ON p.id = lp.property_id; + JOIN property p ON p.id = lp.property_id + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id; `); const scenarioAgg = scenarioMetricsResult.rows[0] as ScenarioAggregates; @@ -124,8 +134,15 @@ export async function GET( COALESCE(fp.total_uplift, 0) )::float AS total_funding FROM latest_plans lp + JOIN property p ON p.id = lp.property_id + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id LEFT JOIN funding_package fp ON fp.plan_id = lp.id - WHERE lp.cost_of_works > 0; + WHERE lp.cost_of_works > 0 + -- TEMP (demo): a plan whose post SAP is below the effective baseline isn't + -- a real upgrade (target-level post_sap on already-compliant homes), so + -- exclude it from the count + cost. COALESCE keeps rows we can't compare + -- (NULL post or NULL baseline). See ADR-0002. + AND COALESCE(lp.post_sap_points >= (${effectiveSapSql}), true); `); const upgraded = upgradedResult.rows[0] as UpgradedAggregates; @@ -187,10 +204,19 @@ export async function GET( const epcRows = await db.execute(sql` SELECT CASE - WHEN lp.id IS NOT NULL THEN lp.post_sap_points - ELSE p.current_sap_points + -- A retrofit scenario can't make a property worse. The engine writes a + -- target-level post_sap (e.g. ~C) even for properties already above the + -- target, so post_sap can sit BELOW the baseline — which made the chart + -- show e.g. B properties "improving" down to C. Clamp to the baseline so + -- the post-scenario band is never worse than before. + WHEN lp.id IS NOT NULL THEN GREATEST(${effectiveSapSql}, lp.post_sap_points) + -- No plan → unchanged property. Use the effective (re-baselined) baseline + -- so this matches the "before" distribution — NOT p.current_sap_points, + -- which is NULL for new-approach properties. See ADR-0002. + ELSE ${effectiveSapSql} END AS effective_sap FROM property p + LEFT JOIN property_baseline_performance bp ON bp.property_id = p.id LEFT JOIN LATERAL ( SELECT * FROM plan From eef9fdc17120295f4c66bf541e800e1c65fbc4da Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 26 Jun 2026 11:25:45 +0000 Subject: [PATCH 22/27] fix(reporting): show scenario bars for EPC bands absent from the baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The breakdown chart looped only over baseline bands (epcBands), which the query returns only for bands that have properties. So a band reached ONLY after the scenario — e.g. A/B when no property starts that high — had no row, and its scenario bar never rendered (portfolio 812: scenario yields 7 A + 2 B, baseline has none). Iterate a canonical A–G+Unknown order over baseline ∪ scenario instead, so post-scenario-only bands still appear. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../(portfolio)/reporting/BreakdownChart.tsx | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/app/portfolio/[slug]/(portfolio)/reporting/BreakdownChart.tsx b/src/app/portfolio/[slug]/(portfolio)/reporting/BreakdownChart.tsx index 9b660ed0..976ecdec 100644 --- a/src/app/portfolio/[slug]/(portfolio)/reporting/BreakdownChart.tsx +++ b/src/app/portfolio/[slug]/(portfolio)/reporting/BreakdownChart.tsx @@ -51,15 +51,27 @@ export function BreakdownChart({ const rows: any[] = []; - for (const d of epcBands) { - const epc = d.epc ?? "Unknown"; + // The baseline query only returns bands that have properties, so a band + // reached ONLY after the scenario (e.g. A/B when nothing starts that high) + // is absent from epcBands — looping it alone drops those scenario bars. + // Iterate a canonical order over baseline ∪ scenario instead. + const baselineByBand = new Map( + epcBands.map((d) => [d.epc ?? "Unknown", d]), + ); + const BAND_ORDER = ["A", "B", "C", "D", "E", "F", "G", "Unknown"]; + const presentBands = BAND_ORDER.filter( + (b) => baselineByBand.has(b) || (scenarioEpcBands?.[b] ?? 0) > 0, + ); + + for (const epc of presentBands) { + const d = baselineByBand.get(epc); const scenarioValue = scenarioEpcBands?.[epc] ?? 0; // Baseline (stacked) rows.push({ label: `${epc}`, - [friendlyKeys.actual]: d.actual ?? 0, - [friendlyKeys.estimated]: d.estimated ?? 0, + [friendlyKeys.actual]: d?.actual ?? 0, + [friendlyKeys.estimated]: d?.estimated ?? 0, [friendlyKeys.scenario]: 0, }); From 2f0f937fb6b74adb91c17513a65b778a755db830 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 26 Jun 2026 11:42:19 +0000 Subject: [PATCH 23/27] fix(reporting): read scenario measures via denormalised recommendation.plan_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario measures modal came up empty because both measures routes gated on EXISTS against plan_recommendations — a retired join table with no rows for new-approach plans (25.7M legacy rows, none for e.g. portfolio 812's plans), so the query returned zero measures. Read the denormalised model instead: drive from the latest plan per property (default or scenario) and JOIN recommendation by the indexed property_id, scoped to the plan via recommendation.plan_id. Portfolio 812 default now returns 5 measures (solar_pv 54 homes/£266k, …) where it returned 0. Also removes the stale commented query block that referenced the retired plan_recommendations / recommendation_materials tables. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario/[scenarioId]/measures/route.ts | 89 ++++--------------- .../scenario/default/measures/route.ts | 33 +++---- 2 files changed, 34 insertions(+), 88 deletions(-) diff --git a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/measures/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/measures/route.ts index dccf47d9..21aaed37 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/measures/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/[scenarioId]/measures/route.ts @@ -20,62 +20,11 @@ export async function GET( const pid = BigInt(portfolioId); const sid = BigInt(scenarioId); - // TEMP: Remove batteries as underspecified - // const result = await db.execute(sql` - // WITH latest_plans AS ( - // SELECT DISTINCT ON (property_id) - // * - // FROM plan - // WHERE portfolio_id = ${pid} - // AND scenario_id = ${sid} - // ORDER BY property_id, created_at DESC - // ), - - // recommendation_flags AS ( - // SELECT - // r.id AS recommendation_id, - // r.measure_type AS measure_type, - // r.property_id AS property_id, - // r.estimated_cost AS estimated_cost, - // BOOL_OR(m.includes_battery) AS includes_battery - - // FROM latest_plans lp - // JOIN plan_recommendations pr - // ON pr.plan_id = lp.id - // JOIN recommendation r - // ON r.id = pr.recommendation_id - - // LEFT JOIN recommendation_materials rm - // ON rm.recommendation_id = r.id - // LEFT JOIN material m - // ON m.id = rm.material_id - // AND m.is_active = true - - // WHERE r.default = true - // AND r.already_installed = false - - // GROUP BY - // r.id, - // r.measure_type, - // r.property_id, - // r.estimated_cost - // ) - - // SELECT - // measure_type, - // COALESCE(includes_battery, false) AS includes_battery, - - // COUNT(DISTINCT property_id)::int AS homes_count, - // SUM(estimated_cost)::float AS total_cost, - // AVG(estimated_cost)::float AS average_cost - - // FROM recommendation_flags - // GROUP BY - // measure_type, - // includes_battery - // ORDER BY total_cost DESC; - // `); - + // Latest plan per property for this scenario, then its recommendations read + // through the DENORMALISED link: join by the indexed recommendation.property_id + // and scope to the plan via recommendation.plan_id. The plan_recommendations + // join table is retired (no rows for new-approach plans), so the old EXISTS + // against it returned zero measures. See the handover / ADR notes. const result = await db.execute(sql` SELECT r.measure_type, @@ -83,23 +32,19 @@ export async function GET( COUNT(DISTINCT r.property_id)::int AS homes_count, SUM(r.estimated_cost)::float AS total_cost, AVG(r.estimated_cost)::float AS average_cost - FROM recommendation r - WHERE r.default = true + FROM ( + SELECT DISTINCT ON (property_id) + id, property_id + FROM plan + WHERE portfolio_id = ${pid} + AND scenario_id = ${sid} + ORDER BY property_id, created_at DESC + ) lp + JOIN recommendation r + ON r.property_id = lp.property_id + AND r.plan_id = lp.id + AND r.default = true AND r.already_installed = false - AND EXISTS ( - SELECT 1 - FROM ( - SELECT DISTINCT ON (p.property_id) - p.id - FROM plan p - WHERE p.portfolio_id = ${pid} - AND p.scenario_id = ${sid} - ORDER BY p.property_id, p.created_at DESC - ) lp - JOIN plan_recommendations pr - ON pr.plan_id = lp.id - WHERE pr.recommendation_id = r.id - ) GROUP BY r.measure_type, r.type diff --git a/src/app/api/portfolio/[portfolioId]/scenario/default/measures/route.ts b/src/app/api/portfolio/[portfolioId]/scenario/default/measures/route.ts index 1e3d2bc1..3640c773 100644 --- a/src/app/api/portfolio/[portfolioId]/scenario/default/measures/route.ts +++ b/src/app/api/portfolio/[portfolioId]/scenario/default/measures/route.ts @@ -19,6 +19,11 @@ export async function GET( const pid = BigInt(portfolioId); + // Latest default plan per property, then its recommendations read through the + // DENORMALISED link: join by the indexed recommendation.property_id and scope + // to the plan via recommendation.plan_id. The plan_recommendations join table + // is retired (no rows for new-approach plans), so the old EXISTS against it + // returned zero measures. const result = await db.execute(sql` SELECT r.measure_type, @@ -26,23 +31,19 @@ export async function GET( COUNT(DISTINCT r.property_id)::int AS homes_count, SUM(r.estimated_cost)::float AS total_cost, AVG(r.estimated_cost)::float AS average_cost - FROM recommendation r - WHERE r.default = true + FROM ( + SELECT DISTINCT ON (property_id) + id, property_id + FROM plan + WHERE portfolio_id = ${pid} + AND is_default = true + ORDER BY property_id, created_at DESC + ) lp + JOIN recommendation r + ON r.property_id = lp.property_id + AND r.plan_id = lp.id + AND r.default = true AND r.already_installed = false - AND EXISTS ( - SELECT 1 - FROM ( - SELECT DISTINCT ON (p.property_id) - p.id - FROM plan p - WHERE p.portfolio_id = ${pid} - AND p.is_default = true - ORDER BY p.property_id, p.created_at DESC - ) lp - JOIN plan_recommendations pr - ON pr.plan_id = lp.id - WHERE pr.recommendation_id = r.id - ) GROUP BY r.measure_type, r.type From 4941e756831f3bac8968cb1b77224f51f01cb9ea Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 1 Jul 2026 15:56:16 +0000 Subject: [PATCH 24/27] fix(epc): follow energyMainsGas rename in new-approach resolver The merge of main renamed epc_property.energyMainsGas -> energyGasConnectionAvailable (migration 0252), but the new-approach resolver read the old property name, which main never touched so it survived the merge with no conflict and broke the build. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/services/epcSources.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index d1a77dc0..d644a1a9 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -355,7 +355,7 @@ async function resolveNewConditionReport( numberHeatedRooms: chosen.heatedRoomsCount ?? null, numberOpenFireplaces: chosen.openChimneysCount ?? null, numberExtensions: chosen.extensionsCount ?? null, - mainsGas: chosen.energyMainsGas ?? null, + mainsGas: chosen.energyGasConnectionAvailable ?? null, // GAP: no energy tariff in the new graph — omitted. energyTariff: null, currentEnergyDemand: sumKwh, From 1dfb1e15340542aec807bd9f24b6f482f02d4ca2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 2 Jul 2026 09:56:59 +0000 Subject: [PATCH 25/27] fix(building-passport): portal tooltip content so it isn't trapped by stacking context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadcn TooltipContent rendered inline, missing the Portal that upstream shadcn ships. On the Current Efficiency card the info tooltip lives inside a `relative z-10` div while the Lodged EPC badge is a sibling at `z-20`, so the popup's own `z-50` only competed within the z-10 subtree and the badge painted over it (also latently clipped by the card's `overflow-hidden`). Wrapping TooltipContent in TooltipPrimitive.Portal renders it at , escaping both the stacking context and the overflow clip — fixes all 9 tooltips. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/shadcn_components/ui/tooltip.tsx | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/app/shadcn_components/ui/tooltip.tsx b/src/app/shadcn_components/ui/tooltip.tsx index 30fc44d9..25576f54 100644 --- a/src/app/shadcn_components/ui/tooltip.tsx +++ b/src/app/shadcn_components/ui/tooltip.tsx @@ -15,15 +15,17 @@ const TooltipContent = React.forwardRef< React.ElementRef, React.ComponentPropsWithoutRef >(({ className, sideOffset = 4, ...props }, ref) => ( - + + + )) TooltipContent.displayName = TooltipPrimitive.Content.displayName From 9bf36ad4b6e04c04af3905a0fc7f103250ab38b7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 2 Jul 2026 10:00:46 +0000 Subject: [PATCH 26/27] perf(backfill): faster, resumable recommendation denormalization + ops tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overhaul the recommendation plan_id/material_* backfill for the 21M-row prod table (see docs/runbooks/recommendation-denormalization-backfill.md): - Single combined pass: set plan_id + all four material_* columns in one write per row (was two full passes, rewriting every row twice). Guard + COALESCE keep it idempotent and resumable — done rows are never rewritten. - Session tuning: synchronous_commit=off, tunable work_mem / maintenance_work_mem. - Range + phase controls (BACKFILL_START_ID/END_ID/SKIP_CARDINALITY/ONLY_FINALIZE) to run disjoint id-range workers in parallel when the disk isn't the bottleneck. - Percent-complete + ETA in the progress log. - Finalize builds the two new indexes plain by default (faster in a quiet window) with a CONCURRENTLY toggle. The real speed-up in prod was dropping the 3 secondary indexes for the backfill then rebuilding — the table is packed (fillfactor 100) so updates can't go HOT and otherwise maintain every index per row. Index drop/rebuild + vacuum are kept as standalone SQL so ops controls exactly when the app loses/regains them: - sql/drop-recommendation-backfill-indexes.sql - sql/rebuild-recommendation-backfill-indexes.sql (1GB maintenance_work_mem — safe on the 4GB instance) - sql/vacuum-recommendation.sql run-sql.ts runs .sql files statement-by-statement, autocommitted (so VACUUM / CREATE INDEX CONCURRENTLY work), for environments without the psql CLI. Wired up as npm run db:run-sql and backfill:recommendation-restore. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...recommendation-denormalization-backfill.md | 140 +++++++++ package.json | 5 +- ...backfill-recommendation-denormalization.ts | 291 +++++++++++------- src/app/db/run-sql.ts | 68 ++++ .../drop-recommendation-backfill-indexes.sql | 26 ++ ...ebuild-recommendation-backfill-indexes.sql | 31 ++ src/app/db/sql/vacuum-recommendation.sql | 7 + 7 files changed, 461 insertions(+), 107 deletions(-) create mode 100644 docs/runbooks/recommendation-denormalization-backfill.md create mode 100644 src/app/db/run-sql.ts create mode 100644 src/app/db/sql/drop-recommendation-backfill-indexes.sql create mode 100644 src/app/db/sql/rebuild-recommendation-backfill-indexes.sql create mode 100644 src/app/db/sql/vacuum-recommendation.sql diff --git a/docs/runbooks/recommendation-denormalization-backfill.md b/docs/runbooks/recommendation-denormalization-backfill.md new file mode 100644 index 00000000..916ff601 --- /dev/null +++ b/docs/runbooks/recommendation-denormalization-backfill.md @@ -0,0 +1,140 @@ +# Runbook: `recommendation` denormalisation backfill + +Populates `recommendation.plan_id` and `recommendation.material_*` from the +`plan_recommendations` / `recommendation_materials` join tables, as the first +step toward making those columns the source of truth and dropping the join +tables. See [ADR 0001](../adr/0001-data-backfills-outside-drizzle.md) for why +this runs outside drizzle. + +The table is ~21m rows. This runbook is the **fast** procedure. + +## Why it's slow by default, and the lever + +`recommendation` is packed (fillfactor 100), so an UPDATE can't do a HOT update — +Postgres writes a new row version and inserts into **every index on the table** +for all ~21m rows. That index write-amplification (and its WAL) is what drains IO. + +The lever: **drop the secondary indexes, backfill, rebuild them.** With them gone, +each updated row only maintains the primary key (~4 index inserts → 1). Combined +with the single-pass rewrite (plan_id + material in one UPDATE, not two) and +`synchronous_commit = off`, this turns a ~1-day run into a few hours. + +`DROP INDEX` is near-instant — the cost people mean is the *rebuild*, which is why +we do it once, at the end, in a low-usage window. + +## What the app needs to keep working + +Only the 3 secondary indexes. The half-filled `plan_id`/`material_id` columns are +NULL-tolerant (the read paths already handle NULLs), and the *new* +plan_id/material_id indexes aren't used by anything yet. So **at any stopping +point, rebuild the 3 secondary indexes and the app is back to today's behaviour** +— a 50%-done backfill is invisible to it. + +## Prerequisites + +- `npm run migration:migrate` has applied 0222/0224 (the column + `NOT VALID` FK adds). +- `.env.local` points at the target DB (`DB_HOST`/`DB_PORT`/`DB_USERNAME`/`DB_PASSWORD`/`DB_NAME`). +- A low-usage window. + +## Files + +- Script: `src/app/db/backfill-recommendation-denormalization.ts` (`npm run backfill:recommendation-denormalization`) +- Drop: `src/app/db/sql/drop-recommendation-backfill-indexes.sql` +- Rebuild:`src/app/db/sql/rebuild-recommendation-backfill-indexes.sql` + +--- + +## Procedure A — one window (a few hours, start to finish) + +```bash +# 1. Drop the secondary indexes (app reads degrade to seq scans until step 4) +psql "$DATABASE_URL" -f src/app/db/sql/drop-recommendation-backfill-indexes.sql + +# 2. Run the full backfill + finalize (builds the 2 new indexes, validates FKs) +npm run backfill:recommendation-denormalization + +# 3. Rebuild the secondary indexes — app back to normal +psql "$DATABASE_URL" -f src/app/db/sql/rebuild-recommendation-backfill-indexes.sql + +# 4. Reclaim the ~2x bloat in-place updates leave behind, refresh stats +psql "$DATABASE_URL" -c "VACUUM (ANALYZE) recommendation;" +``` + +`DATABASE_URL` is illustrative — use whatever psql connection matches `.env.local`. + +--- + +## Procedure B — split across windows (50% now, finish later) + +Each window is self-contained; between windows the app is 100% normal. + +**Every window except the last:** + +```bash +psql "$DATABASE_URL" -f src/app/db/sql/drop-recommendation-backfill-indexes.sql +BACKFILL_FINALIZE=false npm run backfill:recommendation-denormalization # Ctrl-C any time; it's resumable +psql "$DATABASE_URL" -f src/app/db/sql/rebuild-recommendation-backfill-indexes.sql +``` + +`BACKFILL_FINALIZE=false` skips the new-index build + FK validate — there's no +point building the plan_id/material_id indexes until the data is all in. + +**The final window** (when the per-batch logs / the NULL summary show plan_id is +down to ~0 and only genuinely material-less rows remain): + +```bash +psql "$DATABASE_URL" -f src/app/db/sql/drop-recommendation-backfill-indexes.sql +npm run backfill:recommendation-denormalization # FINALIZE defaults on: builds new indexes, validates FKs +psql "$DATABASE_URL" -f src/app/db/sql/rebuild-recommendation-backfill-indexes.sql +psql "$DATABASE_URL" -c "VACUUM (ANALYZE) recommendation;" +``` + +Cost of splitting: you rebuild the secondary indexes once per window. Two of the +three are partial (tiny), so each restore is effectively one full-size index +build (`recommendation_property_id_idx`, a few minutes). + +### Interrupting mid-run + +Ctrl-C is safe. The script commits per batch, so completed batches persist. Re-run +to continue — the `IS NULL` guards skip every row already done. If you're worried +about app performance while stopped, rebuild the secondary indexes first. + +--- + +## Alternative — no index juggling (simpler, slower) + +If you'd rather not touch indexes at all, just run the tuned single pass with the +indexes left in place and stop/resume freely: + +```bash +BACKFILL_FINALIZE=false npm run backfill:recommendation-denormalization +``` + +Slower per row (index write-amplification stays), but the app is always fully +indexed and there's zero drop/rebuild ceremony. Finalize on the last run. + +--- + +## Tunables (env) + +| Var | Default | Notes | +| --- | --- | --- | +| `BACKFILL_BATCH_SIZE` | `100000` | rows scanned per committed batch | +| `BACKFILL_SLEEP_MS` | `0` | inter-batch pause; raise to throttle IO if not in a quiet window | +| `BACKFILL_SYNC_COMMIT` | `off` | `on` to keep synchronous_commit (safe to leave off — run is resumable) | +| `BACKFILL_MAINTENANCE_WORK_MEM` | `1GB` | memory for the finalize index builds | +| `BACKFILL_WORK_MEM` | `128MB` | memory for the batch joins | +| `BACKFILL_INDEX_CONCURRENTLY` | *(off)* | `true` builds the new indexes CONCURRENTLY (non-blocking, slower) | +| `BACKFILL_FINALIZE` | *(on)* | `false` skips new-index build + FK validate (partial windows) | + +## Verifying completion + +The script prints remaining NULLs at the end: + +- `plan_id` NULL → should trend to ~0 (investigate any residue: recommendations with no `plan_recommendations` row). +- `material_id` NULL → **expected** for recommendations that have no material; not an error. + +## After the backfill is fully done + +A later migration can `SET NOT NULL` on `plan_id` and drop the join tables, gated +on this backfill having completed with zero unexpected NULLs. Out of scope here. diff --git a/package.json b/package.json index e205a7e5..92fda474 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,10 @@ "migration:generate": "drizzle-kit generate", "migration:migrate": "drizzle-kit migrate", "create_user": "tsx src/app/db/create_user.ts", - "backfill:recommendation-denormalization": "tsx src/app/db/backfill-recommendation-denormalization.ts" + "backfill:recommendation-denormalization": "tsx src/app/db/backfill-recommendation-denormalization.ts", + "backfill:recommendation-denormalization:pass": "BACKFILL_FINALIZE=false tsx src/app/db/backfill-recommendation-denormalization.ts", + "db:run-sql": "tsx src/app/db/run-sql.ts", + "backfill:recommendation-restore": "tsx src/app/db/run-sql.ts src/app/db/sql/rebuild-recommendation-backfill-indexes.sql src/app/db/sql/vacuum-recommendation.sql" }, "dependencies": { "@aws-sdk/client-s3": "^3.971.0", diff --git a/src/app/db/backfill-recommendation-denormalization.ts b/src/app/db/backfill-recommendation-denormalization.ts index ce8b74d2..b0854571 100644 --- a/src/app/db/backfill-recommendation-denormalization.ts +++ b/src/app/db/backfill-recommendation-denormalization.ts @@ -1,7 +1,8 @@ /** * Backfills the denormalised plan_id / material_* columns on `recommendation` - * from the plan_recommendations and recommendation_materials join tables, then - * builds the supporting indexes and validates the foreign keys — all ONLINE. + * from the plan_recommendations and recommendation_materials join tables in a + * SINGLE keyset-paginated pass, then (optionally) builds the supporting indexes + * and validates the foreign keys — all ONLINE and resumable. * * Why this lives outside drizzle: * drizzle-kit migrate wraps every pending migration in ONE transaction, so it @@ -14,21 +15,55 @@ * Run AFTER `npm run migration:migrate` has applied 0222/0224 (the column adds): * npm run backfill:recommendation-denormalization * + * ── Making it fast (see docs/runbooks/recommendation-denormalization-backfill.md) + * The dominant cost is index write-amplification: `recommendation` is packed + * (fillfactor 100), so each UPDATE can't do a HOT update and must insert into + * every index on the table for every one of ~21m rows. To go roughly an order + * of magnitude faster in a low-usage window: + * 1. DROP the 3 secondary indexes (src/app/db/sql/drop-recommendation-backfill-indexes.sql) + * 2. run this script (BACKFILL_FINALIZE=false for a partial/interrupted window) + * 3. REBUILD them after (src/app/db/sql/rebuild-recommendation-backfill-indexes.sql) + * The backfill is resumable: stop any time, rebuild the indexes, and the app is + * back to today's behaviour (the half-filled columns are NULL-tolerant). Re-run + * to continue — the IS NULL guards never rewrite a row that's already done. + * + * This pass sets plan_id AND the four material_* columns in ONE write per row + * (the previous version did two full passes, rewriting every row twice). + * * Safe to re-run: every step is idempotent (IS NULL guards, IF NOT EXISTS, - * VALIDATE on an already-valid constraint is a no-op). If interrupted, just run - * it again — it resumes from wherever it left off. + * VALIDATE on an already-valid constraint is a no-op). * * Tunables via env: - * BACKFILL_BATCH_SIZE rows scanned per committed batch (default 25000) - * BACKFILL_SLEEP_MS pause between batches, eases IO pressure (default 50) + * BACKFILL_BATCH_SIZE rows scanned per committed batch (default 100000) + * BACKFILL_SLEEP_MS pause between batches, eases IO pressure (default 0) + * BACKFILL_SYNC_COMMIT 'on' keeps synchronous_commit; default 'off' (faster + * batch commits — safe here because the run is resumable) + * BACKFILL_MAINTENANCE_WORK_MEM session maintenance_work_mem for index builds (default '1GB') + * BACKFILL_WORK_MEM session work_mem for the batch joins (default '128MB') + * BACKFILL_INDEX_CONCURRENTLY 'true' builds the new indexes CONCURRENTLY (default plain + * CREATE INDEX — faster, but briefly blocks writes) + * BACKFILL_FINALIZE 'false' skips the index build + FK validate; use this for + * partial/interrupted windows, finalize on the last one */ import dotenv from "dotenv"; import { Pool, type PoolClient } from "pg"; dotenv.config({ path: ".env.local" }); -const BATCH_SIZE = Number(process.env.BACKFILL_BATCH_SIZE ?? 25_000); -const SLEEP_MS = Number(process.env.BACKFILL_SLEEP_MS ?? 50); +const BATCH_SIZE = Number(process.env.BACKFILL_BATCH_SIZE ?? 100_000); +const SLEEP_MS = Number(process.env.BACKFILL_SLEEP_MS ?? 0); +const SYNC_COMMIT = process.env.BACKFILL_SYNC_COMMIT === "on" ? "on" : "off"; +const MAINTENANCE_WORK_MEM = process.env.BACKFILL_MAINTENANCE_WORK_MEM ?? "1GB"; +const WORK_MEM = process.env.BACKFILL_WORK_MEM ?? "128MB"; +const INDEX_CONCURRENTLY = process.env.BACKFILL_INDEX_CONCURRENTLY === "true"; +const FINALIZE = process.env.BACKFILL_FINALIZE !== "false"; + +// Range + phase controls, for running several workers over disjoint id ranges in +// parallel (near-linear speed-up — the ranges never contend for the same rows). +const START_ID = process.env.BACKFILL_START_ID ?? "0"; +const END_ID = process.env.BACKFILL_END_ID ?? null; // inclusive upper bound; null = unbounded +const SKIP_CARDINALITY = process.env.BACKFILL_SKIP_CARDINALITY === "true"; +const ONLY_FINALIZE = process.env.BACKFILL_ONLY_FINALIZE === "true"; const pool = new Pool({ host: process.env.DB_HOST, @@ -65,28 +100,79 @@ async function assertSingleCardinality( /** * Keyset-paginate the whole `recommendation` table by id and, for each batch, - * set the target columns from the join table where a match exists. Each batch - * is its own autocommitted statement (no surrounding BEGIN), so locks are held - * only for the rows in that batch. + * set plan_id and the four material_* columns from the join tables in a single + * UPDATE. Each batch is its own autocommitted statement (no surrounding BEGIN), + * so locks are held only for the rows in that batch. * - * We scan by id (not `WHERE col IS NULL LIMIT n`) so rows with no match don't - * get re-selected forever — they're simply left NULL and we move past them. + * We scan by id (not `WHERE col IS NULL LIMIT n`) so rows with no match — or a + * legitimately NULL material_id — don't get re-selected forever; they're left as + * they are and we move past them. + * + * The guard only touches a row when there is real work to do (a NULL column that + * has a matching source row), and COALESCE never overwrites an already-set value. + * So a row is written at most once, and a re-run after interruption skips every + * row already done — the pass is idempotent and resumable. + * + * The single-cardinality assertion above guarantees the LEFT JOINs cannot fan a + * row out into multiple update targets. */ -async function backfillColumns( - client: PoolClient, - label: string, - updateSql: (idFrom: string, limit: string) => string, -): Promise { - let lastId = "0"; +const fmtDuration = (secs: number): string => { + const s = Math.max(0, Math.round(secs)); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + return h > 0 ? `${h}h${m}m` : m > 0 ? `${m}m${s % 60}s` : `${s}s`; +}; + +async function backfillDenormalized(client: PoolClient): Promise { + let lastId = START_ID; let scannedTotal = 0; let updatedTotal = 0; const startedAt = Date.now(); + // Upper bound of the id keyset, for percent-complete + ETA. When END_ID is set + // (a parallel worker), use it; otherwise ask the table (fast, PK). + let maxId: number; + if (END_ID != null) { + maxId = Number(END_ID); + } else { + const { rows: maxRows } = await client.query( + `SELECT max(id) AS max_id FROM recommendation`, + ); + maxId = Number(maxRows[0].max_id ?? 0); + } + console.log( + `Backfilling id ${Number(START_ID).toLocaleString()} → ${maxId.toLocaleString()}...`, + ); + for (;;) { - const { rows } = await client.query(updateSql("$1", "$2"), [ - lastId, - BATCH_SIZE, - ]); + const { rows } = await client.query( + `WITH batch AS ( + SELECT id FROM recommendation + WHERE id > $1 + AND ($3::bigint IS NULL OR id <= $3::bigint) + ORDER BY id + LIMIT $2 + ), + upd AS ( + UPDATE recommendation r + SET plan_id = COALESCE(r.plan_id, pr.plan_id), + material_id = COALESCE(r.material_id, rm.material_id), + material_quantity = COALESCE(r.material_quantity, rm.quantity), + material_quantity_unit = COALESCE(r.material_quantity_unit, rm.quantity_unit), + material_depth = COALESCE(r.material_depth, rm.depth) + FROM batch b + LEFT JOIN plan_recommendations pr ON pr.recommendation_id = b.id + LEFT JOIN recommendation_materials rm ON rm.recommendation_id = b.id + WHERE r.id = b.id + AND ( (r.plan_id IS NULL AND pr.plan_id IS NOT NULL) + OR (r.material_id IS NULL AND rm.material_id IS NOT NULL) ) + RETURNING r.id + ) + SELECT (SELECT max(id) FROM batch) AS max_id, + (SELECT count(*) FROM batch) AS scanned, + (SELECT count(*) FROM upd) AS updated`, + [lastId, BATCH_SIZE, END_ID], + ); const { max_id, scanned, updated } = rows[0] as { max_id: string | null; scanned: number; @@ -99,17 +185,22 @@ async function backfillColumns( updatedTotal += Number(updated); lastId = max_id; - const rate = Math.round(scannedTotal / ((Date.now() - startedAt) / 1000)); + const elapsed = (Date.now() - startedAt) / 1000; + const rate = Math.round(scannedTotal / elapsed); + const from = Number(START_ID); + const span = maxId - from; + const pct = span > 0 ? Math.min(100, ((Number(lastId) - from) / span) * 100) : 0; + const eta = pct > 0 ? fmtDuration((elapsed / pct) * (100 - pct)) : "?"; console.log( - `[${label}] up to id ${lastId} — scanned ${scannedTotal.toLocaleString()}, ` + - `updated ${updatedTotal.toLocaleString()} (${rate.toLocaleString()} rows/s scan)`, + `${pct.toFixed(1)}% (id ${lastId}) — scanned ${scannedTotal.toLocaleString()}, ` + + `updated ${updatedTotal.toLocaleString()} — ${rate.toLocaleString()} rows/s, ETA ${eta}`, ); if (SLEEP_MS > 0) await sleep(SLEEP_MS); } console.log( - `[${label}] done: scanned ${scannedTotal.toLocaleString()}, ` + + `Backfill pass done: scanned ${scannedTotal.toLocaleString()}, ` + `updated ${updatedTotal.toLocaleString()} in ` + `${Math.round((Date.now() - startedAt) / 1000)}s`, ); @@ -119,91 +210,77 @@ async function main(): Promise { const client = await pool.connect(); try { console.log( - `Config: batch=${BATCH_SIZE.toLocaleString()} rows, sleep=${SLEEP_MS}ms\n`, + `Config: batch=${BATCH_SIZE.toLocaleString()} rows, sleep=${SLEEP_MS}ms, ` + + `synchronous_commit=${SYNC_COMMIT}, work_mem=${WORK_MEM}, ` + + `maintenance_work_mem=${MAINTENANCE_WORK_MEM}, finalize=${FINALIZE}, ` + + `range=(${Number(START_ID).toLocaleString()}, ${END_ID ?? "∞"}]` + + `${ONLY_FINALIZE ? ", ONLY_FINALIZE" : ""}\n`, ); - // 1. Guard the 1:1 assumption before mutating anything. - console.log("Checking cardinality..."); - await assertSingleCardinality(client, "plan_recommendations"); - await assertSingleCardinality(client, "recommendation_materials"); + // Session tuning. synchronous_commit=off is safe here: the backfill is + // idempotent and resumable, so a crash that loses the last few batch commits + // just means the next run continues from there. work_mem feeds the batch + // joins; maintenance_work_mem feeds the CREATE INDEX in the finalize step. + await client.query(`SET synchronous_commit = ${SYNC_COMMIT}`); + await client.query(`SET work_mem = '${WORK_MEM}'`); + await client.query(`SET maintenance_work_mem = '${MAINTENANCE_WORK_MEM}'`); - // 2. Backfill plan_id. - await backfillColumns( - client, - "plan_id", - (idFrom, limit) => ` - WITH batch AS ( - SELECT id FROM recommendation - WHERE id > ${idFrom} - ORDER BY id - LIMIT ${limit} - ), - upd AS ( - UPDATE recommendation r - SET plan_id = pr.plan_id - FROM batch b - JOIN plan_recommendations pr ON pr.recommendation_id = b.id - WHERE r.id = b.id - AND r.plan_id IS NULL - RETURNING r.id - ) - SELECT (SELECT max(id) FROM batch) AS max_id, - (SELECT count(*) FROM batch) AS scanned, - (SELECT count(*) FROM upd) AS updated`, - ); + if (ONLY_FINALIZE) { + // Parallel workflow: the data was written by range workers; this run just + // builds the indexes + validates the FKs. No cardinality check, no pass. + console.log("ONLY_FINALIZE: skipping cardinality check + backfill pass."); + } else { + // 1. Guard the 1:1 assumption before mutating anything. Skippable on + // secondary parallel workers (one worker having checked is enough). + if (SKIP_CARDINALITY) { + console.log("Skipping cardinality check (BACKFILL_SKIP_CARDINALITY=true)."); + } else { + console.log("Checking cardinality..."); + await assertSingleCardinality(client, "plan_recommendations"); + await assertSingleCardinality(client, "recommendation_materials"); + } - // 3. Backfill the four material_* columns in one pass. - await backfillColumns( - client, - "material", - (idFrom, limit) => ` - WITH batch AS ( - SELECT id FROM recommendation - WHERE id > ${idFrom} - ORDER BY id - LIMIT ${limit} - ), - upd AS ( - UPDATE recommendation r - SET material_id = rm.material_id, - material_quantity = rm.quantity, - material_quantity_unit = rm.quantity_unit, - material_depth = rm.depth - FROM batch b - JOIN recommendation_materials rm ON rm.recommendation_id = b.id - WHERE r.id = b.id - AND r.material_id IS NULL - RETURNING r.id - ) - SELECT (SELECT max(id) FROM batch) AS max_id, - (SELECT count(*) FROM batch) AS scanned, - (SELECT count(*) FROM upd) AS updated`, - ); + // 2. Single combined pass: plan_id + material_* in one write per row. + console.log("Backfilling plan_id + material_* (single pass)..."); + await backfillDenormalized(client); + } - // 4. Build indexes CONCURRENTLY (no write lock). Must NOT be in a txn — - // a pooled client with no BEGIN runs each statement autocommitted. - console.log("Creating indexes concurrently..."); - await client.query( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_recommendation_plan_id - ON recommendation USING btree (plan_id)`, - ); - await client.query( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_recommendation_material_id - ON recommendation USING btree (material_id)`, - ); + if (FINALIZE || ONLY_FINALIZE) { + // 3. Build the two new indexes. Plain CREATE INDEX is faster in a quiet + // window; BACKFILL_INDEX_CONCURRENTLY=true avoids the brief write lock. + // Autocommitted (no surrounding BEGIN), which CONCURRENTLY requires. + const cc = INDEX_CONCURRENTLY ? "CONCURRENTLY " : ""; + console.log( + `Creating indexes (${INDEX_CONCURRENTLY ? "concurrently" : "plain"})...`, + ); + await client.query( + `CREATE INDEX ${cc}IF NOT EXISTS idx_recommendation_plan_id + ON recommendation USING btree (plan_id)`, + ); + await client.query( + `CREATE INDEX ${cc}IF NOT EXISTS idx_recommendation_material_id + ON recommendation USING btree (material_id)`, + ); - // 5. Validate the FKs online (ShareUpdateExclusiveLock — allows reads/writes). - console.log("Validating foreign keys..."); - await client.query( - `ALTER TABLE recommendation - VALIDATE CONSTRAINT recommendation_plan_id_plan_id_fk`, - ); - await client.query( - `ALTER TABLE recommendation - VALIDATE CONSTRAINT recommendation_material_id_material_id_fk`, - ); + // 4. Validate the FKs online (ShareUpdateExclusiveLock — allows reads/writes). + console.log("Validating foreign keys..."); + await client.query( + `ALTER TABLE recommendation + VALIDATE CONSTRAINT recommendation_plan_id_plan_id_fk`, + ); + await client.query( + `ALTER TABLE recommendation + VALIDATE CONSTRAINT recommendation_material_id_material_id_fk`, + ); + } else { + console.log( + "Skipping index build + FK validate (BACKFILL_FINALIZE=false). " + + "Run once more with finalize enabled in your final window.", + ); + } - // 6. Report any rows still unlinked (expected for material; investigate for plan). + // 5. Report any rows still unlinked. material_id NULL is expected (not every + // recommendation has a material); plan_id NULL should trend toward ~0. const { rows } = await client.query( `SELECT count(*) FILTER (WHERE plan_id IS NULL) AS plan_null, count(*) FILTER (WHERE material_id IS NULL) AS material_null @@ -213,7 +290,9 @@ async function main(): Promise { `\nRemaining NULLs — plan_id: ${Number(rows[0].plan_null).toLocaleString()}, ` + `material_id: ${Number(rows[0].material_null).toLocaleString()}`, ); - console.log("Backfill complete."); + console.log( + FINALIZE ? "Backfill complete." : "Backfill pass complete (not finalized).", + ); } finally { client.release(); await pool.end(); diff --git a/src/app/db/run-sql.ts b/src/app/db/run-sql.ts new file mode 100644 index 00000000..2b491b8c --- /dev/null +++ b/src/app/db/run-sql.ts @@ -0,0 +1,68 @@ +/** + * Runs one or more .sql files against the DB from .env.local, executing each + * statement autocommitted (no wrapping transaction) so statements that cannot run + * inside a transaction block — VACUUM, CREATE INDEX CONCURRENTLY — work. For + * environments without the psql CLI. + * + * tsx src/app/db/run-sql.ts [more.sql ...] + * + * Statement splitting is deliberately simple: it strips full-line `--` comments + * and splits on `;`. That's correct for the hand-written files in + * src/app/db/sql, but it is NOT a general SQL parser — don't point it at files + * with `;` inside string literals or dollar-quoted function bodies. + */ +import { readFileSync } from "fs"; +import dotenv from "dotenv"; +import { Pool } from "pg"; + +dotenv.config({ path: ".env.local" }); + +const files = process.argv.slice(2); +if (files.length === 0) { + console.error("Usage: tsx src/app/db/run-sql.ts [more.sql ...]"); + process.exit(1); +} + +const pool = new Pool({ + host: process.env.DB_HOST, + port: Number(process.env.DB_PORT), + user: process.env.DB_USERNAME, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, +}); + +function statementsOf(path: string): string[] { + const raw = readFileSync(path, "utf8"); + return raw + .split("\n") + .filter((line) => !line.trim().startsWith("--")) + .join("\n") + .split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +async function main(): Promise { + const client = await pool.connect(); + try { + for (const file of files) { + console.log(`\n=== ${file} ===`); + for (const sql of statementsOf(file)) { + const preview = sql.replace(/\s+/g, " ").slice(0, 70); + const startedAt = Date.now(); + console.log(`> ${preview}${sql.length > 70 ? "…" : ""}`); + await client.query(sql); + console.log(` ok (${Math.round((Date.now() - startedAt) / 1000)}s)`); + } + } + console.log("\nAll statements complete."); + } finally { + client.release(); + await pool.end(); + } +} + +main().catch((err) => { + console.error("SQL run failed:", err); + process.exit(1); +}); diff --git a/src/app/db/sql/drop-recommendation-backfill-indexes.sql b/src/app/db/sql/drop-recommendation-backfill-indexes.sql new file mode 100644 index 00000000..2fd28dd1 --- /dev/null +++ b/src/app/db/sql/drop-recommendation-backfill-indexes.sql @@ -0,0 +1,26 @@ +-- Drop the 3 secondary indexes on `recommendation` so the denormalisation +-- backfill can run without per-row index write-amplification. +-- +-- `recommendation` is packed (fillfactor 100), so each backfill UPDATE cannot do +-- a HOT update and must insert a new entry into every index on the table for all +-- ~21m rows. Dropping these first cuts index inserts per updated row from ~4 to 1 +-- (just the primary key), which is the bulk of the speed-up. +-- +-- These serve production reads (fetch-by-property, the active-defaults paths), so +-- ONLY run this in a low-usage window and rebuild them +-- (rebuild-recommendation-backfill-indexes.sql) as soon as the window's backfill +-- pass finishes. DROP INDEX is near-instant (catalog change + file unlink); it +-- takes a brief ACCESS EXCLUSIVE lock, so if long-running queries are in flight +-- use `DROP INDEX CONCURRENTLY` (cannot run inside a transaction) instead. +-- +-- Kept OUT of the backfill script on purpose so you control exactly when the app +-- loses and regains these indexes. See +-- docs/runbooks/recommendation-denormalization-backfill.md +-- +-- NOTE: idx_recommendation_active_id_property was originally built CONCURRENTLY; +-- if a prior CONCURRENTLY build was ever interrupted it can linger as INVALID — +-- the DROP below clears it either way. + +DROP INDEX IF EXISTS recommendation_property_id_idx; +DROP INDEX IF EXISTS idx_recommendation_active_defaults; +DROP INDEX IF EXISTS idx_recommendation_active_id_property; diff --git a/src/app/db/sql/rebuild-recommendation-backfill-indexes.sql b/src/app/db/sql/rebuild-recommendation-backfill-indexes.sql new file mode 100644 index 00000000..37714e3b --- /dev/null +++ b/src/app/db/sql/rebuild-recommendation-backfill-indexes.sql @@ -0,0 +1,31 @@ +-- Rebuild the 3 secondary indexes on `recommendation` after a backfill window, +-- restoring the app to its normal (pre-backfill) query behaviour. +-- +-- The definitions below reproduce exactly what the migrations created (partial +-- WHERE clauses included). Run any time — the backfill does NOT need to be +-- complete first: these indexes don't reference plan_id/material_id, so a +-- half-finished backfill is invisible to them and to the app. +-- +-- Two of the three are PARTIAL (only rows where default = true and not yet +-- installed), so they cover a small slice of the table and rebuild in seconds; +-- only recommendation_property_id_idx spans all rows and takes a few minutes. +-- +-- Plain CREATE INDEX is fastest but briefly blocks writes while each builds. If +-- you need writes to keep flowing, replace CREATE INDEX with +-- CREATE INDEX CONCURRENTLY (slower, cannot run inside a transaction, and each +-- statement must be sent on its own). + +-- 1GB comfortably holds the ~500MB sort for a 21M-row single-column btree. +-- Keep it well under instance RAM (this DB is 4GB) to avoid memory pressure. +SET maintenance_work_mem = '1GB'; + +CREATE INDEX IF NOT EXISTS recommendation_property_id_idx + ON recommendation USING btree (property_id); + +CREATE INDEX IF NOT EXISTS idx_recommendation_active_defaults + ON recommendation USING btree (id) + WHERE "default" = true AND already_installed = false; + +CREATE INDEX IF NOT EXISTS idx_recommendation_active_id_property + ON recommendation USING btree (id, property_id) + WHERE "default" = true AND already_installed = false; diff --git a/src/app/db/sql/vacuum-recommendation.sql b/src/app/db/sql/vacuum-recommendation.sql new file mode 100644 index 00000000..d5495446 --- /dev/null +++ b/src/app/db/sql/vacuum-recommendation.sql @@ -0,0 +1,7 @@ +-- Reclaim the dead tuples the backfill left behind (it updated ~every row, so the +-- table is ~2x bloated) and refresh planner stats so queries pick up the new +-- indexes. Standard VACUUM (not FULL) — online, no exclusive lock, but it does a +-- full heap + index scan so it can take a while. No progress output; watch +-- pg_stat_progress_vacuum in another session. + +VACUUM (ANALYZE) recommendation; From 051927dda45ce0d3ff49ee01bcea7339d1ae07be Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 2 Jul 2026 11:43:32 +0000 Subject: [PATCH 27/27] fix(reporting): age-band and property-type breakdowns for new-approach properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getCountByAgeBand and getCountByPropertyType read year_built / property_type straight off the property row, which is NULL for new-approach properties — every one fell into the Unknown bucket on /portfolio/[slug]/reporting. Branch them like their already-migrated siblings (getCountByEpcBand, getEstimatedCounts): - propertyTypeSql: epc_property (lodged, else predicted) with RdSAP code→label mapping and dwelling_type fallback, mirroring resolvePropertyDescriptors; legacy stays on property.property_type. - constructionYearSql: the main building part's construction age band mapped to a representative start year (correlated subquery, since extension parts would multiply a plain JOIN), so both paths share the same year→band bucketing; legacy stays on numeric year_built. - The RdSAP label maps move into the shared-fragments section as the single source of truth; AGE_BAND_YEARS becomes CONSTRUCTION_AGE_BANDS carrying label + representative year. Band I (1996–2002) straddles the 1976–1999 / 2000+ boundary and buckets by its start year. Verified live against portfolio 814 (all 338 properties resolve, buckets match the epc_building_part distribution exactly) and legacy portfolios 419/433 (output identical to the pre-fix SQL). Co-Authored-By: Claude Fable 5 --- .../reporting/databaseFunctions.ts | 34 ++-- src/lib/services/epcSources.test.ts | 68 ++++++++ src/lib/services/epcSources.ts | 157 +++++++++++++----- 3 files changed, 200 insertions(+), 59 deletions(-) create mode 100644 src/lib/services/epcSources.test.ts diff --git a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts index 4fbe71d1..44d349c3 100644 --- a/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts +++ b/src/app/portfolio/[slug]/(portfolio)/reporting/databaseFunctions.ts @@ -11,6 +11,8 @@ import { isExpiredSql, effectiveSapSql, effectiveEpcBandSql, + propertyTypeSql, + constructionYearSql, } from "@/lib/services/epcSources"; import type { @@ -71,20 +73,21 @@ export async function getCountByAgeBand( const result = await db.execute(sql` SELECT CASE - WHEN year_built ~ '^[0-9]+$' THEN - CASE - WHEN CAST(year_built AS int) < 1900 THEN '<1900' - WHEN CAST(year_built AS int) BETWEEN 1900 AND 1929 THEN '1900–1929' - WHEN CAST(year_built AS int) BETWEEN 1930 AND 1949 THEN '1930–1949' - WHEN CAST(year_built AS int) BETWEEN 1950 AND 1975 THEN '1950–1975' - WHEN CAST(year_built AS int) BETWEEN 1976 AND 1999 THEN '1976–1999' - ELSE '2000+' - END + WHEN q.built_year < 1900 THEN '<1900' + WHEN q.built_year BETWEEN 1900 AND 1929 THEN '1900–1929' + WHEN q.built_year BETWEEN 1930 AND 1949 THEN '1930–1949' + WHEN q.built_year BETWEEN 1950 AND 1975 THEN '1950–1975' + WHEN q.built_year BETWEEN 1976 AND 1999 THEN '1976–1999' + WHEN q.built_year >= 2000 THEN '2000+' ELSE 'Unknown' END AS age_band, COUNT(*)::int AS count - FROM property - WHERE portfolio_id = ${portfolioId} + FROM ( + SELECT ${constructionYearSql} AS built_year + FROM property p + ${newApproachJoins} + WHERE p.portfolio_id = ${portfolioId} + ) q GROUP BY age_band ORDER BY age_band; `); @@ -149,10 +152,11 @@ export async function getCountByPropertyType( portfolioId: number, ): Promise { const result = await db.execute(sql` - SELECT property_type AS type, COUNT(*)::int AS count - FROM property - WHERE portfolio_id = ${portfolioId} - GROUP BY property_type + SELECT ${propertyTypeSql} AS type, COUNT(*)::int AS count + FROM property p + ${newApproachJoins} + WHERE p.portfolio_id = ${portfolioId} + GROUP BY type ORDER BY count DESC; `); diff --git a/src/lib/services/epcSources.test.ts b/src/lib/services/epcSources.test.ts new file mode 100644 index 00000000..1d85a6ab --- /dev/null +++ b/src/lib/services/epcSources.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + CONSTRUCTION_AGE_BANDS, + PROPERTY_TYPE_LABELS, +} from "./epcSources"; +import { PROPERTY_TYPE_OPTIONS } from "@/app/utils/propertyFilters"; + +// The reporting age-band bucketing in getCountByAgeBand (databaseFunctions.ts), +// mirrored so the CONSTRUCTION_AGE_BANDS representative years stay anchored to +// the buckets the chart renders. +function reportingAgeBand(year: number): string { + if (year < 1900) return "<1900"; + if (year <= 1929) return "1900–1929"; + if (year <= 1949) return "1930–1949"; + if (year <= 1975) return "1950–1975"; + if (year <= 1999) return "1976–1999"; + return "2000+"; +} + +describe("CONSTRUCTION_AGE_BANDS", () => { + it("keeps each representative year consistent with the band's label", () => { + for (const { label, year } of Object.values(CONSTRUCTION_AGE_BANDS)) { + if (label.startsWith("before")) { + expect(year).toBeLessThan(1900); + } else { + const start = Number(label.match(/\d{4}/)![0]); + expect(year).toBe(start); + } + } + }); + + it("buckets every RdSAP band into the expected reporting age band", () => { + const buckets = Object.fromEntries( + Object.entries(CONSTRUCTION_AGE_BANDS).map(([code, { year }]) => [ + code, + reportingAgeBand(year), + ]), + ); + expect(buckets).toEqual({ + A: "<1900", + B: "1900–1929", + C: "1930–1949", + D: "1950–1975", + E: "1950–1975", + F: "1976–1999", + G: "1976–1999", + H: "1976–1999", + // I (1996–2002) straddles the 1976–1999 / 2000+ boundary; it buckets by + // its start year. + I: "1976–1999", + J: "2000+", + K: "2000+", + L: "2000+", + M: "2000+", + }); + }); +}); + +describe("PROPERTY_TYPE_LABELS", () => { + it("maps codes to the labels the property-type filter options use", () => { + const optionLabels = new Set(PROPERTY_TYPE_OPTIONS.map((o) => o.label)); + for (const label of Object.values(PROPERTY_TYPE_LABELS)) { + // Park home has no filter option — it's rare and charts render it as-is. + if (label === "Park home") continue; + expect(optionLabels).toContain(label); + } + }); +}); diff --git a/src/lib/services/epcSources.ts b/src/lib/services/epcSources.ts index d644a1a9..b50cdd78 100644 --- a/src/lib/services/epcSources.ts +++ b/src/lib/services/epcSources.ts @@ -193,6 +193,116 @@ export const isExpiredSql = (e: Alias) => export const mainfuelSql = (e: Alias) => sql`CASE WHEN ${isNewApproachSql} THEN NULL ELSE ${e}.mainfuel END`; +// ───────────────────────────────────────────────────────────────────────────── +// RdSAP code → label maps for the descriptive fields the new pipeline stores as +// codes on epc_property / epc_building_part (the property-row text columns +// aren't written for new-approach properties). Unknown codes fall back to the +// raw value. Shared by the SQL fragments below and resolvePropertyDescriptors — +// the single source of truth for code→label mapping. +// ───────────────────────────────────────────────────────────────────────────── + +export const PROPERTY_TYPE_LABELS: Record = { + "0": "House", + "1": "Bungalow", + "2": "Flat", + "3": "Maisonette", + "4": "Park home", +}; +const BUILT_FORM_LABELS: Record = { + "1": "Detached", + "2": "Semi-Detached", + "3": "End-Terrace", + "4": "Mid-Terrace", + "5": "Enclosed End-Terrace", + "6": "Enclosed Mid-Terrace", +}; +// SAP tenure codes → the text labels already used in the data. NOTE: best-effort +// (standard SAP 1/2/3); most rows store the text label directly, only a few use +// codes. Confirm against the backend tenure enum. +const TENURE_LABELS: Record = { + "1": "Owner Occupied", + "2": "Rented Social", + "3": "Rented Private", +}; +/** + * Matches the backend ConstructionAgeBand enum (RdSAP England & Wales letters). + * `year` is a representative construction year (the band's start) for reporting + * queries that bucket by year — band I (1996–2002) straddles the reporting + * 1976–1999 / 2000+ boundary, so it buckets with its start year. + */ +export const CONSTRUCTION_AGE_BANDS: Record< + string, + { label: string; year: number } +> = { + A: { label: "before 1900", year: 1899 }, // no start year — any pre-1900 value + B: { label: "1900–1929", year: 1900 }, + C: { label: "1930–1949", year: 1930 }, + D: { label: "1950–1966", year: 1950 }, + E: { label: "1967–1975", year: 1967 }, + F: { label: "1976–1982", year: 1976 }, + G: { label: "1983–1990", year: 1983 }, + H: { label: "1991–1995", year: 1991 }, + I: { label: "1996–2002", year: 1996 }, + J: { label: "2003–2006", year: 2003 }, + K: { label: "2007–2011", year: 2007 }, + L: { label: "2012–2022", year: 2012 }, + M: { label: "2023 onwards", year: 2023 }, +}; + +/** `CASE WHEN THEN