Merge remote-tracking branch 'origin/main' into feature/bulk-trigger-modelling

This commit is contained in:
Khalim Conn-Kowlessar 2026-07-07 11:28:34 +00:00
commit 06ad4eec0d
12 changed files with 23847 additions and 198 deletions

View file

@ -0,0 +1 @@
ALTER TABLE "hubspot_deal_data" ADD COLUMN "client_booking_reference" text;

View file

@ -0,0 +1 @@
ALTER TABLE "tasks" ADD COLUMN "inputs" text;

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1821,6 +1821,20 @@
"when": 1783345567793,
"tag": "0260_natural_enchantress",
"breakpoints": true
},
{
"idx": 261,
"version": "7",
"when": 1783352788862,
"tag": "0261_fixed_excalibur",
"breakpoints": true
},
{
"idx": 262,
"version": "7",
"when": 1783420235209,
"tag": "0262_wandering_nocturne",
"breakpoints": true
}
]
}

View file

@ -81,6 +81,7 @@ export const hubspotDealData = pgTable("hubspot_deal_data", {
lastSubmissionDate: timestamp("last_submission_date", { precision: 6, withTimezone: true }),
numberOfAttempts: text("number_of_attempts"),
clientBookingReference: text("client_booking_reference"),
createdAt: timestamp("created_at", { precision: 6, withTimezone: true })
.defaultNow()

View file

@ -21,23 +21,10 @@ import {
type MultiEntrySample,
} from "@/lib/bulkUpload/multiEntry";
import type { MultiEntryOrdering } from "@/app/db/schema/bulk_address_uploads";
import {
PropertyTypeValues,
BuiltFormTypeValues,
WallTypeValues,
RoofTypeValues,
} from "@/app/db/schema/landlord_overrides";
import { CATEGORY_VALUES } from "@/lib/landlordOverrides/categories";
import { CLASSIFIER_FIELDS } from "@/lib/bulkUpload/columnFields";
import { statusLabel, isTerminalStatus } from "@/lib/bulkUpload/types";
// Valid enum options per classifier category, for the editable dropdowns (#299).
const CATEGORY_VALUES: Record<string, readonly string[]> = {
property_type: PropertyTypeValues,
built_form_type: BuiltFormTypeValues,
wall_type: WallTypeValues,
roof_type: RoofTypeValues,
};
// Our category label per classifier field (e.g. built_form_type → "Built Form").
// Distinguishes the categories when several read from the same source column —
// Property Type and Built Form both come from one column, so labelling by the
@ -362,7 +349,8 @@ function VerifyClassificationPanel({
seen.add(entry.description);
return true;
});
const options = CATEGORY_VALUES[column.field] ?? [];
const options =
CATEGORY_VALUES[column.field as keyof typeof CATEGORY_VALUES] ?? [];
return (
<div key={column.field}>
<p className="text-[11px] font-semibold uppercase tracking-wide text-amber-700">
@ -634,7 +622,9 @@ function UnresolvedClassificationsPanel({
<div className="mt-3 space-y-3">
{Object.entries(unknown).map(([field, descriptions]) => {
const options = (CATEGORY_VALUES[field] ?? []).filter((o) => o !== "Unknown");
const options = (
CATEGORY_VALUES[field as keyof typeof CATEGORY_VALUES] ?? []
).filter((o) => o !== "Unknown");
return (
<div key={field}>
<p className="text-[11px] font-semibold uppercase tracking-wide text-red-700">

View file

@ -87,15 +87,25 @@ function OverrideSection({ title, rows }: { title: string; rows: OverrideRow[] }
);
}
// Provenance of a resolved override value. os_places is the deterministic OS
// Places classification-code mapping (ADR-0007); user is a landlord correction;
// classifier is the ML default.
const SOURCE_BADGES: Record<string, { label: string; className: string }> = {
user: { label: "user", className: "bg-emerald-50 text-emerald-700" },
classifier: { label: "classifier", className: "bg-indigo-50 text-indigo-700" },
os_places: { label: "OS Places", className: "bg-sky-50 text-sky-700" },
};
function SourceBadge({ source }: { source: string }) {
const isUser = source === "user";
const badge = SOURCE_BADGES[source] ?? {
label: source,
className: "bg-gray-100 text-gray-600",
};
return (
<span
className={`inline-flex items-center px-2.5 py-1 rounded-full text-xs font-semibold ${
isUser ? "bg-emerald-50 text-emerald-700" : "bg-indigo-50 text-indigo-700"
}`}
className={`inline-flex items-center px-2.5 py-1 rounded-full text-xs font-semibold ${badge.className}`}
>
{isUser ? "user" : "classifier"}
{badge.label}
</span>
);
}

View file

@ -1,15 +1,11 @@
import { db } from "@/app/db/db";
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
import {
landlordPropertyTypeOverrides,
landlordBuiltFormTypeOverrides,
landlordWallTypeOverrides,
landlordRoofTypeOverrides,
PropertyTypeValues,
BuiltFormTypeValues,
WallTypeValues,
RoofTypeValues,
} from "@/app/db/schema/landlord_overrides";
CATEGORY_TABLES,
CATEGORY_VALUES,
LANDLORD_CATEGORIES,
isLandlordCategory,
} from "@/lib/landlordOverrides/categories";
import { tasks } from "@/app/db/schema/tasks/tasks";
import { subTasks } from "@/app/db/schema/tasks/subtask";
import { and, count, desc, eq, inArray, sql } from "drizzle-orm";
@ -135,37 +131,22 @@ export async function saveMultiEntrySummary(
// classified (yet)". Read-only (ADR-0004, issue #298).
export type SampleClassifications = Record<string, Record<string, string>>;
// Look up the classifier's resolved enums for one field's descriptions. One
// branch per category keeps drizzle's per-table enum typing intact.
// Look up the classifier's resolved enums for one field's descriptions, via the
// category registry (all nine landlord categories). Returns null for a field
// that is not a landlord category.
async function lookupOverrides(
field: string,
portfolioId: bigint,
descriptions: string[],
): Promise<Array<{ description: string; value: string }> | null> {
switch (field) {
case "property_type":
return db
.select({ description: landlordPropertyTypeOverrides.description, value: landlordPropertyTypeOverrides.value })
.from(landlordPropertyTypeOverrides)
.where(and(eq(landlordPropertyTypeOverrides.portfolioId, portfolioId), inArray(landlordPropertyTypeOverrides.description, descriptions)));
case "built_form_type":
return db
.select({ description: landlordBuiltFormTypeOverrides.description, value: landlordBuiltFormTypeOverrides.value })
.from(landlordBuiltFormTypeOverrides)
.where(and(eq(landlordBuiltFormTypeOverrides.portfolioId, portfolioId), inArray(landlordBuiltFormTypeOverrides.description, descriptions)));
case "wall_type":
return db
.select({ description: landlordWallTypeOverrides.description, value: landlordWallTypeOverrides.value })
.from(landlordWallTypeOverrides)
.where(and(eq(landlordWallTypeOverrides.portfolioId, portfolioId), inArray(landlordWallTypeOverrides.description, descriptions)));
case "roof_type":
return db
.select({ description: landlordRoofTypeOverrides.description, value: landlordRoofTypeOverrides.value })
.from(landlordRoofTypeOverrides)
.where(and(eq(landlordRoofTypeOverrides.portfolioId, portfolioId), inArray(landlordRoofTypeOverrides.description, descriptions)));
default:
return null;
}
if (!isLandlordCategory(field)) return null;
const { table } = CATEGORY_TABLES[field];
return db
.select({ description: table.description, value: table.value })
.from(table)
.where(
and(eq(table.portfolioId, portfolioId), inArray(table.description, descriptions)),
) as Promise<Array<{ description: string; value: string }>>;
}
// The classifier's enums for the review samples' entries, joined by the
@ -213,103 +194,47 @@ export type UnknownOverrides = Record<string, string[]>;
const UNKNOWN_VALUE = "Unknown";
async function unknownForField(field: string, portfolioId: bigint): Promise<string[]> {
switch (field) {
case "property_type":
return (
await db
.select({ description: landlordPropertyTypeOverrides.description })
.from(landlordPropertyTypeOverrides)
.where(and(eq(landlordPropertyTypeOverrides.portfolioId, portfolioId), eq(landlordPropertyTypeOverrides.value, UNKNOWN_VALUE)))
).map((r) => r.description);
case "built_form_type":
return (
await db
.select({ description: landlordBuiltFormTypeOverrides.description })
.from(landlordBuiltFormTypeOverrides)
.where(and(eq(landlordBuiltFormTypeOverrides.portfolioId, portfolioId), eq(landlordBuiltFormTypeOverrides.value, UNKNOWN_VALUE)))
).map((r) => r.description);
case "wall_type":
return (
await db
.select({ description: landlordWallTypeOverrides.description })
.from(landlordWallTypeOverrides)
.where(and(eq(landlordWallTypeOverrides.portfolioId, portfolioId), eq(landlordWallTypeOverrides.value, UNKNOWN_VALUE)))
).map((r) => r.description);
case "roof_type":
return (
await db
.select({ description: landlordRoofTypeOverrides.description })
.from(landlordRoofTypeOverrides)
.where(and(eq(landlordRoofTypeOverrides.portfolioId, portfolioId), eq(landlordRoofTypeOverrides.value, UNKNOWN_VALUE)))
).map((r) => r.description);
default:
return [];
}
if (!isLandlordCategory(field)) return [];
const { table } = CATEGORY_TABLES[field];
const rows = (await db
.select({ description: table.description })
.from(table)
.where(
and(eq(table.portfolioId, portfolioId), eq(table.value, UNKNOWN_VALUE)),
)) as Array<{ description: string }>;
return rows.map((r) => r.description);
}
export async function getUnknownOverrides(portfolioId: string): Promise<UnknownOverrides> {
const pid = BigInt(portfolioId);
const result: UnknownOverrides = {};
for (const field of ["property_type", "built_form_type", "wall_type", "roof_type"]) {
for (const field of LANDLORD_CATEGORIES) {
const descriptions = await unknownForField(field, pid);
if (descriptions.length > 0) result[field] = descriptions;
}
return result;
}
// Valid enum values per classifier category, for validating a user edit.
const CATEGORY_VALUES: Record<string, readonly string[]> = {
property_type: PropertyTypeValues,
built_form_type: BuiltFormTypeValues,
wall_type: WallTypeValues,
roof_type: RoofTypeValues,
};
// Upsert one user override (source='user') into the right category table. One
// branch per table keeps drizzle's per-table typing intact; the unique
// (portfolio_id, description) drives the conflict. Sets source='user' so the
// classifier's `ON CONFLICT … WHERE source='classifier'` never re-clobbers it.
// Upsert one user override (source='user') into the right category table via the
// registry. The unique (portfolio_id, description) drives the conflict. Sets
// source='user' so the classifier's `ON CONFLICT … WHERE source='classifier'`
// never re-clobbers it. No-op for a field that is not a landlord category
// (setClassificationOverride validates the field before calling).
async function upsertUserOverride(
field: string,
portfolioId: bigint,
description: string,
value: string,
): Promise<void> {
const now = new Date();
switch (field) {
case "property_type":
await db.insert(landlordPropertyTypeOverrides)
.values({ portfolioId, description, value, source: "user" })
.onConflictDoUpdate({
target: [landlordPropertyTypeOverrides.portfolioId, landlordPropertyTypeOverrides.description],
set: { value, source: "user", updatedAt: now },
});
return;
case "built_form_type":
await db.insert(landlordBuiltFormTypeOverrides)
.values({ portfolioId, description, value, source: "user" })
.onConflictDoUpdate({
target: [landlordBuiltFormTypeOverrides.portfolioId, landlordBuiltFormTypeOverrides.description],
set: { value, source: "user", updatedAt: now },
});
return;
case "wall_type":
await db.insert(landlordWallTypeOverrides)
.values({ portfolioId, description, value, source: "user" })
.onConflictDoUpdate({
target: [landlordWallTypeOverrides.portfolioId, landlordWallTypeOverrides.description],
set: { value, source: "user", updatedAt: now },
});
return;
case "roof_type":
await db.insert(landlordRoofTypeOverrides)
.values({ portfolioId, description, value, source: "user" })
.onConflictDoUpdate({
target: [landlordRoofTypeOverrides.portfolioId, landlordRoofTypeOverrides.description],
set: { value, source: "user", updatedAt: now },
});
return;
}
if (!isLandlordCategory(field)) return;
const { table } = CATEGORY_TABLES[field];
await db
.insert(table)
.values({ portfolioId, description, value, source: "user" })
.onConflictDoUpdate({
target: [table.portfolioId, table.description],
set: { value, source: "user", updatedAt: new Date() },
});
}
export type SetClassificationOutcome =
@ -326,8 +251,9 @@ export async function setClassificationOverride(
description: string,
value: string,
): Promise<SetClassificationOutcome> {
if (!isLandlordCategory(field))
return { kind: "invalid", reason: `Unknown category '${field}'` };
const allowed = CATEGORY_VALUES[field];
if (!allowed) return { kind: "invalid", reason: `Unknown category '${field}'` };
if (!allowed.includes(value))
return { kind: "invalid", reason: `'${value}' is not a valid ${field}` };

View file

@ -0,0 +1,52 @@
import { describe, expect, it } from "vitest";
import {
CATEGORY_TABLES,
CATEGORY_VALUES,
LANDLORD_CATEGORIES,
isLandlordCategory,
} from "./categories";
import { CLASSIFIER_FIELD_VALUES } from "@/lib/bulkUpload/columnFields";
import { OverrideComponentValues } from "@/app/db/schema/property_overrides";
// These tests guard the exact failure that left five categories rendering
// "not classified": a category defined in one place (schema table, mapping
// field, override component) but missing from the app-layer registry. If any of
// these sets drift apart, wiring is incomplete somewhere — fail loudly here
// rather than silently in the UI.
describe("landlord override category registry", () => {
const expected = [...LANDLORD_CATEGORIES].sort();
it("registry, value map, and table map cover the identical category set", () => {
expect(Object.keys(CATEGORY_TABLES).sort()).toEqual(expected);
expect(Object.keys(CATEGORY_VALUES).sort()).toEqual(expected);
});
it("covers exactly the classifier mapping fields", () => {
// Every classifier column the user can map must be wired, and vice versa —
// otherwise a mapped column classifies into a table nothing reads.
expect([...CLASSIFIER_FIELD_VALUES].sort()).toEqual(expected);
});
it("matches the property_overrides override_component enum", () => {
// The finaliser maps classifier category -> override_component with no
// translation, so the two vocabularies must be identical.
expect([...OverrideComponentValues].sort()).toEqual(expected);
});
it("every category has a non-empty value list that includes Unknown", () => {
// 'Unknown' is the classifier's fallback and the token the finalise gate
// (ADR-0006) blocks on; a category missing it would let Unknowns slip past.
for (const category of LANDLORD_CATEGORIES) {
expect(CATEGORY_VALUES[category].length).toBeGreaterThan(0);
expect(CATEGORY_VALUES[category]).toContain("Unknown");
}
});
it("isLandlordCategory accepts wired categories and rejects others", () => {
for (const category of LANDLORD_CATEGORIES) {
expect(isLandlordCategory(category)).toBe(true);
}
expect(isLandlordCategory("address_1")).toBe(false);
expect(isLandlordCategory("not_a_category")).toBe(false);
});
});

View file

@ -0,0 +1,98 @@
import type { PgTableWithColumns } from "drizzle-orm/pg-core";
import {
landlordPropertyTypeOverrides,
landlordBuiltFormTypeOverrides,
landlordWallTypeOverrides,
landlordRoofTypeOverrides,
landlordMainFuelOverrides,
landlordGlazingOverrides,
landlordConstructionAgeBandOverrides,
landlordWaterHeatingOverrides,
landlordMainHeatingSystemOverrides,
PropertyTypeValues,
BuiltFormTypeValues,
WallTypeValues,
RoofTypeValues,
MainFuelValues,
GlazingValues,
ConstructionAgeBandValues,
WaterHeatingValues,
MainHeatingSystemValues,
} from "@/app/db/schema/landlord_overrides";
// Single source of truth for the nine landlord-override categories. Adding a
// category here (alongside its schema table + value enum) wires it through the
// whole app layer at once: the read-only page, the verify/edit dropdowns, the
// Unknown finalise gate (ADR-0006), and user-edit validation. This registry
// replaces the per-category `switch` statements that previously had to be
// extended in ~6 places — a category defined in the schema but missing from one
// of those switches silently rendered as "not classified" (that is exactly what
// happened to the five non-core categories). The completeness test in
// categories.test.ts fails if this set ever drifts from CLASSIFIER_FIELDS or the
// value-enum exports.
//
// Order is display order (read-only page + verify panel) and intentionally
// mirrors CLASSIFIER_FIELDS. It is decoupled from the DB enum member order,
// which Postgres cannot reorder anyway — see docs/adr/0002.
export const LANDLORD_CATEGORIES = [
"property_type",
"built_form_type",
"wall_type",
"roof_type",
"main_fuel",
"glazing",
"construction_age_band",
"water_heating",
"main_heating_system",
] as const;
export type LandlordOverrideCategory = (typeof LANDLORD_CATEGORIES)[number];
// Every landlord_*_overrides table is structurally identical — id, portfolio_id,
// description, value<per-category enum>, source, timestamps — differing only in
// the `value` enum's allowed strings. We deliberately erase that per-table enum
// type here so a single generic helper can serve all nine tables; the switches
// this registry replaced kept the typing, but every consumer already reads and
// writes `value` as a plain string and validates it against `values` at runtime.
// The `any` is confined to the table reference; `values` stays fully typed.
type LandlordOverrideTable = PgTableWithColumns<{
name: string;
schema: undefined;
columns: any;
dialect: "pg";
}>;
export interface CategoryConfig {
table: LandlordOverrideTable;
values: readonly string[];
}
export const CATEGORY_TABLES: Record<LandlordOverrideCategory, CategoryConfig> = {
property_type: { table: landlordPropertyTypeOverrides, values: PropertyTypeValues },
built_form_type: { table: landlordBuiltFormTypeOverrides, values: BuiltFormTypeValues },
wall_type: { table: landlordWallTypeOverrides, values: WallTypeValues },
roof_type: { table: landlordRoofTypeOverrides, values: RoofTypeValues },
main_fuel: { table: landlordMainFuelOverrides, values: MainFuelValues },
glazing: { table: landlordGlazingOverrides, values: GlazingValues },
construction_age_band: {
table: landlordConstructionAgeBandOverrides,
values: ConstructionAgeBandValues,
},
water_heating: { table: landlordWaterHeatingOverrides, values: WaterHeatingValues },
main_heating_system: {
table: landlordMainHeatingSystemOverrides,
values: MainHeatingSystemValues,
},
};
// Valid enum values per category — populates the verify-panel edit dropdowns
// (client) and validates a user correction (server). Derived from the registry
// so it can never drift from the table set.
export const CATEGORY_VALUES: Record<LandlordOverrideCategory, readonly string[]> =
Object.fromEntries(
LANDLORD_CATEGORIES.map((category) => [category, CATEGORY_TABLES[category].values]),
) as Record<LandlordOverrideCategory, readonly string[]>;
export function isLandlordCategory(field: string): field is LandlordOverrideCategory {
return field in CATEGORY_TABLES;
}

View file

@ -1,11 +1,12 @@
import { db } from "@/app/db/db";
import {
landlordPropertyTypeOverrides,
landlordBuiltFormTypeOverrides,
landlordWallTypeOverrides,
landlordRoofTypeOverrides,
} from "@/app/db/schema/landlord_overrides";
import { asc, eq } from "drizzle-orm";
import {
CATEGORY_TABLES,
LANDLORD_CATEGORIES,
type LandlordOverrideCategory,
} from "./categories";
export type { LandlordOverrideCategory } from "./categories";
export interface OverrideRow {
description: string;
@ -13,67 +14,40 @@ export interface OverrideRow {
source: string;
}
export type LandlordOverrideCategory =
| "property_type"
| "built_form_type"
| "wall_type"
| "roof_type";
export type LandlordOverrideResults = Record<LandlordOverrideCategory, OverrideRow[]>;
const EMPTY: LandlordOverrideResults = {
property_type: [],
built_form_type: [],
wall_type: [],
roof_type: [],
};
function emptyResults(): LandlordOverrideResults {
const out = {} as LandlordOverrideResults;
for (const category of LANDLORD_CATEGORIES) out[category] = [];
return out;
}
// Reads the four landlord_*_overrides tables for a portfolio. The portfolio id
// is a bigint FK; the bulk-upload flow carries it as a numeric string.
// Reads every landlord_*_overrides table for a portfolio, one query per
// category, keyed by category. The portfolio id is a bigint FK; the bulk-upload
// flow carries it as a numeric string.
export async function getLandlordOverrides(
portfolioId: string
portfolioId: string,
): Promise<LandlordOverrideResults> {
if (!/^\d+$/.test(portfolioId)) return EMPTY;
if (!/^\d+$/.test(portfolioId)) return emptyResults();
const pid = BigInt(portfolioId);
const [property_type, built_form_type, wall_type, roof_type] = await Promise.all([
db
.select({
description: landlordPropertyTypeOverrides.description,
value: landlordPropertyTypeOverrides.value,
source: landlordPropertyTypeOverrides.source,
})
.from(landlordPropertyTypeOverrides)
.where(eq(landlordPropertyTypeOverrides.portfolioId, pid))
.orderBy(asc(landlordPropertyTypeOverrides.description)),
db
.select({
description: landlordBuiltFormTypeOverrides.description,
value: landlordBuiltFormTypeOverrides.value,
source: landlordBuiltFormTypeOverrides.source,
})
.from(landlordBuiltFormTypeOverrides)
.where(eq(landlordBuiltFormTypeOverrides.portfolioId, pid))
.orderBy(asc(landlordBuiltFormTypeOverrides.description)),
db
.select({
description: landlordWallTypeOverrides.description,
value: landlordWallTypeOverrides.value,
source: landlordWallTypeOverrides.source,
})
.from(landlordWallTypeOverrides)
.where(eq(landlordWallTypeOverrides.portfolioId, pid))
.orderBy(asc(landlordWallTypeOverrides.description)),
db
.select({
description: landlordRoofTypeOverrides.description,
value: landlordRoofTypeOverrides.value,
source: landlordRoofTypeOverrides.source,
})
.from(landlordRoofTypeOverrides)
.where(eq(landlordRoofTypeOverrides.portfolioId, pid))
.orderBy(asc(landlordRoofTypeOverrides.description)),
]);
const entries = await Promise.all(
LANDLORD_CATEGORIES.map(async (category) => {
const { table } = CATEGORY_TABLES[category];
const rows = (await db
.select({
description: table.description,
value: table.value,
source: table.source,
})
.from(table)
.where(eq(table.portfolioId, pid))
.orderBy(asc(table.description))) as OverrideRow[];
return [category, rows] as const;
}),
);
return { property_type, built_form_type, wall_type, roof_type };
const out = emptyResults();
for (const [category, rows] of entries) out[category] = rows;
return out;
}