From 4422cebe0bab1a7642c79dccb1efa9fe46e9a42c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 23 Jul 2026 10:15:24 +0000 Subject: [PATCH] =?UTF-8?q?feat(ara-projects):=20import=20mapping=20?= =?UTF-8?q?=E2=80=94=20column=20+=20value=20mapping=20logic=20(#415)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure half of import step 2 of 3, built on #414's parsed-rows-held-client-side model. No database writes, no migrations; four new modules under src/lib/projects/import, all pure except one read-only loader. similarity.ts — the auto-match primitive Normalisation plus a Sørensen–Dice bigram score, with word containment ranked above letter overlap ("Contractor" ⊂ "Contractor Name" beats "Stage"/"Storage"). Headings and cell values match by *different* rules: valueSimilarity adds directional, length-weighted word coverage so "Window Programme" reaches "Windows" (0.91) and "Door Replacement" reaches "Doors", at a higher threshold (0.75 vs 0.62) so "Acme Windows Ltd" does not reach "Acme Doors Ltd" (0.67). The leniency is deliberately kept out of heading matching, where it would map "Forecast Start" onto "Forecast end date" and mis-read every row in the file. columnMapping.ts — file column -> Ara field Seven fields (landlord property id, UPRN, workstream, contractor, stage, WO reference, forecast end date) with alias lists drawn from real asset-management exports. Requirements are modelled as groups: property identifier is satisfied by landlord property id *or* UPRN, so a file may carry either or both. autoMatchColumns assigns greedily — one column per field, best pair first, and no guess below the threshold. Two deliberate non-matches: a Status column is not the Stage field (CONTEXT.md keeps them apart), and nothing matches Priority or cost (v1 ignores them; #426 owns priority). Per-column status is matched / ignored / needs_review, where "ignored" is the user's explicit "don't import" and is distinct from not having looked yet. valueMapping.ts — cell values -> the project's own entities Contractor and stage values are keyed by (resolved workstream, value), never by value alone: "Acme" under Windows is a different assignment from "Acme" under Doors and may not exist under one of them (ADR-0019). Two file spellings of one workstream merge into a single contractor question. Rows whose workstream is blank or unmapped have no scope, so their contractor/stage values are counted (unscopedRows) rather than guessed, and appear the moment the workstream is mapped. Value matching is many-to-one (several spellings, one workstream) where column matching is one-to-one, and a tie between two equally-good candidates is refused rather than guessed. A user's answer that the workstream no longer offers lapses to needs-review instead of persisting as a dangling id. warningRows is counted per row by a second pass, not by summing the tables — a row with an unmapped workstream and a blank contractor is one broken row. mappingTargets.ts — what values may map to Pure assembleMappingTargets + one read-only loadMappingTargets (three SELECTs). Every bigserial id is stringified once, here, because bigint cannot cross the server->client boundary. Stages keep ladder order (stages[0] is the blank-stage fallback at commit); workstreams and contractors sort alphabetically. session.ts — the artefact the mapping is persisted alongside ImportSession carries the file identity plus both mapping layers, JSON-clean end to end. sessionMatchesFile refuses a session whose headers have moved, since a mapping is column indices. Blocking rule (AC): only unmapped *required* columns block progression to validation. Unmapped values are warnings whose rows fail at #417; a blank stage is not a warning at all (it falls back to the workstream's first stage). 112 tests pass in src/lib/projects/import (66 new); tsc clean. Tests mock @/app/db/db and open no connection. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/projects/import/columnMapping.test.ts | 230 +++++++++ src/lib/projects/import/columnMapping.ts | 396 ++++++++++++++ .../projects/import/mappingTargets.test.ts | 164 ++++++ src/lib/projects/import/mappingTargets.ts | 223 ++++++++ src/lib/projects/import/session.test.ts | 72 +++ src/lib/projects/import/session.ts | 88 ++++ src/lib/projects/import/similarity.test.ts | 146 ++++++ src/lib/projects/import/similarity.ts | 226 ++++++++ src/lib/projects/import/valueMapping.test.ts | 355 +++++++++++++ src/lib/projects/import/valueMapping.ts | 488 ++++++++++++++++++ 10 files changed, 2388 insertions(+) create mode 100644 src/lib/projects/import/columnMapping.test.ts create mode 100644 src/lib/projects/import/columnMapping.ts create mode 100644 src/lib/projects/import/mappingTargets.test.ts create mode 100644 src/lib/projects/import/mappingTargets.ts create mode 100644 src/lib/projects/import/session.test.ts create mode 100644 src/lib/projects/import/session.ts create mode 100644 src/lib/projects/import/similarity.test.ts create mode 100644 src/lib/projects/import/similarity.ts create mode 100644 src/lib/projects/import/valueMapping.test.ts create mode 100644 src/lib/projects/import/valueMapping.ts diff --git a/src/lib/projects/import/columnMapping.test.ts b/src/lib/projects/import/columnMapping.test.ts new file mode 100644 index 00000000..e2cd43ec --- /dev/null +++ b/src/lib/projects/import/columnMapping.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; +import { + assignColumn, + autoMatchColumns, + cellFor, + columnMappingRows, + EMPTY_COLUMN_MAPPING, + fieldForColumn, + IMPORT_REQUIREMENTS, + summariseColumnMapping, +} from "./columnMapping"; +import { IMPORT_TEMPLATE_COLUMNS } from "./parse"; + +/** A realistic export from an asset-management system, not our own template. */ +const CLIENT_HEADERS = [ + "Asset Ref", + "Works Order Number", + "Work Type", + "Contractor Name", + "Forecast End", + "Priority", +]; + +describe("autoMatchColumns", () => { + it("maps every column of our own template", () => { + const mapping = autoMatchColumns([...IMPORT_TEMPLATE_COLUMNS]); + + expect(mapping.fields).toEqual({ + landlordPropertyId: 0, + workstream: 1, + contractor: 2, + stage: 3, + workOrderReference: 4, + forecastEndDate: 5, + }); + }); + + it("maps a client's own headings by similarity", () => { + const mapping = autoMatchColumns(CLIENT_HEADERS); + + expect(mapping.fields.landlordPropertyId).toBe(0); // Asset Ref + expect(mapping.fields.workOrderReference).toBe(1); // Works Order Number + expect(mapping.fields.workstream).toBe(2); // Work Type + expect(mapping.fields.contractor).toBe(3); // Contractor Name + expect(mapping.fields.forecastEndDate).toBe(4); // Forecast End + }); + + it("leaves a column it cannot place unmapped rather than guessing", () => { + // v1 has no Priority field (#426 owns priority), so it must stay unmapped + // and show as needs review — not be forced into the nearest field. + const mapping = autoMatchColumns(CLIENT_HEADERS); + expect(fieldForColumn(mapping, 5)).toBeNull(); + expect(Object.values(mapping.fields)).not.toContain(5); + }); + + it("does not read a Status column as the Stage field", () => { + // A contractor's own status vocabulary is not this project's stage ladder, + // so the user has to say so (CONTEXT.md keeps "status" away from Stage). + const mapping = autoMatchColumns(["Property ID", "Workstream", "Status"]); + expect(mapping.fields.stage).toBeUndefined(); + }); + + it("gives the identifier fields a column each when the file carries both", () => { + const mapping = autoMatchColumns(["Property Reference", "UPRN"]); + expect(mapping.fields.landlordPropertyId).toBe(0); + expect(mapping.fields.uprn).toBe(1); + }); + + it("maps nothing, and blames nothing, for headings it has never seen", () => { + const mapping = autoMatchColumns(["Col1", "Col2", "Col3"]); + expect(mapping.fields).toEqual({}); + expect(mapping.ignoredColumns).toEqual([]); + }); +}); + +describe("assignColumn", () => { + it("returns a new mapping and leaves the original untouched", () => { + const before = autoMatchColumns([...IMPORT_TEMPLATE_COLUMNS]); + const after = assignColumn(before, 3, null); + + expect(before.fields.stage).toBe(3); + expect(after.fields.stage).toBeUndefined(); + }); + + it("moves a field off the column that held it, so two columns never share one", () => { + const mapping = assignColumn( + autoMatchColumns([...IMPORT_TEMPLATE_COLUMNS]), + 5, + "workstream", + ); + + expect(mapping.fields.workstream).toBe(5); + // Column 1 was the workstream; it must now supply nothing rather than + // silently continuing to claim the field. + expect(fieldForColumn(mapping, 1)).toBeNull(); + }); + + it("clears the column's previous field when it takes a new one", () => { + const mapping = assignColumn( + autoMatchColumns([...IMPORT_TEMPLATE_COLUMNS]), + 1, + "stage", + ); + + expect(mapping.fields.stage).toBe(1); + expect(mapping.fields.workstream).toBeUndefined(); + }); + + it("records an explicit 'don't import' distinctly from never having looked", () => { + const mapping = assignColumn(EMPTY_COLUMN_MAPPING, 2, "ignore"); + + expect(mapping.ignoredColumns).toEqual([2]); + expect( + columnMappingRows(["a", "b", "c"], mapping).map((r) => r.status), + ).toEqual(["needs_review", "needs_review", "ignored"]); + }); + + it("un-ignores a column when it is given a field", () => { + const ignored = assignColumn(EMPTY_COLUMN_MAPPING, 2, "ignore"); + const mapped = assignColumn(ignored, 2, "workstream"); + + expect(mapped.ignoredColumns).toEqual([]); + expect(mapped.fields.workstream).toBe(2); + }); +}); + +describe("columnMappingRows", () => { + it("returns one row per file column, in file order, with a sample cell", () => { + const rows = columnMappingRows( + ["Asset Ref", "Work Type"], + autoMatchColumns(["Asset Ref", "Work Type"]), + [ + ["", "Window Programme"], + ["100245", "Window Programme"], + ], + ); + + expect(rows.map((r) => r.header)).toEqual(["Asset Ref", "Work Type"]); + // The first *non-blank* cell, so a gap in row 1 does not blank the hint. + expect(rows[0].sample).toBe("100245"); + expect(rows[0].field).toBe("landlordPropertyId"); + expect(rows[0].status).toBe("matched"); + }); + + it("reports an empty sample when the column is blank throughout", () => { + const rows = columnMappingRows(["Stage"], EMPTY_COLUMN_MAPPING, [[""], [""]]); + expect(rows[0].sample).toBe(""); + }); +}); + +describe("summariseColumnMapping", () => { + it("counts matched, ignored and needs-review columns", () => { + const mapping = assignColumn(autoMatchColumns(CLIENT_HEADERS), 5, "ignore"); + const summary = summariseColumnMapping(CLIENT_HEADERS, mapping); + + expect(summary.matched).toBe(5); + expect(summary.ignored).toBe(1); + expect(summary.needsReview).toBe(0); + }); + + it("blocks while a required field has no column, and says which", () => { + // Contractor is required (ADR-0019) and this file does not carry it. + const headers = ["Property ID", "Work Type"]; + const summary = summariseColumnMapping(headers, autoMatchColumns(headers)); + + expect(summary.complete).toBe(false); + expect(summary.missingRequirements.map((r) => r.id)).toEqual(["contractor"]); + }); + + it("treats the property identifier as satisfied by either column", () => { + const withUprn = summariseColumnMapping( + ["UPRN", "Workstream", "Contractor"], + autoMatchColumns(["UPRN", "Workstream", "Contractor"]), + ); + const withLandlordId = summariseColumnMapping( + ["Property Ref", "Workstream", "Contractor"], + autoMatchColumns(["Property Ref", "Workstream", "Contractor"]), + ); + + expect(withUprn.complete).toBe(true); + expect(withLandlordId.complete).toBe(true); + }); + + it("blocks when neither identifier column is mapped", () => { + const headers = ["Workstream", "Contractor"]; + const summary = summariseColumnMapping(headers, autoMatchColumns(headers)); + + expect(summary.complete).toBe(false); + expect(summary.missingRequirements.map((r) => r.id)).toEqual([ + "propertyIdentifier", + ]); + }); + + it("never blocks on an optional field", () => { + // No Stage, no reference, no forecast end — all optional, so this proceeds. + const headers = ["Property ID", "Workstream", "Contractor"]; + const summary = summariseColumnMapping(headers, autoMatchColumns(headers)); + + expect(summary.complete).toBe(true); + expect(summary.missingRequirements).toEqual([]); + expect(summary.unmappedOptionalFields).toEqual([ + "stage", + "workOrderReference", + "forecastEndDate", + ]); + }); + + it("names all three requirements when nothing is mapped", () => { + const summary = summariseColumnMapping(["a", "b"], EMPTY_COLUMN_MAPPING); + expect(summary.missingRequirements).toHaveLength(IMPORT_REQUIREMENTS.length); + expect(summary.complete).toBe(false); + }); +}); + +describe("cellFor", () => { + const mapping = autoMatchColumns([...IMPORT_TEMPLATE_COLUMNS]); + + it("reads the cell the field's column points at", () => { + const row = ["PROP-1", "Windows", "Acme", "Ordered", "WO-1", "2026-09-30"]; + expect(cellFor(row, mapping, "contractor")).toBe("Acme"); + }); + + it("reads an unmapped field as blank rather than undefined", () => { + expect(cellFor(["PROP-1"], EMPTY_COLUMN_MAPPING, "contractor")).toBe(""); + }); + + it("reads a short row as blank rather than undefined", () => { + expect(cellFor(["PROP-1"], mapping, "contractor")).toBe(""); + }); +}); diff --git a/src/lib/projects/import/columnMapping.ts b/src/lib/projects/import/columnMapping.ts new file mode 100644 index 00000000..24ea4999 --- /dev/null +++ b/src/lib/projects/import/columnMapping.ts @@ -0,0 +1,396 @@ +/** + * Work-order import — column mapping (issue #415, step 2 of 3, layer 1 of 2). + * + * Layer 1 answers "which file column is which Ara field?". Layer 2 + * (`./valueMapping`) answers "which of this column's values is which of the + * project's workstreams / contractors / stages?". + * + * Pure and DB-free like `./parse`: it takes the headers the parser found and + * returns data. Every function is total and immutable — the component holds one + * `ImportColumnMapping` in state and swaps it for a new one on each edit, so + * the whole screen (statuses, counts, the value-mapping tables) re-derives from + * that single value with no effect and no cached copy to fall out of step. + * + * ## Required vs optional (ADR-0019) + * + * The parser is header-agnostic on purpose; requiredness is enforced *here*. + * Property identifier, Workstream and Contractor are required; Stage, Work + * order reference and Forecast end date are optional. Property identifier is + * required as a **group**: a file may carry a landlord property id, a UPRN or + * both, and mapping either one satisfies it (which one a row matches on is + * validation's business at #417). + * + * Priority and cost columns are deliberately absent from the field list — v1 + * ignores them (#426 owns priority), so they land as unmapped columns the user + * can dismiss. + */ + +import { + AUTO_MATCH_THRESHOLD, + bestSimilarity, + greedyPairs, +} from "./similarity"; + +/** The Ara fields an import file column can be mapped to. */ +export type ImportFieldKey = + | "landlordPropertyId" + | "uprn" + | "workstream" + | "contractor" + | "stage" + | "workOrderReference" + | "forecastEndDate"; + +export interface ImportFieldDef { + key: ImportFieldKey; + /** What the field is called in the UI, and the first phrase auto-match scores against. */ + label: string; + /** One line under the label in the field picker. */ + hint: string; + /** Extra phrases auto-match scores against — the headings real files use. */ + aliases: readonly string[]; +} + +/** + * Every mappable field, in the order the template ships its columns. + * + * The alias lists are drawn from the headings asset-management exports actually + * use ("Asset Ref", "Works Order Number", "Work Type"). Two deliberate + * omissions: + * - `stage` does **not** alias "status". A file's Status column is usually a + * contractor's own workflow state rather than a stage on this project's + * ladder, and CONTEXT.md keeps "status" away from Stage for that reason. + * It shows as *needs review* so the user makes the call. + * - nothing aliases "priority" or "cost" — v1 has no field for them. + */ +export const IMPORT_FIELDS: readonly ImportFieldDef[] = [ + { + key: "landlordPropertyId", + label: "Landlord property id", + hint: "Your own reference for the property", + aliases: [ + "property id", + "property identifier", + "property reference", + "property ref", + "landlord property id", + "landlord reference", + "asset id", + "asset ref", + "asset reference", + "asset number", + ], + }, + { + key: "uprn", + label: "UPRN", + hint: "Unique Property Reference Number", + aliases: ["uprn", "unique property reference number"], + }, + { + key: "workstream", + label: "Workstream", + hint: "The category of works — mapped to this project's workstreams", + aliases: [ + "workstream", + "work stream", + "work type", + "works type", + "programme", + "work programme", + "works programme", + "job type", + "trade", + ], + }, + { + key: "contractor", + label: "Contractor", + hint: "Who delivers the work — mapped to this workstream's contractors", + aliases: [ + "contractor", + "contractor name", + "supplier", + "supplier name", + "installer", + "delivery partner", + "company", + ], + }, + { + key: "stage", + label: "Stage", + hint: "Optional — blank rows start at the workstream's first stage", + aliases: ["stage", "stage name", "current stage", "work stage", "phase"], + }, + { + key: "workOrderReference", + label: "Work order reference", + hint: "Optional — your reference for the work order", + aliases: [ + "work order reference", + "work order ref", + "work order number", + "works order number", + "wo reference", + "wo ref", + "wo number", + "order number", + "job number", + "job reference", + ], + }, + { + key: "forecastEndDate", + label: "Forecast end date", + hint: "Optional — when the work is expected to finish", + aliases: [ + "forecast end", + "forecast end date", + "forecast completion", + "forecast completion date", + "planned end date", + "target completion date", + "completion date", + "due date", + "end date", + ], + }, +] as const; + +export const IMPORT_FIELDS_BY_KEY: Record = + Object.fromEntries(IMPORT_FIELDS.map((f) => [f.key, f])) as Record< + ImportFieldKey, + ImportFieldDef + >; + +/** + * A requirement is satisfied when **any** of its keys is mapped. + * + * Single-key requirements are the ordinary case; `propertyIdentifier` is the + * reason the concept exists — landlord property id *or* UPRN (or both) will do. + */ +export interface ImportRequirement { + id: string; + label: string; + keys: readonly ImportFieldKey[]; + /** Shown when the requirement is unmet, so the blocker explains itself. */ + reason: string; +} + +export const IMPORT_REQUIREMENTS: readonly ImportRequirement[] = [ + { + id: "propertyIdentifier", + label: "Property identifier", + keys: ["landlordPropertyId", "uprn"], + reason: + "Map a landlord property id or a UPRN column (or both) so rows can be matched to properties.", + }, + { + id: "workstream", + label: "Workstream", + keys: ["workstream"], + reason: "Every row must say which workstream the work belongs to.", + }, + { + id: "contractor", + label: "Contractor", + keys: ["contractor"], + reason: "Every row must name the contractor delivering the work.", + }, +] as const; + +/** Field keys that are optional — everything not named by a requirement. */ +export const OPTIONAL_FIELD_KEYS: readonly ImportFieldKey[] = IMPORT_FIELDS.map( + (f) => f.key, +).filter((key) => !IMPORT_REQUIREMENTS.some((r) => r.keys.includes(key))); + +/** + * The user's declaration of which column is which field. + * + * `fields` is keyed by field rather than by column because that is the question + * every downstream consumer asks ("which column holds the contractor?") and + * because it makes the one-column-per-field invariant unrepresentable-otherwise + * rather than merely enforced. + * + * `ignoredColumns` records columns the user has *decided* not to import, which + * is a different state from a column they have not looked at yet — the first + * shows as settled, the second as needs review. + */ +export interface ImportColumnMapping { + fields: Partial>; + ignoredColumns: number[]; +} + +export const EMPTY_COLUMN_MAPPING: ImportColumnMapping = { + fields: {}, + ignoredColumns: [], +}; + +/** Per-file-column status, as the wireframe's column-mapping table renders it. */ +export type ColumnMappingStatus = "matched" | "ignored" | "needs_review"; + +export interface ColumnMappingRow { + columnIndex: number; + header: string; + /** The field this column supplies, or null when unmapped. */ + field: ImportFieldKey | null; + status: ColumnMappingStatus; + /** A non-blank sample cell for this column, for the "e.g. …" hint. Empty when none. */ + sample: string; +} + +/** + * Suggest a mapping from the file's headings alone. + * + * Scores every (column, field) pair on heading similarity and takes the best + * pairs first, so a file carrying both "Property Ref" and "UPRN" gives each its + * own field instead of letting the first column win both. Columns that match + * nothing are left unmapped — *needs review*, never a guess. + */ +export function autoMatchColumns(headers: string[]): ImportColumnMapping { + const pairs = greedyPairs( + headers, + IMPORT_FIELDS, + (header, field) => bestSimilarity(header, [field.label, ...field.aliases]), + AUTO_MATCH_THRESHOLD, + ); + + const fields: Partial> = {}; + for (const pair of pairs) { + fields[IMPORT_FIELDS[pair.targetIndex].key] = pair.candidateIndex; + } + return { fields, ignoredColumns: [] }; +} + +/** + * Point a column at a field, at nothing, or explicitly at "don't import". + * + * Returns a new mapping; the caller's copy is untouched. Assigning a field that + * another column already supplies moves it, because two columns cannot both be + * the contractor and silently dropping one would be worse than saying so. + */ +export function assignColumn( + mapping: ImportColumnMapping, + columnIndex: number, + target: ImportFieldKey | "ignore" | null, +): ImportColumnMapping { + const fields = { ...mapping.fields }; + + // This column is being repurposed, so it stops supplying whatever it did. + for (const [key, index] of Object.entries(fields)) { + if (index === columnIndex) delete fields[key as ImportFieldKey]; + } + + const ignoredColumns = mapping.ignoredColumns.filter((i) => i !== columnIndex); + + if (target === "ignore") { + ignoredColumns.push(columnIndex); + ignoredColumns.sort((a, b) => a - b); + } else if (target !== null) { + // One column per field: whoever held it before gives it up. + fields[target] = columnIndex; + } + + return { fields, ignoredColumns }; +} + +/** The field a column supplies, or null. */ +export function fieldForColumn( + mapping: ImportColumnMapping, + columnIndex: number, +): ImportFieldKey | null { + for (const field of IMPORT_FIELDS) { + if (mapping.fields[field.key] === columnIndex) return field.key; + } + return null; +} + +/** + * One row per file column, in file order — the left-hand table. + * + * `rows` supplies the "e.g. …" sample under each heading; pass the preview + * rather than all 20,000, the first non-blank cell is all it looks for. + */ +export function columnMappingRows( + headers: string[], + mapping: ImportColumnMapping, + rows: string[][] = [], +): ColumnMappingRow[] { + return headers.map((header, columnIndex) => { + const field = fieldForColumn(mapping, columnIndex); + const ignored = mapping.ignoredColumns.includes(columnIndex); + return { + columnIndex, + header, + field, + status: field ? "matched" : ignored ? "ignored" : "needs_review", + sample: firstNonBlank(rows, columnIndex), + }; + }); +} + +/** The first non-blank cell in a column, for the sample hint. */ +function firstNonBlank(rows: string[][], columnIndex: number): string { + for (const row of rows) { + const cell = row[columnIndex]; + if (cell !== undefined && cell !== "") return cell; + } + return ""; +} + +export interface ColumnMappingSummary { + /** Columns pointed at a field — the "Auto-matched" figure once nothing has been edited. */ + matched: number; + /** Columns the user has explicitly dismissed. */ + ignored: number; + /** Columns neither mapped nor dismissed — the "Needs review" figure. */ + needsReview: number; + /** Requirements with no column behind them. Empty means the step may proceed. */ + missingRequirements: ImportRequirement[]; + /** Optional fields with no column — informational, never a blocker. */ + unmappedOptionalFields: ImportFieldKey[]; + /** + * True when every required field has a column. This — and only this — is what + * gates progression to validation; unmapped *values* are warnings whose rows + * fail downstream (#417), not blockers here. + */ + complete: boolean; +} + +export function summariseColumnMapping( + headers: string[], + mapping: ImportColumnMapping, +): ColumnMappingSummary { + const rows = columnMappingRows(headers, mapping); + const missingRequirements = IMPORT_REQUIREMENTS.filter( + (requirement) => + !requirement.keys.some((key) => mapping.fields[key] !== undefined), + ); + + return { + matched: rows.filter((r) => r.status === "matched").length, + ignored: rows.filter((r) => r.status === "ignored").length, + needsReview: rows.filter((r) => r.status === "needs_review").length, + missingRequirements, + unmappedOptionalFields: OPTIONAL_FIELD_KEYS.filter( + (key) => mapping.fields[key] === undefined, + ), + complete: missingRequirements.length === 0, + }; +} + +/** + * A row's cell for a field, or `""` when the field is unmapped or the row is + * short. Shared by the value-mapping layer and (at #417) by validation, so + * "unmapped reads as blank" is decided in exactly one place. + */ +export function cellFor( + row: string[], + mapping: ImportColumnMapping, + field: ImportFieldKey, +): string { + const columnIndex = mapping.fields[field]; + if (columnIndex === undefined) return ""; + return row[columnIndex] ?? ""; +} diff --git a/src/lib/projects/import/mappingTargets.test.ts b/src/lib/projects/import/mappingTargets.test.ts new file mode 100644 index 00000000..4bc861e3 --- /dev/null +++ b/src/lib/projects/import/mappingTargets.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from "vitest"; + +// The module imports `@/app/db/db` for its loader. Stub it so importing the +// module opens no connection and the production database is never touched; the +// pure assembly tests never reach it and the loader tests drive it explicitly. +const mockSelect = vi.fn(); +vi.mock("@/app/db/db", () => ({ db: { select: () => mockSelect() } })); + +import { + assembleMappingTargets, + loadMappingTargets, + UNNAMED_CONTRACTOR, + type ContractorTargetFacts, + type StageTargetFacts, + type WorkstreamTargetFacts, +} from "./mappingTargets"; + +const WORKSTREAMS: WorkstreamTargetFacts[] = [ + { projectWorkstreamId: 2n, workstreamId: 11n, workstreamName: "Windows" }, + { projectWorkstreamId: 1n, workstreamId: 10n, workstreamName: "Doors" }, +]; + +const STAGES: StageTargetFacts[] = [ + { id: 203n, projectWorkstreamId: 2n, name: "Completed", order: 3 }, + { id: 201n, projectWorkstreamId: 2n, name: "Ordered", order: 1 }, + { id: 202n, projectWorkstreamId: 2n, name: "In progress", order: 2 }, +]; + +const CONTRACTORS: ContractorTargetFacts[] = [ + { + id: 403n, + projectWorkstreamId: 2n, + organisationId: "org-b", + organisationName: "Sunrise Glazing", + }, + { + id: 402n, + projectWorkstreamId: 2n, + organisationId: "org-a", + organisationName: "Acme Windows Ltd", + }, +]; + +describe("assembleMappingTargets", () => { + it("nests stages and contractors under their workstream", () => { + const targets = assembleMappingTargets(WORKSTREAMS, STAGES, CONTRACTORS); + const windows = targets.workstreams.find((w) => w.name === "Windows")!; + + expect(windows.id).toBe("2"); + expect(windows.stages).toHaveLength(3); + expect(windows.contractors).toHaveLength(2); + expect(targets.workstreams.find((w) => w.name === "Doors")!.stages).toEqual([]); + }); + + it("stringifies every bigint id, so the result can cross to a client component", () => { + const targets = assembleMappingTargets(WORKSTREAMS, STAGES, CONTRACTORS); + + expect(() => JSON.stringify(targets)).not.toThrow(); + const windows = targets.workstreams.find((w) => w.name === "Windows")!; + expect(windows.workstreamId).toBe("11"); + expect(windows.stages[0].id).toBe("201"); + expect(windows.contractors[0].id).toBe("402"); + }); + + it("orders stages by their ladder order, not by name or id", () => { + const windows = assembleMappingTargets(WORKSTREAMS, STAGES, []) + .workstreams.find((w) => w.name === "Windows")!; + + // stages[0] is load-bearing: a blank Stage cell falls back to it (ADR-0019). + expect(windows.stages.map((s) => s.name)).toEqual([ + "Ordered", + "In progress", + "Completed", + ]); + }); + + it("orders workstreams and contractors alphabetically", () => { + const targets = assembleMappingTargets(WORKSTREAMS, STAGES, CONTRACTORS); + + expect(targets.workstreams.map((w) => w.name)).toEqual(["Doors", "Windows"]); + expect( + targets.workstreams.find((w) => w.name === "Windows")!.contractors.map((c) => c.name), + ).toEqual(["Acme Windows Ltd", "Sunrise Glazing"]); + }); + + it("labels an organisation with no name rather than showing a blank option", () => { + const targets = assembleMappingTargets( + WORKSTREAMS, + [], + [ + { + id: 404n, + projectWorkstreamId: 1n, + organisationId: "org-x", + organisationName: null, + }, + ], + ); + + expect( + targets.workstreams.find((w) => w.name === "Doors")!.contractors[0].name, + ).toBe(UNNAMED_CONTRACTOR); + }); + + it("drops children belonging to no workstream on this project", () => { + // Only reachable if a caller mixes projects — attaching them elsewhere + // would put a contractor in front of work they are not assigned to. + const targets = assembleMappingTargets( + WORKSTREAMS, + [{ id: 999n, projectWorkstreamId: 77n, name: "Elsewhere", order: 1 }], + [ + { + id: 998n, + projectWorkstreamId: 77n, + organisationId: "org-z", + organisationName: "Other Project Ltd", + }, + ], + ); + + expect(targets.workstreams.flatMap((w) => w.stages)).toEqual([]); + expect(targets.workstreams.flatMap((w) => w.contractors)).toEqual([]); + }); + + it("returns no workstreams for a project that has selected none", () => { + expect(assembleMappingTargets([], [], []).workstreams).toEqual([]); + }); +}); + +describe("loadMappingTargets", () => { + /** A Drizzle select chain that resolves to `rows` when awaited. */ + function chainResolving(rows: unknown[]) { + const chain: Record = {}; + for (const method of ["from", "innerJoin", "where", "orderBy"]) { + chain[method] = () => chain; + } + chain.then = (resolve: (value: unknown) => unknown) => resolve(rows); + return chain; + } + + it("assembles the three read-only queries into one target set", async () => { + mockSelect + .mockReturnValueOnce(chainResolving(WORKSTREAMS)) + .mockReturnValueOnce(chainResolving(STAGES)) + .mockReturnValueOnce(chainResolving(CONTRACTORS)); + + const targets = await loadMappingTargets(42n); + + expect(mockSelect).toHaveBeenCalledTimes(3); + expect(targets.workstreams.map((w) => w.name)).toEqual(["Doors", "Windows"]); + expect( + targets.workstreams.find((w) => w.name === "Windows")!.stages.map((s) => s.id), + ).toEqual(["201", "202", "203"]); + }); + + it("returns an empty target set for a project with no workstreams", async () => { + mockSelect + .mockReturnValueOnce(chainResolving([])) + .mockReturnValueOnce(chainResolving([])) + .mockReturnValueOnce(chainResolving([])); + + expect((await loadMappingTargets(7n)).workstreams).toEqual([]); + }); +}); diff --git a/src/lib/projects/import/mappingTargets.ts b/src/lib/projects/import/mappingTargets.ts new file mode 100644 index 00000000..e3e8d1f7 --- /dev/null +++ b/src/lib/projects/import/mappingTargets.ts @@ -0,0 +1,223 @@ +/** + * Work-order import — what the file's values may be mapped *to* (issue #415). + * + * The value-mapping layer offers the project's own configuration as the target + * set: its selected workstreams (`project_workstream`), and per workstream its + * stage ladder (`project_workstream_stage`) and its assigned contractors + * (`project_workstream_contractor` → `organisation`). Nothing else is on offer + * — import never creates a workstream, a stage or a contractor (ADR-0019). + * + * Split the way `./readiness` splits, for the same reason: + * - `assembleMappingTargets` is **pure** — three flat fact lists in, one + * nested, sorted, client-safe structure out. Unit-tested with fixtures and + * no connection. + * - `loadMappingTargets` is the read-only round trip that gathers them. + * + * ## Why the ids come back as strings + * + * Every id here is a `bigserial`, which Drizzle hands back as a `bigint`, and a + * `bigint` cannot cross the server→client boundary (it is not JSON-serialisable + * and React refuses to serialise it into a client component's props). They are + * stringified once, here, so the mapping UI and the persisted mapping hold the + * same textual ids and #417 parses them back exactly once at commit. + * + * ## Population + * + * These three tables are filled in by the setup wizard (#410/#411/#413), which + * is in flight. Against a project whose setup has not run, this returns an empty + * workstream list — which is precisely the state the import page's readiness + * gate (ADR-0019) refuses to show the drop zone for, so the mapping step never + * sees it in practice. + */ + +import { db } from "@/app/db/db"; +import { asc, eq } from "drizzle-orm"; +import { + projectWorkstream, + projectWorkstreamContractor, + projectWorkstreamStage, + workstream, +} from "@/app/db/schema/projects/projects"; +import { organisation } from "@/app/db/schema/organisation"; + +/** One stage on a workstream's ladder. */ +export interface StageTarget { + /** `project_workstream_stage.id`, as a string. */ + id: string; + name: string; + /** `project_workstream_stage.order` — ascending, the first is the ladder's start. */ + order: number; +} + +/** One contractor assigned to a workstream. */ +export interface ContractorTarget { + /** + * `project_workstream_contractor.id`, as a string — the *assignment*, not the + * organisation. `work_order.project_workstream_contractor_id` points at the + * assignment, so that is what a mapped value must resolve to. + */ + id: string; + /** `organisation.id` (a uuid), carried for display and for #417's checks. */ + organisationId: string; + name: string; +} + +/** One workstream the project selected, with everything a row may map into. */ +export interface WorkstreamTarget { + /** `project_workstream.id`, as a string. */ + id: string; + /** `workstream.id` (the reference-data row), as a string. */ + workstreamId: string; + name: string; + /** Ascending by `order`; `stages[0]` is the fallback for a blank Stage cell. */ + stages: StageTarget[]; + /** Alphabetical by name. Only these may receive this workstream's work. */ + contractors: ContractorTarget[]; +} + +export interface ImportMappingTargets { + /** Alphabetical by name. */ + workstreams: WorkstreamTarget[]; +} + +export const EMPTY_MAPPING_TARGETS: ImportMappingTargets = { workstreams: [] }; + +/** Facts as the database yields them, before assembly. */ +export interface WorkstreamTargetFacts { + projectWorkstreamId: bigint; + workstreamId: bigint; + workstreamName: string; +} + +export interface StageTargetFacts { + id: bigint; + projectWorkstreamId: bigint; + name: string; + order: number; +} + +export interface ContractorTargetFacts { + id: bigint; + projectWorkstreamId: bigint; + organisationId: string; + /** `organisation.name` is nullable in the schema. */ + organisationName: string | null; +} + +/** Shown when an assigned organisation has no name recorded. */ +export const UNNAMED_CONTRACTOR = "Unnamed organisation"; + +/** + * Nest and sort the facts into the target structure, stringifying every id. + * + * Stages and contractors whose `projectWorkstreamId` names no workstream on + * this project are dropped rather than silently attached elsewhere — that can + * only happen if the caller mixes projects, and inventing a home for such a row + * would put a contractor in front of work they are not assigned to. + */ +export function assembleMappingTargets( + workstreams: WorkstreamTargetFacts[], + stages: StageTargetFacts[], + contractors: ContractorTargetFacts[], +): ImportMappingTargets { + const byId = new Map(); + for (const w of workstreams) { + byId.set(String(w.projectWorkstreamId), { + id: String(w.projectWorkstreamId), + workstreamId: String(w.workstreamId), + name: w.workstreamName, + stages: [], + contractors: [], + }); + } + + for (const stage of stages) { + byId.get(String(stage.projectWorkstreamId))?.stages.push({ + id: String(stage.id), + name: stage.name, + order: stage.order, + }); + } + + for (const contractor of contractors) { + byId.get(String(contractor.projectWorkstreamId))?.contractors.push({ + id: String(contractor.id), + organisationId: contractor.organisationId, + name: contractor.organisationName ?? UNNAMED_CONTRACTOR, + }); + } + + const sorted = [...byId.values()].sort((a, b) => + a.name.localeCompare(b.name, "en-GB"), + ); + for (const target of sorted) { + // The ladder's order is meaning, not presentation: stages[0] is the stage a + // blank Stage cell falls back to at commit (ADR-0019). + target.stages.sort((a, b) => a.order - b.order || a.name.localeCompare(b.name, "en-GB")); + target.contractors.sort((a, b) => a.name.localeCompare(b.name, "en-GB")); + } + + return { workstreams: sorted }; +} + +/** + * Gather one project's mapping targets. Read-only: three SELECTs, no writes. + * + * Three queries rather than one join: joining stages and contractors to + * workstreams together yields a stages×contractors product per workstream that + * would have to be de-duplicated in JS anyway, and the three are independent so + * they run concurrently. + */ +export async function loadMappingTargets( + projectId: bigint, +): Promise { + const [workstreams, stages, contractors] = await Promise.all([ + db + .select({ + projectWorkstreamId: projectWorkstream.id, + workstreamId: projectWorkstream.workstreamId, + workstreamName: workstream.name, + }) + .from(projectWorkstream) + .innerJoin(workstream, eq(projectWorkstream.workstreamId, workstream.id)) + .where(eq(projectWorkstream.projectId, projectId)), + + db + .select({ + id: projectWorkstreamStage.id, + projectWorkstreamId: projectWorkstreamStage.projectWorkstreamId, + name: projectWorkstreamStage.name, + order: projectWorkstreamStage.order, + }) + .from(projectWorkstreamStage) + .innerJoin( + projectWorkstream, + eq(projectWorkstreamStage.projectWorkstreamId, projectWorkstream.id), + ) + .where(eq(projectWorkstream.projectId, projectId)) + .orderBy(asc(projectWorkstreamStage.order)), + + db + .select({ + id: projectWorkstreamContractor.id, + projectWorkstreamId: projectWorkstreamContractor.projectWorkstreamId, + organisationId: projectWorkstreamContractor.organisationId, + organisationName: organisation.name, + }) + .from(projectWorkstreamContractor) + .innerJoin( + projectWorkstream, + eq( + projectWorkstreamContractor.projectWorkstreamId, + projectWorkstream.id, + ), + ) + .innerJoin( + organisation, + eq(projectWorkstreamContractor.organisationId, organisation.id), + ) + .where(eq(projectWorkstream.projectId, projectId)), + ]); + + return assembleMappingTargets(workstreams, stages, contractors); +} diff --git a/src/lib/projects/import/session.test.ts b/src/lib/projects/import/session.test.ts new file mode 100644 index 00000000..b8f2ab5f --- /dev/null +++ b/src/lib/projects/import/session.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { autoMatchColumns } from "./columnMapping"; +import { + EMPTY_VALUE_MAPPING, + setWorkstreamValue, + valueKey, +} from "./valueMapping"; +import { + buildImportSession, + IMPORT_SESSION_VERSION, + sessionMatchesFile, +} from "./session"; + +const FILE = { + filename: "stonewater_planned_works.xlsx", + headers: ["Asset Ref", "Work Type", "Contractor Name"], + rowCount: 1248, +}; + +function session() { + return buildImportSession( + FILE, + autoMatchColumns(FILE.headers), + setWorkstreamValue(EMPTY_VALUE_MAPPING, valueKey("Window Programme"), "2"), + ); +} + +describe("buildImportSession", () => { + it("carries the file's identity and both mapping layers", () => { + const built = session(); + + expect(built.version).toBe(IMPORT_SESSION_VERSION); + expect(built.filename).toBe(FILE.filename); + expect(built.rowCount).toBe(1248); + expect(built.columnMapping.fields.workstream).toBe(1); + expect(built.valueMapping.workstreams["window programme"]).toBe("2"); + }); + + it("does not carry the rows — they stay where the parse step put them", () => { + expect(session()).not.toHaveProperty("rows"); + }); + + it("survives a JSON round trip unchanged, so it can be POSTed at commit", () => { + const built = session(); + expect(JSON.parse(JSON.stringify(built))).toEqual(built); + }); +}); + +describe("sessionMatchesFile", () => { + it("accepts the file it was built against", () => { + expect(sessionMatchesFile(session(), FILE)).toBe(true); + }); + + it("rejects a file whose headings moved, since the mapping is column indices", () => { + expect( + sessionMatchesFile(session(), { + headers: ["Work Type", "Asset Ref", "Contractor Name"], + }), + ).toBe(false); + }); + + it("rejects a file with a different number of columns", () => { + expect( + sessionMatchesFile(session(), { headers: ["Asset Ref", "Work Type"] }), + ).toBe(false); + }); + + it("rejects a session written by a different version of the shape", () => { + const stale = { ...session(), version: 0 as typeof IMPORT_SESSION_VERSION }; + expect(sessionMatchesFile(stale, FILE)).toBe(false); + }); +}); diff --git a/src/lib/projects/import/session.ts b/src/lib/projects/import/session.ts new file mode 100644 index 00000000..e401f062 --- /dev/null +++ b/src/lib/projects/import/session.ts @@ -0,0 +1,88 @@ +/** + * Work-order import — the import session (issue #415). + * + * The session is everything the wizard knows about one import in progress: the + * file it parsed, and the two mapping layers the user declared over it. It is + * the artefact the ticket means by "persist the mapping alongside the import + * session". + * + * ## Where it lives + * + * Client-side, in the import screen's React state — the same place #414 put the + * parsed rows, and for the same reason: nothing about an import is written + * until commit (#417). There is no `import_session` table to hang the mapping + * off; adding one is a migration, and the mapping alone is not worth a table + * that the file's rows would still not fit in. So "alongside the session" is + * taken literally: `ImportSession` carries the mappings in the same object as + * the file identity, and that object is what #417 POSTs at commit. See + * ADR-0020 for the trade-off and what would change the answer. + * + * The shape is therefore deliberately **JSON-serialisable end to end** — plain + * objects, string ids, no `bigint`, no `Date`, no class instances — so it can + * be POSTed verbatim, and so a future server-side session can persist it as a + * JSON column without reshaping — which is how + * `bulk_address_uploads.column_mapping` already stores the older BulkUpload + * journey's equivalent. + */ + +import type { ImportColumnMapping } from "./columnMapping"; +import type { ImportValueMapping } from "./valueMapping"; + +/** + * Bumped whenever the persisted shape changes meaning, so a session written by + * an older client can be recognised rather than silently misread. + */ +export const IMPORT_SESSION_VERSION = 1; + +export interface ImportSession { + version: typeof IMPORT_SESSION_VERSION; + /** The uploaded file's name, as the user sees it in the header. */ + filename: string; + /** The file's headings, in file order — what the column mapping's indices point into. */ + headers: string[]; + /** Data rows in the file, for the counts the summary shows. */ + rowCount: number; + /** Which column is which field. */ + columnMapping: ImportColumnMapping; + /** Which of those columns' values are which of the project's entities. */ + valueMapping: ImportValueMapping; +} + +/** + * Assemble the session. Takes the parsed file's identity rather than its rows: + * the rows stay where the parse step put them, and re-sending 20,000 of them + * with every mapping keystroke is exactly what holding them client-side avoids. + */ +export function buildImportSession( + file: { filename: string; headers: string[]; rowCount: number }, + columnMapping: ImportColumnMapping, + valueMapping: ImportValueMapping, +): ImportSession { + return { + version: IMPORT_SESSION_VERSION, + filename: file.filename, + headers: file.headers, + rowCount: file.rowCount, + columnMapping, + valueMapping, + }; +} + +/** + * True when this session was written against the file currently in hand. + * + * A mapping is a set of *column indices*, so it is meaningless against a + * different file — restoring one over a new upload would silently map the wrong + * columns. Headers are compared in order, which is the only thing the indices + * depend on. + */ +export function sessionMatchesFile( + session: ImportSession, + file: { headers: string[] }, +): boolean { + return ( + session.version === IMPORT_SESSION_VERSION && + session.headers.length === file.headers.length && + session.headers.every((header, i) => header === file.headers[i]) + ); +} diff --git a/src/lib/projects/import/similarity.test.ts b/src/lib/projects/import/similarity.test.ts new file mode 100644 index 00000000..49a41a54 --- /dev/null +++ b/src/lib/projects/import/similarity.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { + AUTO_MATCH_THRESHOLD, + bestSimilarity, + greedyPairs, + normaliseForMatch, + similarity, + VALUE_MATCH_THRESHOLD, + valueSimilarity, +} from "./similarity"; + +describe("normaliseForMatch", () => { + it("lowercases, turns punctuation into spaces and collapses whitespace", () => { + expect(normaliseForMatch(" Work_Order / REFERENCE ")).toBe( + "work order reference", + ); + }); + + it("collapses a value that is only punctuation to the empty string", () => { + expect(normaliseForMatch(" -- ")).toBe(""); + }); +}); + +describe("similarity", () => { + it("scores an exact match, ignoring case and punctuation, as 1", () => { + expect(similarity("Work Type", "work_type")).toBe(1); + }); + + it("scores a blank against anything as 0", () => { + expect(similarity("", "Workstream")).toBe(0); + expect(similarity("—", "Workstream")).toBe(0); + }); + + it("ranks word containment above letter overlap", () => { + // "Contractor" ⊂ "Contractor Name" is a much stronger signal than the + // bigram overlap of "Stage"/"Storage", and must outrank it. + expect(similarity("Contractor", "Contractor Name")).toBe(0.9); + expect(similarity("Stage", "Storage")).toBeLessThan(0.9); + }); + + it("scores near-misses between 0 and 1", () => { + const score = similarity("Forecast End", "Forecast end date"); + expect(score).toBeGreaterThan(AUTO_MATCH_THRESHOLD); + expect(score).toBeLessThan(1); + }); + + it("keeps unrelated headings below the auto-match threshold", () => { + expect(similarity("Postcode", "Contractor")).toBeLessThan( + AUTO_MATCH_THRESHOLD, + ); + expect(similarity("Priority", "Workstream")).toBeLessThan( + AUTO_MATCH_THRESHOLD, + ); + // The one that matters most: a Status column is not a Stage column. + expect(similarity("Status", "Stage")).toBeLessThan(AUTO_MATCH_THRESHOLD); + }); +}); + +describe("valueSimilarity", () => { + it("matches a project's name buried in a longer file value", () => { + // The cases the wireframe shows: the file's vocabulary is wordier than the + // project's, and plain bigram overlap punishes it for the extra words. + expect(valueSimilarity("Window Programme", "Windows")).toBeGreaterThan( + VALUE_MATCH_THRESHOLD, + ); + expect(valueSimilarity("Door Replacement", "Doors")).toBeGreaterThan( + VALUE_MATCH_THRESHOLD, + ); + // This one the heading rule already handles — "Decorations" is a whole word + // of the file's value — so it keeps the word-containment score. + expect(valueSimilarity("Cyclical Decorations", "Decorations")).toBe(0.9); + }); + + it("refuses two contractor names that share only their filler words", () => { + expect( + valueSimilarity("Acme Windows Ltd", "Acme Doors Ltd"), + ).toBeLessThan(VALUE_MATCH_THRESHOLD); + expect( + valueSimilarity("Someone Else Ltd", "Acme Windows Ltd"), + ).toBeLessThan(VALUE_MATCH_THRESHOLD); + expect( + valueSimilarity("Sunrise Glazing", "Sunrise Roofing Ltd"), + ).toBeLessThan(VALUE_MATCH_THRESHOLD); + }); + + it("is never stricter than the heading rule it builds on", () => { + for (const [a, b] of [ + ["Windows", "Windows"], + ["Contractor", "Contractor Name"], + ["Stage", "Storage"], + ]) { + expect(valueSimilarity(a, b)).toBeGreaterThanOrEqual(similarity(a, b)); + } + }); +}); + +describe("bestSimilarity", () => { + it("takes the best score across a field's label and aliases", () => { + expect(bestSimilarity("Asset Ref", ["Landlord property id", "asset ref"])).toBe(1); + }); + + it("is 0 when there are no phrases to score against", () => { + expect(bestSimilarity("Anything", [])).toBe(0); + }); +}); + +describe("greedyPairs", () => { + const score = (a: string, b: string) => similarity(a, b); + + it("gives each target to its best candidate, one apiece", () => { + const pairs = greedyPairs( + ["Property Ref", "UPRN"], + ["Property reference", "UPRN"], + score, + ); + + expect(pairs).toHaveLength(2); + expect(pairs.map((p) => [p.candidateIndex, p.targetIndex])).toEqual( + expect.arrayContaining([ + [0, 0], + [1, 1], + ]), + ); + }); + + it("does not let one candidate take two targets, or one target two candidates", () => { + // Both headings look like the same target; only the better one may have it. + const pairs = greedyPairs( + ["Workstream", "Workstream 2"], + ["Workstream"], + score, + ); + + expect(pairs).toHaveLength(1); + expect(pairs[0].candidateIndex).toBe(0); + }); + + it("drops pairs below the threshold rather than matching them weakly", () => { + expect(greedyPairs(["Postcode"], ["Contractor"], score)).toEqual([]); + }); + + it("breaks ties on candidate order, so a given file always maps the same way", () => { + const pairs = greedyPairs(["Stage", "Stage"], ["Stage"], score); + expect(pairs).toEqual([{ candidateIndex: 0, targetIndex: 0, score: 1 }]); + }); +}); diff --git a/src/lib/projects/import/similarity.ts b/src/lib/projects/import/similarity.ts new file mode 100644 index 00000000..b14752cc --- /dev/null +++ b/src/lib/projects/import/similarity.ts @@ -0,0 +1,226 @@ +/** + * Header/value similarity — the auto-match primitive shared by both mapping + * layers of the work-order import (issue #415). + * + * Column mapping matches a file *heading* to an Ara field; value mapping + * matches a file *cell value* to one of the project's workstreams, contractors + * or stages. Both are "the user typed something close to what we call it", so + * both score the same way rather than each growing its own fuzzy matcher. + * + * Pure and dependency-free: no database, no React, no I/O. + */ + +/** + * Fold a heading or value to its comparable form: lowercase, punctuation and + * underscores to spaces, runs of whitespace collapsed. + * + * Deliberately *not* stemming or dropping stop-words — "Work order reference" + * and "Work order number" must stay distinguishable from each other, and a + * stemmer would need a dictionary this module has no business carrying. + */ +export function normaliseForMatch(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +/** The normalised form's words, empty array for a blank input. */ +function tokens(normalised: string): string[] { + return normalised === "" ? [] : normalised.split(" "); +} + +/** Character bigrams of a normalised string with its spaces removed. */ +function bigrams(normalised: string): string[] { + const compact = normalised.replace(/ /g, ""); + const out: string[] = []; + for (let i = 0; i < compact.length - 1; i++) out.push(compact.slice(i, i + 2)); + return out; +} + +/** + * Sørensen–Dice coefficient over character bigrams, in `[0, 1]`. + * + * Chosen over Levenshtein because it is length-insensitive in the way headings + * need: "WO Ref" vs "Work order reference" is a *containment* relationship that + * an edit-distance ratio punishes for the length difference alone. + */ +function diceCoefficient(a: string, b: string): number { + const left = bigrams(a); + const right = bigrams(b); + if (left.length === 0 || right.length === 0) return a === b ? 1 : 0; + + // Multiset intersection: each bigram in `left` consumes one occurrence in + // `right`, so a repeated bigram cannot be matched twice. + const counts = new Map(); + for (const gram of right) counts.set(gram, (counts.get(gram) ?? 0) + 1); + + let shared = 0; + for (const gram of left) { + const remaining = counts.get(gram) ?? 0; + if (remaining > 0) { + shared++; + counts.set(gram, remaining - 1); + } + } + + return (2 * shared) / (left.length + right.length); +} + +/** True when every word of the shorter phrase appears in the longer one. */ +function isTokenSubset(a: string[], b: string[]): boolean { + if (a.length === 0 || b.length === 0) return false; + const [short, long] = a.length <= b.length ? [a, b] : [b, a]; + const pool = new Set(long); + return short.every((token) => pool.has(token)); +} + +/** + * How alike two headings/values are, in `[0, 1]`. + * + * Three tiers, highest first: + * 1. `1` — identical once normalised ("Work Type" vs "work type"). + * 2. `0.9` — one phrase's words are a subset of the other's ("Contractor" vs + * "Contractor Name"). Ranked above any Dice score because word containment + * is a much stronger signal than shared letters: "Stage" and "Storage" + * score well on bigrams and mean nothing to each other. + * 3. the Dice coefficient — everything else, including near-misses and + * typos. + */ +export function similarity(a: string, b: string): number { + const left = normaliseForMatch(a); + const right = normaliseForMatch(b); + if (left === "" || right === "") return 0; + if (left === right) return 1; + if (isTokenSubset(tokens(left), tokens(right))) return 0.9; + return diceCoefficient(left, right); +} + +/** + * How much of `from`'s text finds a counterpart in `to`, weighted by word + * length so a long distinguishing word counts for more than a short filler one. + * + * Directional on purpose — see `valueSimilarity`. + */ +function tokenCoverage(from: string[], to: string[]): number { + let matched = 0; + let total = 0; + for (const token of from) { + total += token.length; + let best = 0; + for (const other of to) { + const score = token === other ? 1 : diceCoefficient(token, other); + if (score > best) best = score; + } + matched += token.length * best; + } + return total === 0 ? 0 : matched / total; +} + +/** + * Similarity for *cell values*, which are messier than headings. + * + * A file's workstream column says "Window Programme" where the project calls it + * "Windows", and "Door Replacement" where it says "Doors" — the project's name + * is one word of a longer phrase, in a form `similarity` scores at 0.5 because + * the extra words dilute the bigram overlap. So this takes the best *directional* + * word coverage as well: "Windows" is almost entirely present in "Window + * Programme" (0.91) even though the reverse is not true (0.36). + * + * Kept apart from `similarity` rather than folded into it, because the same + * leniency is wrong for headings: "Forecast Start" would cover "Forecast end + * date" well enough to be auto-mapped to the wrong field, and a heading mapped + * to the wrong field mis-reads every row in the file. + */ +export function valueSimilarity(a: string, b: string): number { + const base = similarity(a, b); + if (base >= 0.9) return base; + const left = tokens(normaliseForMatch(a)); + const right = tokens(normaliseForMatch(b)); + return Math.max(base, tokenCoverage(left, right), tokenCoverage(right, left)); +} + +/** + * The bar a *value* auto-match must clear — higher than the heading bar, + * because the coverage rule above is more generous and because contractor names + * share filler words ("Ltd", "Group", "Services") that must not be enough on + * their own. "Acme Windows Ltd" vs "Acme Doors Ltd" scores 0.67 and is + * correctly refused. + */ +export const VALUE_MATCH_THRESHOLD = 0.75; + +/** + * The best score of `candidate` against any of `phrases` — a field's label plus + * its aliases, or a target's name. + */ +export function bestSimilarity( + candidate: string, + phrases: readonly string[], +): number { + let best = 0; + for (const phrase of phrases) { + const score = similarity(candidate, phrase); + if (score > best) best = score; + } + return best; +} + +/** + * The bar an auto-match must clear. + * + * 0.62 sits above the noise floor of unrelated four-to-six-letter headings + * (which land around 0.3–0.5 on Dice) and below "Forecast End" vs "Forecast end + * date" (0.83). A wrong auto-match is worse than none — an unmatched column + * shows as *needs review* and asks the user, whereas a confidently wrong one + * has to be *noticed* first. + */ +export const AUTO_MATCH_THRESHOLD = 0.62; + +/** + * Greedily pair candidates to targets, best score first, one target per + * candidate and one candidate per target. + * + * Greedy rather than optimal (Hungarian) assignment: the score matrix is tiny, + * the user can override any row, and "the best match wins first" is the rule a + * person would predict when they look at the result. Ties break on candidate + * order then target order, so the outcome is stable for a given file. + */ +export function greedyPairs( + candidates: readonly C[], + targets: readonly T[], + score: (candidate: C, target: T) => number, + threshold: number = AUTO_MATCH_THRESHOLD, +): Array<{ candidateIndex: number; targetIndex: number; score: number }> { + const scored: Array<{ + candidateIndex: number; + targetIndex: number; + score: number; + }> = []; + + candidates.forEach((candidate, candidateIndex) => { + targets.forEach((target, targetIndex) => { + const value = score(candidate, target); + if (value >= threshold) scored.push({ candidateIndex, targetIndex, score: value }); + }); + }); + + scored.sort( + (a, b) => + b.score - a.score || + a.candidateIndex - b.candidateIndex || + a.targetIndex - b.targetIndex, + ); + + const usedCandidates = new Set(); + const usedTargets = new Set(); + const pairs: typeof scored = []; + for (const pair of scored) { + if (usedCandidates.has(pair.candidateIndex)) continue; + if (usedTargets.has(pair.targetIndex)) continue; + usedCandidates.add(pair.candidateIndex); + usedTargets.add(pair.targetIndex); + pairs.push(pair); + } + return pairs; +} diff --git a/src/lib/projects/import/valueMapping.test.ts b/src/lib/projects/import/valueMapping.test.ts new file mode 100644 index 00000000..ea2f9d8d --- /dev/null +++ b/src/lib/projects/import/valueMapping.test.ts @@ -0,0 +1,355 @@ +import { describe, expect, it } from "vitest"; +import { autoMatchColumns } from "./columnMapping"; +import type { ImportMappingTargets } from "./mappingTargets"; +import { + buildValueMappingModel, + EMPTY_VALUE_MAPPING, + scopedValueKey, + setContractorValue, + setStageValue, + setWorkstreamValue, + valueKey, +} from "./valueMapping"; + +const HEADERS = ["Property ID", "Work Type", "Contractor", "Stage"]; +const COLUMNS = autoMatchColumns(HEADERS); + +/** + * Windows has two contractors and a three-stage ladder; Doors has one + * contractor that is *also* called Acme — the case the scoping rule exists for. + */ +const TARGETS: ImportMappingTargets = { + workstreams: [ + { + id: "1", + workstreamId: "10", + name: "Doors", + stages: [ + { id: "301", name: "Ordered", order: 1 }, + { id: "302", name: "Completed", order: 2 }, + ], + contractors: [{ id: "401", organisationId: "org-c", name: "Acme Doors Ltd" }], + }, + { + id: "2", + workstreamId: "11", + name: "Windows", + stages: [ + { id: "201", name: "Ordered", order: 1 }, + { id: "202", name: "In progress", order: 2 }, + { id: "203", name: "Completed", order: 3 }, + ], + contractors: [ + { id: "402", organisationId: "org-a", name: "Acme Windows Ltd" }, + { id: "403", organisationId: "org-b", name: "Sunrise Glazing" }, + ], + }, + ], +}; + +function model( + rows: string[][], + valueMapping = EMPTY_VALUE_MAPPING, + columns = COLUMNS, +) { + return buildValueMappingModel(rows, columns, valueMapping, TARGETS); +} + +describe("buildValueMappingModel — workstream values", () => { + it("lists distinct values with their row counts, biggest first", () => { + const result = model([ + ["P1", "Windows", "Acme Windows Ltd", "Ordered"], + ["P2", "Windows", "Acme Windows Ltd", "Ordered"], + ["P3", "Doors", "Acme Doors Ltd", "Ordered"], + ]); + + expect(result.workstreams.map((e) => [e.value, e.rowCount])).toEqual([ + ["Windows", 2], + ["Doors", 1], + ]); + }); + + it("auto-matches a value that is close to a workstream's name", () => { + const result = model([["P1", "Window Programme", "Acme Windows Ltd", ""]]); + + expect(result.workstreams[0]).toMatchObject({ + targetId: "2", + matchedBy: "auto", + status: "matched", + }); + }); + + it("treats differently-cased spellings of one value as one decision", () => { + const result = model([ + ["P1", "WINDOWS", "Acme Windows Ltd", ""], + ["P2", "windows", "Acme Windows Ltd", ""], + ]); + + expect(result.workstreams).toHaveLength(1); + expect(result.workstreams[0].rowCount).toBe(2); + }); + + it("leaves a value matching nothing as needs review", () => { + const result = model([["P1", "Bathroom Programme", "Acme Doors Ltd", ""]]); + + expect(result.workstreams[0]).toMatchObject({ + targetId: null, + status: "needs_review", + }); + expect(result.needsReview.workstreams).toBe(1); + }); + + it("prefers the user's choice over the auto-match, and records that it did", () => { + const rows = [["P1", "Window Programme", "Acme Windows Ltd", ""]]; + const chosen = setWorkstreamValue( + EMPTY_VALUE_MAPPING, + valueKey("Window Programme"), + "1", + ); + + expect(model(rows, chosen).workstreams[0]).toMatchObject({ + targetId: "1", + matchedBy: "user", + }); + }); + + it("does not put the suggestion back when the user clears a value", () => { + const rows = [["P1", "Windows", "Acme Windows Ltd", ""]]; + const cleared = setWorkstreamValue(EMPTY_VALUE_MAPPING, valueKey("Windows"), null); + + expect(model(rows, cleared).workstreams[0]).toMatchObject({ + targetId: null, + matchedBy: null, + status: "needs_review", + }); + }); + + it("counts blank workstream cells rather than listing them as a value", () => { + const result = model([ + ["P1", "", "Acme Windows Ltd", ""], + ["P2", "Windows", "Acme Windows Ltd", ""], + ]); + + expect(result.workstreams).toHaveLength(1); + expect(result.blankRows.workstream).toBe(1); + }); +}); + +describe("buildValueMappingModel — contractor values are scoped to the workstream", () => { + it("offers only the contractors assigned to that row's workstream", () => { + // "Acme" reads as Acme Windows under Windows and Acme Doors under Doors. + const result = model([ + ["P1", "Windows", "Acme Windows Ltd", ""], + ["P2", "Doors", "Acme Doors Ltd", ""], + ]); + + expect(result.contractors).toHaveLength(2); + const windows = result.contractors.find((e) => e.workstreamName === "Windows"); + const doors = result.contractors.find((e) => e.workstreamName === "Doors"); + expect(windows).toMatchObject({ targetId: "402", status: "matched" }); + expect(doors).toMatchObject({ targetId: "401", status: "matched" }); + }); + + it("keeps the same contractor text as two decisions under two workstreams", () => { + const result = model([ + ["P1", "Windows", "Sunrise Glazing", ""], + ["P2", "Doors", "Sunrise Glazing", ""], + ]); + + expect(result.contractors).toHaveLength(2); + // Assigned to Windows only, so under Doors it resolves to nothing. + expect( + result.contractors.find((e) => e.workstreamName === "Windows"), + ).toMatchObject({ targetId: "403", status: "matched" }); + expect( + result.contractors.find((e) => e.workstreamName === "Doors"), + ).toMatchObject({ targetId: null, status: "needs_review" }); + }); + + it("merges two spellings of one workstream into a single contractor decision", () => { + const result = model([ + ["P1", "Windows", "Sunrise Glazing", ""], + ["P2", "Window Programme", "Sunrise Glazing", ""], + ]); + + expect(result.workstreams).toHaveLength(2); + // Both spellings are the Windows workstream, so "Sunrise Glazing" beneath + // them is one question, asked once, over two rows. + expect(result.contractors).toHaveLength(1); + expect(result.contractors[0].rowCount).toBe(2); + }); + + it("does not list contractor values whose workstream is not mapped yet", () => { + const result = model([ + ["P1", "Bathroom Programme", "Some Bathroom Co", ""], + ["P2", "Windows", "Sunrise Glazing", ""], + ]); + + expect(result.contractors.map((e) => e.value)).toEqual(["Sunrise Glazing"]); + expect(result.unscopedRows).toBe(1); + }); + + it("surfaces them as soon as the workstream is mapped", () => { + const rows = [["P1", "Bathroom Programme", "Sunrise Glazing", ""]]; + const mapped = setWorkstreamValue( + EMPTY_VALUE_MAPPING, + valueKey("Bathroom Programme"), + "2", + ); + + const result = model(rows, mapped); + expect(result.unscopedRows).toBe(0); + expect(result.contractors[0]).toMatchObject({ + value: "Sunrise Glazing", + targetId: "403", + }); + }); + + it("counts a blank contractor rather than listing it", () => { + const result = model([["P1", "Windows", "", ""]]); + + expect(result.contractors).toEqual([]); + expect(result.blankRows.contractor).toBe(1); + }); + + it("drops a user's choice that the workstream no longer offers", () => { + // Mapped Acme Doors under Doors, then re-pointed the value at Windows — + // assignment 401 is not a Windows contractor, so the answer lapses. + const rows = [["P1", "Doors", "Acme Doors Ltd", ""]]; + const stale = setContractorValue( + setWorkstreamValue(EMPTY_VALUE_MAPPING, valueKey("Doors"), "2"), + scopedValueKey("2", "Acme Doors Ltd"), + "401", + ); + + expect(model(rows, stale).contractors[0]).toMatchObject({ + targetId: null, + status: "needs_review", + }); + }); + + it("flags a workstream with no contractor assigned at all", () => { + const barren: ImportMappingTargets = { + workstreams: [ + { id: "9", workstreamId: "19", name: "Roofs", stages: [], contractors: [] }, + ], + }; + const result = buildValueMappingModel( + [["P1", "Roofs", "Any Co", "Ordered"]], + COLUMNS, + EMPTY_VALUE_MAPPING, + barren, + ); + + expect(result.contractors[0].noTargets).toBe(true); + expect(result.stages[0].noTargets).toBe(true); + }); +}); + +describe("buildValueMappingModel — stage values", () => { + it("matches a stage against that workstream's own ladder", () => { + const result = model([["P1", "Windows", "Sunrise Glazing", "In progress"]]); + + expect(result.stages[0]).toMatchObject({ + targetId: "202", + status: "matched", + }); + }); + + it("does not offer another workstream's stage", () => { + // "In progress" is on the Windows ladder, not the Doors one. + const result = model([["P1", "Doors", "Acme Doors Ltd", "In progress"]]); + + expect(result.stages[0]).toMatchObject({ targetId: null, status: "needs_review" }); + }); + + it("leaves blank stages out of the table — they are legitimate", () => { + const result = model([ + ["P1", "Windows", "Sunrise Glazing", ""], + ["P2", "Windows", "Sunrise Glazing", "Ordered"], + ]); + + expect(result.stages).toHaveLength(1); + expect(result.blankRows.stage).toBe(1); + }); + + it("accepts the user's own stage choice", () => { + const rows = [["P1", "Windows", "Sunrise Glazing", "Fitted"]]; + const chosen = setStageValue( + EMPTY_VALUE_MAPPING, + scopedValueKey("2", "Fitted"), + "203", + ); + + expect(model(rows, chosen).stages[0]).toMatchObject({ + targetId: "203", + matchedBy: "user", + }); + }); +}); + +describe("buildValueMappingModel — warning rows", () => { + it("is zero when everything resolves", () => { + const result = model([ + ["P1", "Windows", "Sunrise Glazing", "Ordered"], + ["P2", "Doors", "Acme Doors Ltd", ""], + ]); + + expect(result.warningRows).toBe(0); + }); + + it("counts a row once even when several of its values are broken", () => { + // Unmapped workstream *and* an unmappable contractor: one broken row. + const result = model([["P1", "Bathroom Programme", "", ""]]); + expect(result.warningRows).toBe(1); + }); + + it("counts blank and unmapped contractors", () => { + const result = model([ + ["P1", "Windows", "", ""], + ["P2", "Windows", "Someone Else Ltd", ""], + ["P3", "Windows", "Sunrise Glazing", ""], + ]); + + expect(result.warningRows).toBe(2); + }); + + it("counts a provided-but-unmatched stage, but never a blank one", () => { + const result = model([ + ["P1", "Windows", "Sunrise Glazing", ""], + ["P2", "Windows", "Sunrise Glazing", "Snagging"], + ]); + + expect(result.warningRows).toBe(1); + }); +}); + +describe("buildValueMappingModel — unmapped columns", () => { + it("reports which value columns exist at all", () => { + const noStage = autoMatchColumns(["Property ID", "Work Type", "Contractor"]); + const result = buildValueMappingModel( + [["P1", "Windows", "Sunrise Glazing"]], + noStage, + EMPTY_VALUE_MAPPING, + TARGETS, + ); + + expect(result.hasWorkstreamColumn).toBe(true); + expect(result.hasContractorColumn).toBe(true); + expect(result.hasStageColumn).toBe(false); + expect(result.stages).toEqual([]); + }); + + it("lists nothing at all when the workstream column is unmapped", () => { + const result = buildValueMappingModel( + [["P1", "Windows", "Sunrise Glazing", "Ordered"]], + { fields: {}, ignoredColumns: [] }, + EMPTY_VALUE_MAPPING, + TARGETS, + ); + + expect(result.workstreams).toEqual([]); + expect(result.contractors).toEqual([]); + expect(result.blankRows.workstream).toBe(1); + }); +}); diff --git a/src/lib/projects/import/valueMapping.ts b/src/lib/projects/import/valueMapping.ts new file mode 100644 index 00000000..0c055bff --- /dev/null +++ b/src/lib/projects/import/valueMapping.ts @@ -0,0 +1,488 @@ +/** + * Work-order import — value mapping (issue #415, step 2 of 3, layer 2 of 2). + * + * Layer 1 (`./columnMapping`) said which column is the workstream. This layer + * says which of *that column's values* is which of the project's workstreams — + * and then, scoped to each of those, which contractor and which stage. + * + * ## The scoping rule (ADR-0019) + * + * A contractor value does not map to "a contractor"; it maps to a contractor + * **assigned to that row's workstream** (`project_workstream_contractor`), and + * a stage maps to a stage on **that workstream's own ladder** + * (`project_workstream_stage`). The same text can therefore need two different + * answers — "Acme" under Windows is a different assignment from "Acme" under + * Doors, and may not exist at all under one of them. So contractor and stage + * entries are keyed by `(resolved workstream, value)`, never by value alone, + * and each offers only that workstream's assignments. + * + * A consequence worth stating: rows whose workstream value is blank or not yet + * mapped have no scope, so their contractor and stage values cannot be listed + * yet. They are counted (`unscopedRows`) rather than guessed at, and appear as + * soon as their workstream is mapped — which is why the whole model is derived + * fresh from `(rows, columnMapping, valueMapping, targets)` on every change + * instead of being accumulated in state. + * + * ## One-to-one vs many-to-one + * + * Column mapping is one-to-one (one column per field), so it assigns greedily. + * Value mapping is deliberately **many-to-one**: "Window Programme" and + * "Windows - Phase 2" may both be the Windows workstream, so each distinct + * value takes its own best match independently. + * + * ## What blocks what + * + * Nothing here blocks progression to validation — that is column mapping's job. + * An unmapped value is a *warning* whose rows become validation errors at #417, + * except a blank Stage, which is legitimate and falls back to the workstream's + * first stage at commit. + * + * Pure and DB-free. + */ + +import { + normaliseForMatch, + VALUE_MATCH_THRESHOLD, + valueSimilarity, +} from "./similarity"; +import { cellFor, type ImportColumnMapping } from "./columnMapping"; +import type { ImportMappingTargets, WorkstreamTarget } from "./mappingTargets"; + +/** + * The user's value-level decisions. + * + * A key that is **absent** has never been touched, so auto-match speaks for it. + * A key mapped to **null** was explicitly cleared, and auto-match must not + * quietly put its suggestion back — "I looked at this and none of them is + * right" is an answer. + * + * Contractor and stage keys carry their workstream scope (see `scopedValueKey`). + */ +export interface ImportValueMapping { + /** value key → `project_workstream.id`. */ + workstreams: Record; + /** scoped key → `project_workstream_contractor.id`. */ + contractors: Record; + /** scoped key → `project_workstream_stage.id`. */ + stages: Record; +} + +export const EMPTY_VALUE_MAPPING: ImportValueMapping = { + workstreams: {}, + contractors: {}, + stages: {}, +}; + +/** + * The key a file value is remembered by: its normalised form, so "ACME LTD", + * "Acme Ltd" and "acme ltd" are one decision rather than three. + */ +export function valueKey(value: string): string { + return normaliseForMatch(value); +} + +/** The key a contractor/stage value is remembered by, within one workstream. */ +export function scopedValueKey( + projectWorkstreamId: string, + value: string, +): string { + // The workstream id is all digits (a `bigserial`), so the first space is + // unambiguously the separator however many spaces the value itself carries. + return `${projectWorkstreamId} ${valueKey(value)}`; +} + +export type ValueMappingStatus = "matched" | "needs_review"; + +/** Where a mapped value's answer came from — the UI badges auto-matches. */ +export type ValueMatchSource = "user" | "auto" | null; + +export interface WorkstreamValueEntry { + /** The value as it first appears in the file, for display. */ + value: string; + /** Its `valueKey`, for reading and writing `ImportValueMapping.workstreams`. */ + key: string; + /** How many file rows carry this value. */ + rowCount: number; + /** `project_workstream.id`, or null when unmapped. */ + targetId: string | null; + matchedBy: ValueMatchSource; + status: ValueMappingStatus; +} + +export interface ScopedValueEntry { + /** The workstream this decision is scoped to — the row's mapped workstream. */ + workstreamId: string; + workstreamName: string; + value: string; + /** Its `scopedValueKey`, for reading and writing the mapping. */ + key: string; + rowCount: number; + /** `project_workstream_contractor.id` / `project_workstream_stage.id`, or null. */ + targetId: string | null; + matchedBy: ValueMatchSource; + status: ValueMappingStatus; + /** + * True when the workstream this value sits under has nothing to offer — no + * contractor assigned, or no stage on its ladder. The readiness gate + * (ADR-0019) makes this unreachable from the import page, but a caller that + * skipped the gate would otherwise show an empty picker with no explanation. + */ + noTargets: boolean; +} + +export interface ValueMappingModel { + /** Distinct workstream values, biggest row count first. */ + workstreams: WorkstreamValueEntry[]; + /** Distinct (workstream, contractor value) pairs, biggest row count first. */ + contractors: ScopedValueEntry[]; + /** Distinct (workstream, stage value) pairs. Blank stage cells are not listed. */ + stages: ScopedValueEntry[]; + /** Whether the column behind each value table is mapped at all. */ + hasWorkstreamColumn: boolean; + hasContractorColumn: boolean; + hasStageColumn: boolean; + /** Rows whose cell is blank, per field. A blank stage is legitimate; the others are not. */ + blankRows: { workstream: number; contractor: number; stage: number }; + /** + * Rows whose workstream value is blank or unmapped, so their contractor and + * stage values cannot be scoped and are not listed yet. + */ + unscopedRows: number; + /** Entries still needing review, per table. */ + needsReview: { workstreams: number; contractors: number; stages: number }; + /** + * Rows that will fail validation at #417 as things stand: a blank or unmapped + * workstream, a blank or unmapped contractor, or a *provided* stage that + * matches nothing. Counted per row, so a row with two problems counts once. + */ + warningRows: number; +} + +/** Row-level view of one file row's three mappable values. */ +interface RowValues { + workstream: string; + contractor: string; + stage: string; +} + +/** + * Build the whole value-mapping picture from the file, the column mapping, the + * user's value decisions and the project's targets. + * + * One pass over the rows to count distinct values, then a second, tiny pass + * over the distinct sets to resolve each. It is re-run on every edit — the + * inputs are small enough (20,000 rows at most, three cells each) that deriving + * beats caching, and caching is what would let the tables disagree with the + * mapping the user is looking at. + */ +export function buildValueMappingModel( + rows: string[][], + columnMapping: ImportColumnMapping, + valueMapping: ImportValueMapping, + targets: ImportMappingTargets, +): ValueMappingModel { + const hasWorkstreamColumn = columnMapping.fields.workstream !== undefined; + const hasContractorColumn = columnMapping.fields.contractor !== undefined; + const hasStageColumn = columnMapping.fields.stage !== undefined; + + const blankRows = { workstream: 0, contractor: 0, stage: 0 }; + + // Pass 1 — distinct workstream values, and the contractor/stage values seen + // beneath each of them (keyed by the *file's* workstream value, since the + // workstream it resolves to is not known until pass 2). + const workstreamValues = new Map(); + const rowsByWorkstreamKey = new Map(); + + for (const row of rows) { + const values: RowValues = { + workstream: cellFor(row, columnMapping, "workstream"), + contractor: cellFor(row, columnMapping, "contractor"), + stage: cellFor(row, columnMapping, "stage"), + }; + + if (values.workstream === "") blankRows.workstream++; + if (values.contractor === "") blankRows.contractor++; + if (values.stage === "") blankRows.stage++; + + if (values.workstream === "") continue; + + const key = valueKey(values.workstream); + const seen = workstreamValues.get(key); + if (seen) seen.rowCount++; + else workstreamValues.set(key, { value: values.workstream, rowCount: 1 }); + + const bucket = rowsByWorkstreamKey.get(key); + if (bucket) bucket.push(values); + else rowsByWorkstreamKey.set(key, [values]); + } + + // Pass 2 — resolve each distinct workstream value. + const workstreams: WorkstreamValueEntry[] = []; + const resolvedByKey = new Map(); + const targetsById = new Map(targets.workstreams.map((w) => [w.id, w])); + + for (const [key, { value, rowCount }] of workstreamValues) { + const resolved = resolveValue( + key, + value, + valueMapping.workstreams, + targets.workstreams, + (target) => target.name, + (target) => target.id, + ); + workstreams.push({ value, key, rowCount, ...resolved }); + const target = resolved.targetId + ? targetsById.get(resolved.targetId) + : undefined; + if (target) resolvedByKey.set(key, target); + } + + // Pass 3 — contractor and stage values, grouped by the workstream their rows + // resolved to. Two file spellings of the same workstream merge here, which is + // right: they are one workstream, so "Acme" beneath either is one decision. + const contractorTally = new Map< + string, + { workstream: WorkstreamTarget; value: string; rowCount: number } + >(); + const stageTally = new Map< + string, + { workstream: WorkstreamTarget; value: string; rowCount: number } + >(); + let unscopedRows = blankRows.workstream; + + for (const [key, bucket] of rowsByWorkstreamKey) { + const workstreamTarget = resolvedByKey.get(key); + if (!workstreamTarget) { + // Unmapped workstream: no scope, so nothing beneath it can be listed yet. + unscopedRows += bucket.length; + continue; + } + for (const values of bucket) { + if (hasContractorColumn && values.contractor !== "") { + tally(contractorTally, workstreamTarget, values.contractor); + } + // A blank stage is legitimate — it falls back to the workstream's first + // stage at commit — so it is counted as blank, never listed for mapping. + if (hasStageColumn && values.stage !== "") { + tally(stageTally, workstreamTarget, values.stage); + } + } + } + + const contractors = [...contractorTally.entries()].map( + ([key, { workstream, value, rowCount }]) => ({ + workstreamId: workstream.id, + workstreamName: workstream.name, + value, + key, + rowCount, + noTargets: workstream.contractors.length === 0, + ...resolveValue( + key, + value, + valueMapping.contractors, + workstream.contractors, + (target) => target.name, + (target) => target.id, + ), + }), + ); + + const stages = [...stageTally.entries()].map( + ([key, { workstream, value, rowCount }]) => ({ + workstreamId: workstream.id, + workstreamName: workstream.name, + value, + key, + rowCount, + noTargets: workstream.stages.length === 0, + ...resolveValue( + key, + value, + valueMapping.stages, + workstream.stages, + (target) => target.name, + (target) => target.id, + ), + }), + ); + + byRowCount(workstreams); + byRowCount(contractors); + byRowCount(stages); + + return { + workstreams, + contractors, + stages, + hasWorkstreamColumn, + hasContractorColumn, + hasStageColumn, + blankRows, + unscopedRows, + needsReview: { + workstreams: workstreams.filter((e) => e.status === "needs_review").length, + contractors: contractors.filter((e) => e.status === "needs_review").length, + stages: stages.filter((e) => e.status === "needs_review").length, + }, + warningRows: countWarningRows( + rowsByWorkstreamKey, + resolvedByKey, + blankRows.workstream, + new Set( + contractors.filter((e) => e.status === "needs_review").map((e) => e.key), + ), + new Set( + stages.filter((e) => e.status === "needs_review").map((e) => e.key), + ), + ), + }; +} + +/** Add one row's contribution to a scoped tally. */ +function tally( + into: Map< + string, + { workstream: WorkstreamTarget; value: string; rowCount: number } + >, + workstream: WorkstreamTarget, + value: string, +): void { + const key = scopedValueKey(workstream.id, value); + const seen = into.get(key); + if (seen) seen.rowCount++; + else into.set(key, { workstream, value, rowCount: 1 }); +} + +/** + * Decide one value: the user's answer if they gave one, otherwise the best + * target above the auto-match threshold, otherwise nothing. + * + * A user answer that names a target no longer on offer (they mapped a + * contractor, then re-pointed the workstream at one that contractor is not + * assigned to) is discarded rather than kept as a dangling id — the value falls + * back to needing review, which is the truth of the situation. + */ +function resolveValue( + key: string, + value: string, + decisions: Record, + candidates: readonly T[], + nameOf: (target: T) => string, + idOf: (target: T) => string, +): { targetId: string | null; matchedBy: ValueMatchSource; status: ValueMappingStatus } { + if (Object.prototype.hasOwnProperty.call(decisions, key)) { + const chosen = decisions[key]; + if (chosen === null) { + return { targetId: null, matchedBy: null, status: "needs_review" }; + } + if (candidates.some((target) => idOf(target) === chosen)) { + return { targetId: chosen, matchedBy: "user", status: "matched" }; + } + return { targetId: null, matchedBy: null, status: "needs_review" }; + } + + let best: { id: string; score: number } | null = null; + let tied = false; + for (const target of candidates) { + const score = valueSimilarity(value, nameOf(target)); + if (score < VALUE_MATCH_THRESHOLD) continue; + if (best === null || score > best.score) { + best = { id: idOf(target), score }; + tied = false; + } else if (score === best.score) { + tied = true; + } + } + + // Two candidates the value fits equally well is not a match, it is a + // question: "Acme" beneath a workstream carrying both Acme Windows and Acme + // Doors must be asked, not guessed. (ADR-0019 leaves same-org disambiguation + // to #417; this keeps the guess from being made here in the meantime.) + return best && !tied + ? { targetId: best.id, matchedBy: "auto", status: "matched" } + : { targetId: null, matchedBy: null, status: "needs_review" }; +} + +/** Biggest first, then alphabetical — stable as the user maps, since neither key changes. */ +function byRowCount(entries: Array<{ rowCount: number; value: string }>): void { + entries.sort( + (a, b) => b.rowCount - a.rowCount || a.value.localeCompare(b.value, "en-GB"), + ); +} + +/** + * How many rows will fail validation at #417 as things stand. + * + * Counted **per row**, by walking the rows again rather than by summing the + * tables' row counts: a row with an unmapped workstream *and* a blank + * contractor is one broken row, not two, and summing the tables would report it + * twice. + * + * A row is broken when its workstream is blank or unmapped, its contractor is + * blank or unmapped, or it carries a stage that matches nothing. A *blank* + * stage is not broken — it falls back to the workstream's first stage. + */ +function countWarningRows( + rowsByWorkstreamKey: Map, + resolvedByKey: Map, + blankWorkstreamRows: number, + unmappedContractorKeys: Set, + unmappedStageKeys: Set, +): number { + // A blank workstream cell is unmappable, so every such row is already broken. + let broken = blankWorkstreamRows; + + for (const [key, bucket] of rowsByWorkstreamKey) { + const workstream = resolvedByKey.get(key); + if (!workstream) { + // The whole bucket shares one unmapped workstream value. + broken += bucket.length; + continue; + } + for (const values of bucket) { + const contractorBroken = + values.contractor === "" || + unmappedContractorKeys.has(scopedValueKey(workstream.id, values.contractor)); + const stageBroken = + values.stage !== "" && + unmappedStageKeys.has(scopedValueKey(workstream.id, values.stage)); + if (contractorBroken || stageBroken) broken++; + } + } + + return broken; +} + +/** Record one workstream-value decision. `null` means "none of these". */ +export function setWorkstreamValue( + mapping: ImportValueMapping, + key: string, + targetId: string | null, +): ImportValueMapping { + return { + ...mapping, + workstreams: { ...mapping.workstreams, [key]: targetId }, + }; +} + +/** Record one contractor decision, scoped to its workstream. */ +export function setContractorValue( + mapping: ImportValueMapping, + key: string, + targetId: string | null, +): ImportValueMapping { + return { + ...mapping, + contractors: { ...mapping.contractors, [key]: targetId }, + }; +} + +/** Record one stage decision, scoped to its workstream. */ +export function setStageValue( + mapping: ImportValueMapping, + key: string, + targetId: string | null, +): ImportValueMapping { + return { ...mapping, stages: { ...mapping.stages, [key]: targetId } }; +}