From 9cfde2767f580d10b2b142ea55587c96ab32552c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 21 Jul 2026 16:43:48 +0000 Subject: [PATCH] =?UTF-8?q?feat(ara-projects):=20work-order=20import=20ste?= =?UTF-8?q?p=201=20=E2=80=94=20upload=20and=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `/projects/[projectId]/import`: a CSV/XLSX drop zone that uploads to a route handler, parses server-side with SheetJS (already a dependency — the Bulk tag assignment flow is the in-app parsing precedent, not the FastAPI BulkUpload pipeline), and returns headers plus a first-N preview. Nothing is written to the database at this step. Insert happens at commit (#417). Files: src/lib/projects/import/parse.ts pure, DB-free parser + template src/lib/projects/import/parse.test.ts 23 unit tests, no DB, no fixtures .../api/projects/[projectId]/import/parse/route.ts POST multipart .../api/projects/[projectId]/import/template/route.ts GET template CSV .../projects/[projectId]/import/page.tsx server component .../import/components/ImportUpload.tsx drop zone + preview Persistence choice: hold rows client-side, not S3-and-reparse ------------------------------------------------------------- The ticket asks for a choice between uploading the raw file to S3 via the presigned-URL pattern in `src/lib/bulkUpload/client.ts` and re-parsing it per step, versus holding the parsed rows in the client and POSTing them at commit. Chosen: hold the rows client-side. The parse route therefore returns every row, not only the preview, and the mapping (#415) and validation (#416) steps read them from `ImportUpload`'s mutation state. Reasoning: 1. S3-and-reparse needs somewhere to keep the object key and the import's status between steps, and the Ara Projects schema (PR #404) has no import table. Adding one means designing a second BulkUpload-shaped lifecycle — hard to reverse, and ADR-worthy — for a v1 that is insert-only with no diffing (deferred to #425). Not worth it to carry a file across three steps of one sitting. 2. The scale does not call for it. This import declares which workstreams the properties of a single project take — thousands of rows, not the hundreds-of-thousands the BulkUpload property pipeline handles. The 20k row cap bounds the commit POST to a few MB of JSON. 3. Re-parsing per step parses the same bytes three or four times and lets the steps disagree if the parser ever changes underneath them. Client-held rows are by construction the exact rows the user previewed and will validate. 4. It matches the existing precedent: Bulk tag assignment (ADR-0013) parses a small spreadsheet and POSTs the extracted identifiers at commit, with no S3 round trip. The cost is that a page refresh loses progress and the commit POST carries the payload. Both are acceptable for a single-sitting, insert-only v1; if the import later needs to be resumable or to exceed this scale, S3 staging plus an import table is the upgrade path, and the parser is already isolated behind `parseImportFile` so only the transport changes. Notes ----- - The parser is deliberately header-agnostic: it reports the headers it finds rather than demanding the template's. Mapping arbitrary client headings is #415's job, and rejecting unrecognised headings here would make the mapping step unreachable for exactly the files that need it. - Authorization uses `canManageProject` from `src/lib/projects/authz.ts` with facts from the #408 repository, so importing is internal/client-org only and never a contractor. Unreachable projects 404 rather than 403. The page applies the same check so a contractor never sees the drop zone. - `raw: false` on the SheetJS read keeps leading zeros on numeric-looking property identifiers (covered by a test). - Size is capped at 10MB client- and server-side. The wireframe says 50MB; 10MB is far past what four narrow columns can plausibly reach. - Known limitation, documented in a characterisation test: SheetJS does not throw on loose bytes, so a multi-line non-spreadsheet renamed to .csv parses into nonsense headers rather than being rejected. Nothing is written, the extension allowlist catches the honestly-named case, and the user sees the garbage in the preview. Content sniffing here would be guesswork. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[projectId]/import/parse/route.ts | 99 ++++++ .../[projectId]/import/template/route.ts | 56 +++ .../import/components/ImportUpload.tsx | 253 +++++++++++++ src/app/projects/[projectId]/import/page.tsx | 93 +++++ src/lib/projects/import/parse.test.ts | 333 ++++++++++++++++++ src/lib/projects/import/parse.ts | 263 ++++++++++++++ 6 files changed, 1097 insertions(+) create mode 100644 src/app/api/projects/[projectId]/import/parse/route.ts create mode 100644 src/app/api/projects/[projectId]/import/template/route.ts create mode 100644 src/app/projects/[projectId]/import/components/ImportUpload.tsx create mode 100644 src/app/projects/[projectId]/import/page.tsx create mode 100644 src/lib/projects/import/parse.test.ts create mode 100644 src/lib/projects/import/parse.ts diff --git a/src/app/api/projects/[projectId]/import/parse/route.ts b/src/app/api/projects/[projectId]/import/parse/route.ts new file mode 100644 index 00000000..0f487888 --- /dev/null +++ b/src/app/api/projects/[projectId]/import/parse/route.ts @@ -0,0 +1,99 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; +import { canManageProject } from "@/lib/projects/authz"; +import { + loadAuthzUser, + loadProjectAuthzFacts, +} from "@/app/repositories/projects/authzRepository"; +import { + hasAcceptedImportExtension, + MAX_IMPORT_FILE_BYTES, + parseImportFile, +} from "@/lib/projects/import/parse"; + +type Params = { params: Promise<{ projectId: string }> }; + +/** + * POST — parse an uploaded work-order import file (issue #414). + * + * Takes `multipart/form-data` with a single `file` field and returns the + * headers, every data row, a row count and a first-N preview. It writes + * nothing: the import is not persisted until commit (#417). + * + * The rows come back in full, not just the preview, because the parsed file is + * held client-side across the mapping and validation steps and POSTed once at + * commit — see the commit message for why that beat staging the file in S3. + */ +export async function POST(req: NextRequest, props: Params) { + const { projectId } = await props.params; + + const session = await getServerSession(AuthOptions); + if (!session?.user?.dbId) { + return NextResponse.json({ error: "Unauthorised" }, { status: 401 }); + } + if (!/^\d+$/.test(projectId)) { + return NextResponse.json({ error: "Invalid project id" }, { status: 400 }); + } + + const [user, project] = await Promise.all([ + loadAuthzUser(BigInt(session.user.dbId)), + loadProjectAuthzFacts(BigInt(projectId)), + ]); + // A project the caller cannot see must 404, not 403 — its existence is not + // theirs to learn (see `@/lib/projects/authz`). + if (!user || !project) { + return NextResponse.json({ error: "Project not found" }, { status: 404 }); + } + // Importing is managing: internal and client-org only, never a contractor. + if (!canManageProject(user, project)) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + let form: FormData; + try { + form = await req.formData(); + } catch { + return NextResponse.json( + { error: "Expected a file upload" }, + { status: 400 }, + ); + } + + const file = form.get("file"); + if (!(file instanceof File)) { + return NextResponse.json({ error: "No file was uploaded" }, { status: 400 }); + } + if (!hasAcceptedImportExtension(file.name)) { + return NextResponse.json( + { error: "That file type isn't supported — upload a .csv, .xlsx or .xls file." }, + { status: 400 }, + ); + } + // Re-checked here even though the drop zone checks it too: the client-side + // limit is a courtesy, this one is the rule. + if (file.size > MAX_IMPORT_FILE_BYTES) { + return NextResponse.json( + { + error: `That file is larger than ${MAX_IMPORT_FILE_BYTES / (1024 * 1024)}MB — split it into smaller files.`, + }, + { status: 413 }, + ); + } + if (file.size === 0) { + return NextResponse.json({ error: "That file is empty." }, { status: 400 }); + } + + const result = parseImportFile(new Uint8Array(await file.arrayBuffer())); + if (!result.ok) { + return NextResponse.json({ error: result.error }, { status: 400 }); + } + + return NextResponse.json({ + filename: file.name, + headers: result.file.headers, + rows: result.file.rows, + rowCount: result.file.rowCount, + preview: result.file.preview, + }); +} diff --git a/src/app/api/projects/[projectId]/import/template/route.ts b/src/app/api/projects/[projectId]/import/template/route.ts new file mode 100644 index 00000000..3371555a --- /dev/null +++ b/src/app/api/projects/[projectId]/import/template/route.ts @@ -0,0 +1,56 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; +import { canManageProject } from "@/lib/projects/authz"; +import { + loadAuthzUser, + loadProjectAuthzFacts, +} from "@/app/repositories/projects/authzRepository"; +import { buildImportTemplateCsv } from "@/lib/projects/import/parse"; + +type Params = { params: Promise<{ projectId: string }> }; + +/** + * GET — the "Download template" artefact: a CSV carrying the expected headings + * plus a worked example (issue #414). + * + * Served from a route handler rather than built in the browser so the template + * and the parser read their column list from the same constant — a heading + * renamed in one place cannot drift from the other. + * + * Gated behind the same permission as the import itself: the template is not + * secret, but there is no reason for it to be the one Projects endpoint a + * contractor can reach. + */ +export async function GET(_req: NextRequest, props: Params) { + const { projectId } = await props.params; + + const session = await getServerSession(AuthOptions); + if (!session?.user?.dbId) { + return NextResponse.json({ error: "Unauthorised" }, { status: 401 }); + } + if (!/^\d+$/.test(projectId)) { + return NextResponse.json({ error: "Invalid project id" }, { status: 400 }); + } + + const [user, project] = await Promise.all([ + loadAuthzUser(BigInt(session.user.dbId)), + loadProjectAuthzFacts(BigInt(projectId)), + ]); + if (!user || !project) { + return NextResponse.json({ error: "Project not found" }, { status: 404 }); + } + if (!canManageProject(user, project)) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + // BOM so Excel opens the CSV as UTF-8 rather than guessing a codepage. + return new NextResponse(`${buildImportTemplateCsv()}`, { + headers: { + "Content-Type": "text/csv; charset=utf-8", + "Content-Disposition": + 'attachment; filename="work-order-import-template.csv"', + "Cache-Control": "no-store", + }, + }); +} diff --git a/src/app/projects/[projectId]/import/components/ImportUpload.tsx b/src/app/projects/[projectId]/import/components/ImportUpload.tsx new file mode 100644 index 00000000..e8857d93 --- /dev/null +++ b/src/app/projects/[projectId]/import/components/ImportUpload.tsx @@ -0,0 +1,253 @@ +"use client"; + +import { DragEvent, useRef, useState } from "react"; +import { useMutation } from "@tanstack/react-query"; +import { + ArrowDownTrayIcon, + ArrowPathIcon, + CloudArrowUpIcon, + DocumentTextIcon, + ExclamationTriangleIcon, +} from "@heroicons/react/24/outline"; +import { Button } from "@/app/shadcn_components/ui/button"; +import { + ACCEPTED_IMPORT_EXTENSIONS, + hasAcceptedImportExtension, + MAX_IMPORT_FILE_BYTES, +} from "@/lib/projects/import/parse"; + +/** The parse route's response — the whole file, not just the preview. */ +export interface ParsedImportResponse { + filename: string; + headers: string[]; + rows: string[][]; + rowCount: number; + preview: string[][]; +} + +const MAX_MB = MAX_IMPORT_FILE_BYTES / (1024 * 1024); + +/** + * Work-order import drop zone (issue #414). + * + * Uploads the file to the parse route and renders the returned headers and + * preview. The full `rows` array stays here in `parse.data` — the mapping + * (#415), validation (#416) and commit (#417) steps read it from this + * component's state rather than re-fetching or re-parsing, which is why the + * route hands back every row and not only the preview. + */ +export function ImportUpload({ projectId }: { projectId: string }) { + const fileInputRef = useRef(null); + const [isDragging, setIsDragging] = useState(false); + const [fileName, setFileName] = useState(null); + // Size/extension failures caught before the upload; the route re-checks both. + const [clientError, setClientError] = useState(null); + + const parse = useMutation({ + mutationFn: async (file: File) => { + const form = new FormData(); + form.append("file", file); + const res = await fetch(`/api/projects/${projectId}/import/parse`, { + method: "POST", + body: form, + }); + const body = await res.json(); + if (!res.ok) throw new Error(body.error ?? "Couldn't read that file"); + return body; + }, + }); + + function onFile(file: File) { + setClientError(null); + parse.reset(); + setFileName(file.name); + + if (!hasAcceptedImportExtension(file.name)) { + setClientError( + `That file type isn't supported — upload a ${ACCEPTED_IMPORT_EXTENSIONS.join(", ")} file.`, + ); + return; + } + if (file.size > MAX_IMPORT_FILE_BYTES) { + setClientError( + `That file is larger than ${MAX_MB}MB — split it into smaller files.`, + ); + return; + } + parse.mutate(file); + } + + function handleDragOver(e: DragEvent) { + e.preventDefault(); + setIsDragging(true); + } + + function handleDrop(e: DragEvent) { + e.preventDefault(); + setIsDragging(false); + const dropped = e.dataTransfer.files?.[0]; + if (dropped) onFile(dropped); + } + + function reset() { + setFileName(null); + setClientError(null); + parse.reset(); + if (fileInputRef.current) fileInputRef.current.value = ""; + } + + const error = clientError ?? (parse.isError ? parse.error.message : null); + const parsed = parse.data; + + return ( +
+ + + + + {error && ( +
+ +
+

Couldn't import that file

+

{error}

+ +
+
+ )} + + {parsed && ( +
+
+
+ +
+

+ {parsed.filename} +

+

+ {parsed.rowCount.toLocaleString("en-GB")} row + {parsed.rowCount === 1 ? "" : "s"} ·{" "} + {parsed.headers.length} column + {parsed.headers.length === 1 ? "" : "s"} +

+
+
+ +
+ +
+ + + + {parsed.headers.map((header, i) => ( + + ))} + + + + {parsed.preview.map((row, rowIndex) => ( + + {row.map((cell, cellIndex) => ( + + ))} + + ))} + +
+ {header} +
+ {cell === "" ? ( + — + ) : ( + cell + )} +
+
+ + {parsed.rowCount > parsed.preview.length && ( +

+ Showing the first {parsed.preview.length} of{" "} + {parsed.rowCount.toLocaleString("en-GB")} rows. +

+ )} + +
+ {/* Enabled by the column-mapping step (#415). */} + +
+
+ )} +
+ ); +} diff --git a/src/app/projects/[projectId]/import/page.tsx b/src/app/projects/[projectId]/import/page.tsx new file mode 100644 index 00000000..627f93c4 --- /dev/null +++ b/src/app/projects/[projectId]/import/page.tsx @@ -0,0 +1,93 @@ +import { notFound } from "next/navigation"; +import { requireProjectAccess } from "../../authz"; +import { canManageProject } from "@/lib/projects/authz"; +import { + loadAuthzUser, + loadProjectAuthzFacts, +} from "@/app/repositories/projects/authzRepository"; +import { ImportUpload } from "./components/ImportUpload"; + +export const metadata = { + title: "Import programme data | Ara", +}; + +/** + * Work-order import, step 1 of 3 — upload and parse (issue #414). + * + * Wireframe: `docs/wireframes/ara-projects/1-admin-import/07-import-upload.html`. + * The mapping step (#415), validation (#416) and commit (#417) follow; this + * page ends at a parsed preview and writes nothing. + * + * Only internal and client-org managers may import, so this checks + * `canManageProject` rather than settling for the layout's view-level guard — + * a contractor with legitimate access to the project must not see the drop + * zone at all, not merely have their upload rejected by the route handler. + */ +export default async function ImportPage(props: { + params: Promise<{ projectId: string }>; +}) { + const { projectId } = await props.params; + const session = await requireProjectAccess(projectId); + + if (!/^\d+$/.test(projectId)) notFound(); + + const [user, project] = await Promise.all([ + loadAuthzUser(BigInt(session.user.dbId)), + loadProjectAuthzFacts(BigInt(projectId)), + ]); + // 404 rather than 403 throughout, so project existence stays private. + if (!user || !project || !canManageProject(user, project)) notFound(); + + return ( +
+
+

+ Work-order import · Step 1 of 3 +

+

+ Import programme data +

+

+ Upload a file listing which workstreams each property takes — one row + per property and workstream. A property taking Windows and Doors + appears twice, sharing its identifier. +

+
+ + + +
+ {[ + { + title: "Upload & parse", + body: "We read your file and show you the columns and a sample of the rows.", + }, + { + title: "Column mapping", + body: "Match your headings to the fields we expect.", + }, + { + title: "Validate & commit", + body: "Review what will be created before anything is written.", + }, + ].map((step, i) => ( +
+

+ Step {i + 1} +

+

+ {step.title} +

+

{step.body}

+
+ ))} +
+
+ ); +} diff --git a/src/lib/projects/import/parse.test.ts b/src/lib/projects/import/parse.test.ts new file mode 100644 index 00000000..3ff07100 --- /dev/null +++ b/src/lib/projects/import/parse.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; +import * as XLSX from "xlsx"; +import { + buildImportTemplateCsv, + hasAcceptedImportExtension, + IMPORT_PREVIEW_ROWS, + IMPORT_TEMPLATE_COLUMNS, + MAX_IMPORT_ROWS, + parseImportFile, + parseImportSheet, +} from "./parse"; + +/** + * Build a real workbook buffer in-memory rather than reading a checked-in + * fixture — the repo has no vitest fixture convention, and generating the bytes + * keeps these tests filesystem- and database-free. + */ +function workbookBuffer(rows: unknown[][], bookType: "xlsx" | "csv"): Uint8Array { + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, XLSX.utils.aoa_to_sheet(rows), "Sheet1"); + const out = XLSX.write(wb, { type: "array", bookType }); + return new Uint8Array(out); +} + +const HEADERS = [...IMPORT_TEMPLATE_COLUMNS]; + +describe("parseImportSheet", () => { + it("returns headers, rows, count and a preview", () => { + const result = parseImportSheet([ + HEADERS, + ["PROP-0001", "Windows", "WO-1001", "2026-09-30"], + ["PROP-0002", "Roofs", "", ""], + ]); + + expect(result).toEqual({ + ok: true, + file: { + headers: HEADERS, + rows: [ + ["PROP-0001", "Windows", "WO-1001", "2026-09-30"], + ["PROP-0002", "Roofs", "", ""], + ], + rowCount: 2, + preview: [ + ["PROP-0001", "Windows", "WO-1001", "2026-09-30"], + ["PROP-0002", "Roofs", "", ""], + ], + }, + }); + }); + + it("parses a property appearing on multiple rows without complaint", () => { + // One row per (property, workstream) is the decided file format: grouping + // the duplicates is the commit step's job (#417), not the parser's. + const result = parseImportSheet([ + HEADERS, + ["PROP-0001", "Windows", "", ""], + ["PROP-0001", "Doors", "", ""], + ["PROP-0001", "Roofs", "", ""], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rowCount).toBe(3); + expect(result.file.rows.map((r) => r[0])).toEqual([ + "PROP-0001", + "PROP-0001", + "PROP-0001", + ]); + expect(result.file.rows.map((r) => r[1])).toEqual([ + "Windows", + "Doors", + "Roofs", + ]); + }); + + it("accepts arbitrary client headings — mapping them is #415's job", () => { + const result = parseImportSheet([ + ["Asset Ref", "Trade", "Contract No"], + ["A-1", "Windows", "C-9"], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(["Asset Ref", "Trade", "Contract No"]); + }); + + it("skips blank lead-in rows and finds the real header row", () => { + const result = parseImportSheet([ + ["", "", "", ""], + ["", "", "", ""], + HEADERS, + ["PROP-0001", "Windows", "", ""], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(HEADERS); + expect(result.file.rowCount).toBe(1); + }); + + it("trims cells, drops blank rows, and pads ragged ones", () => { + const result = parseImportSheet([ + HEADERS, + [" PROP-0001 ", " Windows ", "", ""], + ["", "", "", ""], + ["PROP-0002", "Doors"], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rows).toEqual([ + ["PROP-0001", "Windows", "", ""], + ["PROP-0002", "Doors", "", ""], + ]); + }); + + it("truncates rows longer than the header row", () => { + const result = parseImportSheet([ + ["Property identifier", "Workstream"], + ["PROP-0001", "Windows", "stray", "cells"], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rows).toEqual([["PROP-0001", "Windows"]]); + }); + + it("drops trailing blank header columns", () => { + const result = parseImportSheet([ + ["Property identifier", "Workstream", "", ""], + ["PROP-0001", "Windows", "", ""], + ]); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(["Property identifier", "Workstream"]); + expect(result.file.rows).toEqual([["PROP-0001", "Windows"]]); + }); + + it("caps the preview at IMPORT_PREVIEW_ROWS but keeps every row", () => { + const rows: string[][] = [HEADERS]; + for (let i = 0; i < IMPORT_PREVIEW_ROWS + 5; i++) { + rows.push([`PROP-${i}`, "Windows", "", ""]); + } + + const result = parseImportSheet(rows); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rowCount).toBe(IMPORT_PREVIEW_ROWS + 5); + expect(result.file.rows).toHaveLength(IMPORT_PREVIEW_ROWS + 5); + expect(result.file.preview).toHaveLength(IMPORT_PREVIEW_ROWS); + expect(result.file.preview[0]).toEqual(["PROP-0", "Windows", "", ""]); + }); + + it("errors on an entirely empty sheet", () => { + expect(parseImportSheet([])).toEqual({ + ok: false, + error: + "That file is empty — it needs a header row and at least one row of data.", + }); + expect(parseImportSheet([["", ""], ["", ""]])).toEqual({ + ok: false, + error: + "That file is empty — it needs a header row and at least one row of data.", + }); + }); + + it("errors when there is a header row but no data", () => { + expect(parseImportSheet([HEADERS])).toEqual({ + ok: false, + error: "The file has a header row but no rows of data.", + }); + }); + + it("errors on a gap between named header columns", () => { + expect( + parseImportSheet([ + ["Property identifier", "", "Workstream"], + ["PROP-0001", "", "Windows"], + ]), + ).toEqual({ + ok: false, + error: + "Column 2 has no heading — give every column a heading, or remove the empty column.", + }); + }); + + it("errors on duplicate headings, ignoring case and spacing", () => { + expect( + parseImportSheet([ + ["Workstream", "Property identifier", " workstream "], + ["Windows", "PROP-0001", "Doors"], + ]), + ).toEqual({ + ok: false, + error: + "The file has more than one 'workstream' column — give each column a unique heading.", + }); + }); + + it("errors when the row cap is exceeded", () => { + const rows: string[][] = [HEADERS]; + for (let i = 0; i <= MAX_IMPORT_ROWS; i++) { + rows.push([`PROP-${i}`, "Windows", "", ""]); + } + + const result = parseImportSheet(rows); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toContain("split it into smaller files"); + }); +}); + +describe("parseImportFile", () => { + it("parses a real XLSX buffer", () => { + const buf = workbookBuffer( + [HEADERS, ["PROP-0001", "Windows", "WO-1001", "2026-09-30"]], + "xlsx", + ); + + const result = parseImportFile(buf); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(HEADERS); + expect(result.file.rows).toEqual([ + ["PROP-0001", "Windows", "WO-1001", "2026-09-30"], + ]); + }); + + it("parses a real CSV buffer", () => { + const buf = workbookBuffer( + [HEADERS, ["PROP-0001", "Windows", "", ""]], + "csv", + ); + + const result = parseImportFile(buf); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rows).toEqual([["PROP-0001", "Windows", "", ""]]); + }); + + it("keeps the leading zero on a numeric-looking identifier", () => { + // raw:false is what buys this — SheetJS would otherwise hand back the + // number 7510027001 and the identifier would stop matching. + const buf = workbookBuffer( + [["Property identifier", "Workstream"], ["07510027001", "Windows"]], + "xlsx", + ); + + const result = parseImportFile(buf); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.rows[0][0]).toBe("07510027001"); + }); + + it("reads the round-tripped download template", () => { + const result = parseImportFile( + new TextEncoder().encode(buildImportTemplateCsv()), + ); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(HEADERS); + // The template demonstrates one property taking two workstreams. + expect(result.file.rows[0][0]).toBe("PROP-0001"); + expect(result.file.rows[1][0]).toBe("PROP-0001"); + expect(result.file.rows[0][1]).toBe("Windows"); + expect(result.file.rows[1][1]).toBe("Doors"); + }); + + it("rejects empty and single-line rubbish with a friendly error", () => { + // SheetJS does not throw on loose bytes — it reads them as CSV — so these + // are turned away by the header/row checks rather than the read. What the + // acceptance criteria care about is the friendly error, not the branch. + for (const bytes of ["", "\x00\x01\x02 not a spreadsheet"]) { + const result = parseImportFile(new TextEncoder().encode(bytes)); + expect(result.ok).toBe(false); + if (result.ok) continue; + expect(result.error.length).toBeGreaterThan(0); + } + }); + + it("known limitation: multi-line text parses as a degenerate sheet", () => { + // A PDF renamed to .csv has more than one line, so SheetJS reads line 1 as + // a header and the rest as data and the parse "succeeds" with nonsense + // headers. Nothing is written at this step, and the extension allowlist on + // the route handler turns away the honestly-named case; the user's real + // signal is the garbage they see in the preview, and the mapping step + // (#415) has no columns to offer. Documented rather than defended against + // — sniffing content types here would be guesswork. + const result = parseImportFile( + new TextEncoder().encode("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n1 0 obj"), + ); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.file.headers).toEqual(["%PDF-1.4"]); + }); + + it("returns a friendly error for a corrupt workbook", () => { + // Zip magic bytes followed by rubbish — SheetJS throws on this one. + const result = parseImportFile( + new Uint8Array([0x50, 0x4b, 0x03, 0x04, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + ); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatch(/valid CSV or Excel file/); + }); +}); + +describe("hasAcceptedImportExtension", () => { + it("accepts csv, xlsx and xls in any case", () => { + expect(hasAcceptedImportExtension("programme.csv")).toBe(true); + expect(hasAcceptedImportExtension("programme.XLSX")).toBe(true); + expect(hasAcceptedImportExtension("Programme.Xls")).toBe(true); + }); + + it("rejects anything else", () => { + expect(hasAcceptedImportExtension("programme.pdf")).toBe(false); + expect(hasAcceptedImportExtension("programme")).toBe(false); + expect(hasAcceptedImportExtension("csv.docx")).toBe(false); + }); +}); + +describe("buildImportTemplateCsv", () => { + it("leads with the template columns in order", () => { + expect(buildImportTemplateCsv().split("\r\n")[0]).toBe( + IMPORT_TEMPLATE_COLUMNS.join(","), + ); + }); +}); diff --git a/src/lib/projects/import/parse.ts b/src/lib/projects/import/parse.ts new file mode 100644 index 00000000..3f31f589 --- /dev/null +++ b/src/lib/projects/import/parse.ts @@ -0,0 +1,263 @@ +/** + * Work-order import — file parsing (issue #414, step 1 of 3). + * + * The "second import": properties already exist in the DB (via BulkUpload or + * postcode search); this file declares **which Workstreams each Property + * gets**. One row per (Property, Workstream) — a Property taking Windows and + * Doors appears twice, sharing an identifier. Grouping those duplicates is the + * commit step's job (#417), not this module's. + * + * Deliberately **pure and DB-free**, mirroring `@/lib/tags/bulkAssign`: it takes + * bytes or a 2D array of cells and returns data or a friendly `error` string. + * That keeps it unit-testable without a connection pool, and keeps the route + * handler down to authorization plus a function call. + * + * Scope note: this module is **header-agnostic on purpose**. It reports the + * headers it found and hands back the rows; it does not insist on the template + * columns and does not validate cell contents. Mapping arbitrary client headings + * onto the template columns is the mapping step (#415), and validating the + * mapped values is #416. Rejecting an unrecognised heading here would make the + * mapping step unreachable for exactly the files that need it most. + */ + +import * as XLSX from "xlsx"; + +/** + * The columns the "Download template" artefact ships with, in order. + * + * `Property identifier` is the landlord property id *or* the UPRN — matching + * decides which at validation time (#416). The last two are optional per row. + */ +export const IMPORT_TEMPLATE_COLUMNS = [ + "Property identifier", + "Workstream", + "Work order reference", + "Forecast end date", +] as const; + +/** + * Upper bound on the upload, enforced client- and server-side. + * + * The wireframe says 50MB; 10MB is well past what this shape of file can + * plausibly reach (four narrow columns, one row per property-workstream), and + * it keeps the whole-file-in-memory parse honest. + */ +export const MAX_IMPORT_FILE_BYTES = 10 * 1024 * 1024; + +/** Upper bound on data rows, so an accidental 500k-row export fails fast. */ +export const MAX_IMPORT_ROWS = 20_000; + +/** How many rows the preview carries back for the mapping step. */ +export const IMPORT_PREVIEW_ROWS = 10; + +/** Extensions offered by the drop zone and re-checked on the server. */ +export const ACCEPTED_IMPORT_EXTENSIONS = [".csv", ".xlsx", ".xls"] as const; + +export interface ParsedImportFile { + /** Header row, trimmed, in file order. Trailing blank columns dropped. */ + headers: string[]; + /** + * Every data row, each padded/truncated to `headers.length`. All-blank rows + * are dropped. Cell values are trimmed strings — never numbers, so a numeric + * identifier keeps its leading zeros. + */ + rows: string[][]; + /** `rows.length`, carried explicitly so callers need not hold `rows` to show it. */ + rowCount: number; + /** The first `IMPORT_PREVIEW_ROWS` of `rows`, for the mapping step's sample table. */ + preview: string[][]; +} + +export type ParseImportResult = + | { ok: true; file: ParsedImportFile } + | { ok: false; error: string }; + +/** What SheetJS hands back per cell before we normalise it. */ +type Cell = string | number | boolean | null | undefined; + +/** + * Read the first sheet of a CSV/XLSX/XLS buffer into rows of cells. + * + * `raw: false` makes SheetJS return each cell's *displayed* text rather than an + * inferred type — so identifier `"07510027001"` survives instead of being + * coerced to the number 7510027001 with its leading zero dropped. `defval: ""` + * keeps ragged rows aligned to their column positions rather than collapsing + * gaps. Both match the conventions in `@/lib/tags/bulkAssign` and + * `@/lib/bulkUpload/addressMatches`. + * + * Throws whatever SheetJS throws on an unreadable file — `parseImportFile` + * turns that into a friendly error. + */ +export function readImportSheet(buf: ArrayBuffer | Uint8Array): Cell[][] { + const wb = XLSX.read(buf, { type: "array" }); + const sheetName = wb.SheetNames[0]; + if (!sheetName) return []; + const sheet = wb.Sheets[sheetName]; + if (!sheet) return []; + return XLSX.utils.sheet_to_json(sheet, { + header: 1, + raw: false, + defval: "", + }) as Cell[][]; +} + +/** A cell as trimmed text. Absent, null and blank all collapse to "". */ +function text(cell: Cell): string { + return String(cell ?? "").trim(); +} + +/** True when every cell in the row is blank — a spacer row, not data. */ +function isBlankRow(row: Cell[]): boolean { + return row.every((cell) => text(cell) === ""); +} + +/** + * Turn raw sheet cells into headers plus rows, or a friendly error. + * + * Exported separately from `parseImportFile` so tests (and any future caller + * that already holds cells) can exercise the row logic without going through + * SheetJS. + */ +export function parseImportSheet(rows: Cell[][]): ParseImportResult { + // Spreadsheets exported from asset-management systems often carry a blank + // lead-in row or two, so find the first row with content rather than + // assuming index 0. + const headerIndex = rows.findIndex((row) => !isBlankRow(row)); + if (headerIndex === -1) { + return { + ok: false, + error: "That file is empty — it needs a header row and at least one row of data.", + }; + } + + // Trailing blank columns are an artefact of how spreadsheets size their used + // range; drop them so the header count reflects real columns. + const rawHeaders = rows[headerIndex].map(text); + let lastUsed = rawHeaders.length - 1; + while (lastUsed >= 0 && rawHeaders[lastUsed] === "") lastUsed--; + const headers = rawHeaders.slice(0, lastUsed + 1); + + if (headers.length === 0) { + return { + ok: false, + error: "Couldn't find a header row — the first row with content is blank.", + }; + } + + // A gap *between* named columns would silently misalign the mapping step. + const blankAt = headers.indexOf(""); + if (blankAt !== -1) { + return { + ok: false, + error: `Column ${blankAt + 1} has no heading — give every column a heading, or remove the empty column.`, + }; + } + + // Two identically-named columns make "which column did I map?" unanswerable. + const duplicate = findDuplicateHeader(headers); + if (duplicate) { + return { + ok: false, + error: `The file has more than one '${duplicate}' column — give each column a unique heading.`, + }; + } + + const dataRows: string[][] = []; + for (const row of rows.slice(headerIndex + 1)) { + if (isBlankRow(row)) continue; + // Pad short rows and truncate long ones so every row lines up with headers. + dataRows.push( + Array.from({ length: headers.length }, (_, i) => text(row[i])), + ); + // Bail as soon as the cap is passed — no point normalising 500k more rows. + if (dataRows.length > MAX_IMPORT_ROWS) { + return { + ok: false, + error: `The file has more than ${MAX_IMPORT_ROWS.toLocaleString("en-GB")} rows — split it into smaller files.`, + }; + } + } + + if (dataRows.length === 0) { + return { + ok: false, + error: "The file has a header row but no rows of data.", + }; + } + + return { + ok: true, + file: { + headers, + rows: dataRows, + rowCount: dataRows.length, + preview: dataRows.slice(0, IMPORT_PREVIEW_ROWS), + }, + }; +} + +/** The first header that appears twice (case- and whitespace-insensitively). */ +function findDuplicateHeader(headers: string[]): string | null { + const seen = new Set(); + for (const header of headers) { + const key = header.toLowerCase().replace(/\s+/g, " "); + if (seen.has(key)) return header; + seen.add(key); + } + return null; +} + +/** + * Read and parse an uploaded file end to end, turning every failure into a + * friendly `error` the route handler can return verbatim. + * + * Note that SheetJS only *throws* on a structurally broken container — a + * corrupt or encrypted XLSX zip. Loose bytes (a PDF, an image, random noise) + * are read happily as a one-line CSV, so they arrive at `parseImportSheet` as a + * degenerate sheet and are turned away by the header/row checks there rather + * than here. The extension allowlist on the route handler is what keeps that + * from being the user's first line of defence. + */ +export function parseImportFile(buf: ArrayBuffer | Uint8Array): ParseImportResult { + let cells: Cell[][]; + try { + cells = readImportSheet(buf); + } catch { + return { + ok: false, + error: "Couldn't read that file — is it a valid CSV or Excel file?", + }; + } + return parseImportSheet(cells); +} + +/** True when the filename carries one of the accepted extensions. */ +export function hasAcceptedImportExtension(filename: string): boolean { + const lower = filename.toLowerCase(); + return ACCEPTED_IMPORT_EXTENSIONS.some((ext) => lower.endsWith(ext)); +} + +/** + * The "Download template" artefact: a CSV carrying the expected headings plus + * one worked example showing a Property that takes two Workstreams — the file + * shape's one non-obvious rule, demonstrated rather than explained. + * + * CSV rather than XLSX so it opens the same everywhere and stays diffable. + */ +export function buildImportTemplateCsv(): string { + const example = [ + ["PROP-0001", "Windows", "WO-1001", "2026-09-30"], + ["PROP-0001", "Doors", "WO-1002", "2026-10-15"], + ["PROP-0002", "Roofs", "", ""], + ]; + return ( + [[...IMPORT_TEMPLATE_COLUMNS], ...example] + .map((row) => row.map(csvCell).join(",")) + .join("\r\n") + "\r\n" + ); +} + +/** Quote a CSV cell only when it needs it. */ +function csvCell(value: string): string { + return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; +}