assessment-model/src/lib/projects/import/parse.ts

/**
 * Work-order import — file parsing (issue #414, step 1 of 3).
 *
 * The "second import": properties already exist in the DB (via BulkUpload or
 * postcode search); this file declares **which Workstreams each Property
 * gets**. One row per (Property, Workstream) — a Property taking Windows and
 * Doors appears twice, sharing an identifier. Grouping those duplicates is the
 * commit step's job (#417), not this module's.
 *
 * Deliberately **pure and DB-free**, mirroring `@/lib/tags/bulkAssign`: it takes
 * bytes or a 2D array of cells and returns data or a friendly `error` string.
 * That keeps it unit-testable without a connection pool, and keeps the route
 * handler down to authorization plus a function call.
 *
 * Scope note: this module is **header-agnostic on purpose**. It reports the
 * headers it found and hands back the rows; it does not insist on the template
 * columns and does not validate cell contents. In particular it does **not**
 * enforce that `Contractor` is present or that `Workstream`/`Contractor` name a
 * real assignment — required/optional is the mapping step's job (#415) and
 * value validation is the commit step's (#417). Rejecting an unrecognised
 * heading here would make the mapping step unreachable for exactly the files
 * that need it most.
 */

import * as XLSX from "xlsx";

/**
 * The columns the "Download template" artefact ships with, in order.
 *
 * `Property identifier` is the landlord property id *or* the UPRN — matching
 * decides which at validation time (#417).
 *
 * Required vs optional (enforced downstream, **not** here — see the module
 * header; this list is only the template's shape):
 *   - `Property identifier` — required.
 *   - `Workstream`          — required; must be one the project selected.
 *   - `Contractor`          — required; must match a contractor already
 *                             assigned to that workstream (ADR-0019).
 *                             Contractor creation is out of scope — the
 *                             existing "invite company to Ara" flow owns it.
 *   - `Stage`               — optional; blank falls back to the workstream's
 *                             first stage at commit (ADR-0019).
 *   - `Work order reference`— optional per row.
 *   - `Forecast end date`   — optional per row.
 */
export const IMPORT_TEMPLATE_COLUMNS = [
  "Property identifier",
  "Workstream",
  "Contractor",
  "Stage",
  "Work order reference",
  "Forecast end date",
] as const;

/**
 * Upper bound on the upload, enforced client- and server-side.
 *
 * The wireframe says 50MB; 10MB is well past what this shape of file can
 * plausibly reach (four narrow columns, one row per property-workstream), and
 * it keeps the whole-file-in-memory parse honest.
 */
export const MAX_IMPORT_FILE_BYTES = 10 * 1024 * 1024;

/** Upper bound on data rows, so an accidental 500k-row export fails fast. */
export const MAX_IMPORT_ROWS = 20_000;

/** How many rows the preview carries back for the mapping step. */
export const IMPORT_PREVIEW_ROWS = 10;

/** Extensions offered by the drop zone and re-checked on the server. */
export const ACCEPTED_IMPORT_EXTENSIONS = [".csv", ".xlsx", ".xls"] as const;

export interface ParsedImportFile {
  /** Header row, trimmed, in file order. Trailing blank columns dropped. */
  headers: string[];
  /**
   * Every data row, each padded/truncated to `headers.length`. All-blank rows
   * are dropped. Cell values are trimmed strings — never numbers, so a numeric
   * identifier keeps its leading zeros.
   */
  rows: string[][];
  /** `rows.length`, carried explicitly so callers need not hold `rows` to show it. */
  rowCount: number;
  /** The first `IMPORT_PREVIEW_ROWS` of `rows`, for the mapping step's sample table. */
  preview: string[][];
}

export type ParseImportResult =
  | { ok: true; file: ParsedImportFile }
  | { ok: false; error: string };

/** What SheetJS hands back per cell before we normalise it. */
type Cell = string | number | boolean | null | undefined;

/**
 * Read the first sheet of a CSV/XLSX/XLS buffer into rows of cells.
 *
 * `raw: false` makes SheetJS return each cell's *displayed* text rather than an
 * inferred type — so identifier `"07510027001"` survives instead of being
 * coerced to the number 7510027001 with its leading zero dropped. `defval: ""`
 * keeps ragged rows aligned to their column positions rather than collapsing
 * gaps. Both match the conventions in `@/lib/tags/bulkAssign` and
 * `@/lib/bulkUpload/addressMatches`.
 *
 * Throws whatever SheetJS throws on an unreadable file — `parseImportFile`
 * turns that into a friendly error.
 */
export function readImportSheet(buf: ArrayBuffer | Uint8Array): Cell[][] {
  const wb = XLSX.read(buf, { type: "array" });
  const sheetName = wb.SheetNames[0];
  if (!sheetName) return [];
  const sheet = wb.Sheets[sheetName];
  if (!sheet) return [];
  return XLSX.utils.sheet_to_json(sheet, {
    header: 1,
    raw: false,
    defval: "",
  }) as Cell[][];
}

/** A cell as trimmed text. Absent, null and blank all collapse to "". */
function text(cell: Cell): string {
  return String(cell ?? "").trim();
}

/** True when every cell in the row is blank — a spacer row, not data. */
function isBlankRow(row: Cell[]): boolean {
  return row.every((cell) => text(cell) === "");
}

/**
 * Turn raw sheet cells into headers plus rows, or a friendly error.
 *
 * Exported separately from `parseImportFile` so tests (and any future caller
 * that already holds cells) can exercise the row logic without going through
 * SheetJS.
 */
export function parseImportSheet(rows: Cell[][]): ParseImportResult {
  // Spreadsheets exported from asset-management systems often carry a blank
  // lead-in row or two, so find the first row with content rather than
  // assuming index 0.
  const headerIndex = rows.findIndex((row) => !isBlankRow(row));
  if (headerIndex === -1) {
    return {
      ok: false,
      error: "That file is empty — it needs a header row and at least one row of data.",
    };
  }

  // Trailing blank columns are an artefact of how spreadsheets size their used
  // range; drop them so the header count reflects real columns.
  const rawHeaders = rows[headerIndex].map(text);
  let lastUsed = rawHeaders.length - 1;
  while (lastUsed >= 0 && rawHeaders[lastUsed] === "") lastUsed--;
  const headers = rawHeaders.slice(0, lastUsed + 1);

  if (headers.length === 0) {
    return {
      ok: false,
      error: "Couldn't find a header row — the first row with content is blank.",
    };
  }

  // A gap *between* named columns would silently misalign the mapping step.
  const blankAt = headers.indexOf("");
  if (blankAt !== -1) {
    return {
      ok: false,
      error: `Column ${blankAt + 1} has no heading — give every column a heading, or remove the empty column.`,
    };
  }

  // Two identically-named columns make "which column did I map?" unanswerable.
  const duplicate = findDuplicateHeader(headers);
  if (duplicate) {
    return {
      ok: false,
      error: `The file has more than one '${duplicate}' column — give each column a unique heading.`,
    };
  }

  const dataRows: string[][] = [];
  for (const row of rows.slice(headerIndex + 1)) {
    if (isBlankRow(row)) continue;
    // Pad short rows and truncate long ones so every row lines up with headers.
    dataRows.push(
      Array.from({ length: headers.length }, (_, i) => text(row[i])),
    );
    // Bail as soon as the cap is passed — no point normalising 500k more rows.
    if (dataRows.length > MAX_IMPORT_ROWS) {
      return {
        ok: false,
        error: `The file has more than ${MAX_IMPORT_ROWS.toLocaleString("en-GB")} rows — split it into smaller files.`,
      };
    }
  }

  if (dataRows.length === 0) {
    return {
      ok: false,
      error: "The file has a header row but no rows of data.",
    };
  }

  return {
    ok: true,
    file: {
      headers,
      rows: dataRows,
      rowCount: dataRows.length,
      preview: dataRows.slice(0, IMPORT_PREVIEW_ROWS),
    },
  };
}

/** The first header that appears twice (case- and whitespace-insensitively). */
function findDuplicateHeader(headers: string[]): string | null {
  const seen = new Set<string>();
  for (const header of headers) {
    const key = header.toLowerCase().replace(/\s+/g, " ");
    if (seen.has(key)) return header;
    seen.add(key);
  }
  return null;
}

/**
 * Read and parse an uploaded file end to end, turning every failure into a
 * friendly `error` the route handler can return verbatim.
 *
 * Note that SheetJS only *throws* on a structurally broken container — a
 * corrupt or encrypted XLSX zip. Loose bytes (a PDF, an image, random noise)
 * are read happily as a one-line CSV, so they arrive at `parseImportSheet` as a
 * degenerate sheet and are turned away by the header/row checks there rather
 * than here. The extension allowlist on the route handler is what keeps that
 * from being the user's first line of defence.
 */
export function parseImportFile(buf: ArrayBuffer | Uint8Array): ParseImportResult {
  let cells: Cell[][];
  try {
    cells = readImportSheet(buf);
  } catch {
    return {
      ok: false,
      error: "Couldn't read that file — is it a valid CSV or Excel file?",
    };
  }
  return parseImportSheet(cells);
}

/** True when the filename carries one of the accepted extensions. */
export function hasAcceptedImportExtension(filename: string): boolean {
  const lower = filename.toLowerCase();
  return ACCEPTED_IMPORT_EXTENSIONS.some((ext) => lower.endsWith(ext));
}

/**
 * The "Download template" artefact: a CSV carrying the expected headings plus a
 * worked example that demonstrates the file shape's non-obvious rules rather
 * than explaining them:
 *   - a Property taking two Workstreams appears on two rows sharing its
 *     identifier (rows 1 and 2, `PROP-0001`);
 *   - `Contractor` is filled on every row (it is required, and must already be
 *     assigned to that workstream — ADR-0019);
 *   - `Stage` is blank on rows 2 and 3 to show it is optional (it falls back to
 *     the workstream's first stage at commit);
 *   - `Work order reference` and `Forecast end date` are blank on the last row
 *     to show they are optional too.
 *
 * CSV rather than XLSX so it opens the same everywhere and stays diffable.
 */
export function buildImportTemplateCsv(): string {
  const example = [
    ["PROP-0001", "Windows", "Acme Windows Ltd", "Ordered", "WO-1001", "2026-09-30"],
    ["PROP-0001", "Doors", "Acme Doors Ltd", "", "WO-1002", "2026-10-15"],
    ["PROP-0002", "Roofs", "Sunrise Roofing Ltd", "", "", ""],
  ];
  return (
    [[...IMPORT_TEMPLATE_COLUMNS], ...example]
      .map((row) => row.map(csvCell).join(","))
      .join("\r\n") + "\r\n"
  );
}

/** Quote a CSV cell only when it needs it. */
function csvCell(value: string): string {
  return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value;
}