mirror of
https://github.com/Hestia-Homes/assessment-model.git
synced 2026-06-08 11:37:25 +00:00
Merge pull request #243 from Hestia-Homes/feature/onbarding_of_addresses
Some checks are pending
Test Suite / unit-tests (push) Waiting to run
Some checks are pending
Test Suite / unit-tests (push) Waiting to run
Feature/onbarding of addresses
This commit is contained in:
Jun-te Kim
GitHub
commit
6f9fabb622
33 changed files with 2394 additions and 6963 deletions
|
|
@ -23,6 +23,7 @@ services:
|
|||
- GITHUB_TOKEN=${GITHUB_TOKEN:-}
|
||||
networks:
|
||||
- frontend-net
|
||||
- shared-dev
|
||||
|
||||
pgadmin:
|
||||
image: dpage/pgadmin4
|
||||
|
|
@ -38,3 +39,6 @@ services:
|
|||
networks:
|
||||
frontend-net:
|
||||
driver: bridge
|
||||
shared-dev:
|
||||
external: true
|
||||
name: shared-dev
|
||||
|
|
|
|||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -37,3 +37,7 @@ cypress.env.json
|
|||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
||||
backlog/**
|
||||
|
||||
docs/adr/**
|
||||
|
|
|
|||
16
CLAUDE.md
Normal file
16
CLAUDE.md
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# Claude guidance for this project
|
||||
|
||||
## Project conventions
|
||||
|
||||
- **Domain language lives in [`CONTEXT.md`](./CONTEXT.md).** Read it before naming or discussing BulkUpload, Portfolio, Property, etc. concepts.
|
||||
- **Architectural decisions live in [`docs/adr/`](./docs/adr/).** Read existing ADRs before proposing changes that touch state machines or core flows. Write a new ADR for any decision that's hard to reverse, surprising without context, and the result of a real trade-off.
|
||||
|
||||
## React
|
||||
|
||||
- **Avoid `useEffect` and `useMemo`.** Derive values inline, prefer Server Components + Route Handlers, prefer event handlers. If a hook is genuinely the only option, flag it and ask before using it.
|
||||
- **Use TanStack Query (`@tanstack/react-query`), not raw `fetch`, for client-side HTTP.** Reads use `useQuery`; writes use `useMutation`. This project is on **v4** — note that `refetchInterval`'s callback signature is `(data, query)`, not v5's `(query)`.
|
||||
|
||||
## Next.js 15 route handlers
|
||||
|
||||
- `params` is a `Promise` — type as `{ params: Promise<{ ... }> }` and `await params` before destructuring.
|
||||
- Error responses use `{ error: string }` consistently. Don't drift to `{ msg }` or other shapes.
|
||||
73
CONTEXT.md
Normal file
73
CONTEXT.md
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
# Context
|
||||
|
||||
This document captures the domain language used in this project. Terms here are the **canonical** ones — when more than one word exists for a concept, we pick one and treat the others as aliases to avoid.
|
||||
|
||||
This file grows as terms are resolved during design conversations. Concepts that haven't been examined yet are not listed.
|
||||
|
||||
## Language
|
||||
|
||||
### Bulk upload
|
||||
|
||||
**BulkUpload**:
|
||||
A user-supplied spreadsheet of addresses for a Portfolio, transformed and matched to UPRNs before being inserted as Properties. Has an explicit lifecycle from upload through finalisation.
|
||||
_Avoid_: import, batch, file upload, ingest
|
||||
|
||||
**ColumnMapping**:
|
||||
The user's declaration of which spreadsheet column means what (e.g. column "Property Address" means `address_1`). Stored as JSON on the BulkUpload row.
|
||||
_Avoid_: schema, header map, field mapping
|
||||
|
||||
**UPRN**:
|
||||
Unique Property Reference Number — the UK national identifier for an address. Address matching attaches a UPRN to each row where possible.
|
||||
|
||||
**Address matching**:
|
||||
The pipeline stage that splits the source file by postcode, looks up UPRNs, and produces matched-address output. Triggered via FastAPI.
|
||||
_Avoid_: postcode lookup, address resolution, address lookup
|
||||
|
||||
**Combiner**:
|
||||
The pipeline stage that aggregates the per-postcode address-matching outputs into a single combined CSV in S3, ready for review.
|
||||
_Avoid_: aggregator, merger
|
||||
|
||||
**Finalise**:
|
||||
The terminal action that reads the combiner output, inserts rows as Properties on the Portfolio, and decides whether the BulkUpload needs further review.
|
||||
_Avoid_: import, commit, ingest
|
||||
|
||||
## Lifecycle
|
||||
|
||||
A **BulkUpload** moves through these statuses:
|
||||
|
||||
```
|
||||
ready_for_processing
|
||||
→ mapping_complete (user submits ColumnMapping; Next.js writes)
|
||||
→ processing (Address matching triggered; Next.js writes)
|
||||
→ combining (Combiner stage running; FastAPI writes directly)
|
||||
→ awaiting_review (Combiner output in S3; FastAPI writes directly)
|
||||
→ complete (Finalise succeeded; Next.js writes)
|
||||
→ failed (FastAPI reports in-flight failure — schema only, not yet wired)
|
||||
```
|
||||
|
||||
`complete` and `failed` are terminal.
|
||||
|
||||
Re-mapping (PATCHing `columnMapping`) is legal only in `ready_for_processing` and `mapping_complete`. Any later state rejects with 409.
|
||||
|
||||
**Two writers**: Next.js owns transitions out of `mapping_complete`, into `processing`, and the terminal Finalise outcomes. FastAPI owns `combining` and `awaiting_review` — writing them direct to the DB during the combiner run. The BulkUpload aggregate observes both.
|
||||
|
||||
See [ADR-0001](./docs/adr/0001-bulk-upload-state-machine.md) for the deliberate "not yet" decisions baked into this lifecycle.
|
||||
|
||||
## Relationships
|
||||
|
||||
- A **Portfolio** has many **BulkUploads**.
|
||||
- A **BulkUpload** produces zero or more **Properties** when finalised.
|
||||
- A **BulkUpload** has at most one **Task** (the orchestration handle for the FastAPI pipeline run); a Task has many **SubTasks** (one per pipeline stage: address matching, combiner).
|
||||
|
||||
## Example dialogue
|
||||
|
||||
> **Dev:** "If the **Combiner** finishes but the user hasn't clicked Finalise, what does the user see?"
|
||||
> **Domain expert:** "The BulkUpload sits in `awaiting_review`. The frontend polls and shows a 'review and confirm' button. Nothing's been written to **Properties** yet."
|
||||
>
|
||||
> **Dev:** "And if **Finalise** runs and 30% of rows have no **UPRN**?"
|
||||
> **Domain expert:** "Those still get imported as **Properties** — just without a UPRN — and the BulkUpload moves to `complete`. Manual cleanup happens later in the property table."
|
||||
|
||||
## Flagged ambiguities
|
||||
|
||||
- "Upload" is used in the codebase to mean both the file-on-S3 and the BulkUpload row. We standardise on **BulkUpload** for the row; the file is just "the source file."
|
||||
- "Onboarding" appears in some route paths (`bulk_onboarding_inputs/...`) but isn't part of this glossary — we use **BulkUpload** end-to-end.
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { requestCombineRetrigger } from "@/lib/bulkUpload/server";
|
||||
import { readSessionToken } from "@/lib/session";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
|
||||
export async function POST(
|
||||
request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
|
||||
) {
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
|
||||
const { uploadId } = await params;
|
||||
const result = await requestCombineRetrigger({
|
||||
uploadId,
|
||||
sessionToken: readSessionToken(request),
|
||||
});
|
||||
|
||||
switch (result.kind) {
|
||||
case "triggered":
|
||||
return NextResponse.json(
|
||||
{ taskId: result.taskId, subTaskId: result.subTaskId },
|
||||
{ status: 200 }
|
||||
);
|
||||
case "already_combined":
|
||||
return NextResponse.json({ alreadyCombined: true }, { status: 200 });
|
||||
case "not_found":
|
||||
return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
case "missing_task":
|
||||
return NextResponse.json({ error: "Upload has no task" }, { status: 422 });
|
||||
case "trigger_failed":
|
||||
return NextResponse.json({ error: result.message }, { status: result.status });
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { property } from "@/app/db/schema/property";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { revalidatePath } from "next/cache";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
import { createRetrofitDataS3Client } from "@/app/utils/s3";
|
||||
import * as XLSX from "xlsx";
|
||||
import { loadForFinalize, markFinalized } from "@/lib/bulkUpload/server";
|
||||
|
||||
const ADDRESS_COLS = ["Address 1", "Address 2", "Address 3"] as const;
|
||||
const POSTCODE_COL = "postcode";
|
||||
const INTERNAL_REF_COL = "Internal Reference";
|
||||
const UPRN_COL = "address2uprn_uprn";
|
||||
const MATCHED_ADDRESS_COL = "address2uprn_address";
|
||||
const LEXISCORE_COL = "address2uprn_lexiscore";
|
||||
const MISSING_SENTINEL = "invalid postcode";
|
||||
const UK_POSTCODE_RE = /[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}/i;
|
||||
|
||||
function normalize(v: unknown): string {
|
||||
if (v === null || v === undefined) return "";
|
||||
return String(v).trim();
|
||||
}
|
||||
|
||||
function isMissing(v: string): boolean {
|
||||
return v === "" || v.toLowerCase() === MISSING_SENTINEL;
|
||||
}
|
||||
|
||||
function parseUprn(raw: unknown): bigint | null {
|
||||
const v = normalize(raw);
|
||||
if (isMissing(v)) return null;
|
||||
try {
|
||||
return BigInt(v);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseLexiscore(raw: unknown): number | null {
|
||||
const v = normalize(raw);
|
||||
if (isMissing(v)) return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function extractPostcode(matched: string | null, fallback: string): string | null {
|
||||
if (matched) {
|
||||
const m = matched.match(UK_POSTCODE_RE);
|
||||
if (m) return m[0].toUpperCase();
|
||||
}
|
||||
return fallback || null;
|
||||
}
|
||||
|
||||
function parseS3Uri(uri: string): { bucket: string; key: string } | null {
|
||||
if (!uri.startsWith("s3://")) return null;
|
||||
const rest = uri.slice(5);
|
||||
const slash = rest.indexOf("/");
|
||||
if (slash < 0) return null;
|
||||
return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
||||
}
|
||||
|
||||
export async function POST(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
|
||||
) {
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
|
||||
const { uploadId } = await params;
|
||||
|
||||
const guarded = await loadForFinalize(uploadId);
|
||||
switch (guarded.kind) {
|
||||
case "not_found":
|
||||
return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
case "already_finalized":
|
||||
return new NextResponse(null, { status: 200 });
|
||||
case "wrong_state":
|
||||
return NextResponse.json(
|
||||
{ error: `Upload not ready to finalize (state: ${guarded.current})` },
|
||||
{ status: 409 }
|
||||
);
|
||||
case "not_yet_combined":
|
||||
return NextResponse.json({ error: "Combiner not finished" }, { status: 409 });
|
||||
}
|
||||
const upload = guarded.upload;
|
||||
|
||||
const parsed = parseS3Uri(upload.combinedOutputS3Uri!);
|
||||
if (!parsed) {
|
||||
return NextResponse.json({ error: "Invalid combined output S3 URI" }, { status: 500 });
|
||||
}
|
||||
|
||||
const s3 = createRetrofitDataS3Client();
|
||||
|
||||
let rawRows: Record<string, unknown>[];
|
||||
try {
|
||||
const obj = await s3
|
||||
.getObject({ Bucket: parsed.bucket, Key: parsed.key })
|
||||
.promise();
|
||||
const buf = Buffer.from(obj.Body as Uint8Array);
|
||||
const wb = XLSX.read(buf, { type: "buffer" });
|
||||
const sheet = wb.Sheets[wb.SheetNames[0]];
|
||||
rawRows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: "" });
|
||||
} catch (err) {
|
||||
console.error("Failed to read combined CSV from S3:", err);
|
||||
return NextResponse.json({ error: "Failed to read combined CSV" }, { status: 502 });
|
||||
}
|
||||
|
||||
const portfolioIdBig = BigInt(upload.portfolioId);
|
||||
|
||||
const values = rawRows.map((raw) => {
|
||||
const userInputtedAddress =
|
||||
ADDRESS_COLS.map((c) => normalize(raw[c])).filter(Boolean).join(", ") || null;
|
||||
const userInputtedPostcode = normalize(raw[POSTCODE_COL]) || null;
|
||||
|
||||
const uprn = parseUprn(raw[UPRN_COL]);
|
||||
|
||||
const matchedAddressRaw = normalize(raw[MATCHED_ADDRESS_COL]);
|
||||
const matchedAddress = isMissing(matchedAddressRaw) ? null : matchedAddressRaw;
|
||||
|
||||
const address = matchedAddress ?? userInputtedAddress;
|
||||
const postcode = extractPostcode(matchedAddress, userInputtedPostcode ?? "");
|
||||
|
||||
const internalRef = normalize(raw[INTERNAL_REF_COL]) || null;
|
||||
const lexiscore = parseLexiscore(raw[LEXISCORE_COL]);
|
||||
|
||||
return {
|
||||
portfolioId: portfolioIdBig,
|
||||
creationStatus: "READY" as const,
|
||||
uprn,
|
||||
landlordPropertyId: internalRef,
|
||||
address,
|
||||
postcode,
|
||||
userInputtedAddress,
|
||||
userInputtedPostcode,
|
||||
lexiscore,
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
if (values.length > 0) {
|
||||
await db
|
||||
.insert(property)
|
||||
.values(values)
|
||||
.onConflictDoNothing({
|
||||
target: [property.portfolioId, property.uprn],
|
||||
where: sql`${property.uprn} IS NOT NULL`,
|
||||
});
|
||||
}
|
||||
|
||||
await markFinalized(uploadId);
|
||||
|
||||
revalidatePath("/portfolio/[slug]", "layout");
|
||||
|
||||
return new NextResponse(null, { status: 200 });
|
||||
} catch (err) {
|
||||
console.error("Failed to finalize bulk upload:", err);
|
||||
return NextResponse.json({ error: "Failed to import properties" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import { getProgressView } from "@/lib/bulkUpload/server";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
|
||||
export async function GET(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
|
||||
) {
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
|
||||
const { uploadId } = await params;
|
||||
const view = await getProgressView(uploadId);
|
||||
if (!view) return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
return NextResponse.json(view, { status: 200 });
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import { setColumnMapping } from "@/lib/bulkUpload/server";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
|
||||
const PatchSchema = z.object({
|
||||
columnMapping: z.record(z.string(), z.string()),
|
||||
});
|
||||
|
||||
export async function PATCH(
|
||||
request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
|
||||
) {
|
||||
const { uploadId } = await params;
|
||||
|
||||
let body;
|
||||
try {
|
||||
body = PatchSchema.parse(await request.json());
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid input" }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await setColumnMapping(uploadId, body.columnMapping);
|
||||
switch (result.kind) {
|
||||
case "ok":
|
||||
return NextResponse.json(result.upload, { status: 200 });
|
||||
case "not_found":
|
||||
return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
case "invalid_status":
|
||||
return NextResponse.json(
|
||||
{ error: `Cannot remap upload in state '${result.current}'` },
|
||||
{ status: 409 }
|
||||
);
|
||||
case "invalid_mapping":
|
||||
return NextResponse.json({ error: result.reason }, { status: 422 });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to save column mapping:", error);
|
||||
return NextResponse.json({ error: "Internal server error" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
import { createS3Client, createRetrofitDataS3Client, retrofitDataS3Bucket } from "@/app/utils/s3";
|
||||
import * as XLSX from "xlsx";
|
||||
import { loadForAddressMatching, triggerAddressMatching } from "@/lib/bulkUpload/server";
|
||||
import { readSessionToken } from "@/lib/session";
|
||||
|
||||
const FIELD_RENAME: Record<string, string> = {
|
||||
address_1: "Address 1",
|
||||
address_2: "Address 2",
|
||||
address_3: "Address 3",
|
||||
postcode: "postcode",
|
||||
internal_reference: "Internal Reference",
|
||||
};
|
||||
|
||||
function transformFile(
|
||||
buffer: Buffer,
|
||||
columnMapping: Record<string, string>
|
||||
): { csv: string; error?: never } | { csv?: never; error: string } {
|
||||
const wb = XLSX.read(buffer, { type: "buffer" });
|
||||
const sheet = wb.Sheets[wb.SheetNames[0]];
|
||||
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: "" });
|
||||
|
||||
if (rows.length === 0) return { error: "Empty file" };
|
||||
|
||||
const sourceHeaders = Object.keys(rows[0]);
|
||||
const outputHeaders: string[] = [];
|
||||
const sourceToOutput: Record<string, string> = {};
|
||||
|
||||
for (const src of sourceHeaders) {
|
||||
const mapped = columnMapping[src];
|
||||
if (!mapped || mapped === "skip") continue;
|
||||
const renamed = FIELD_RENAME[mapped] ?? mapped;
|
||||
outputHeaders.push(renamed);
|
||||
sourceToOutput[src] = renamed;
|
||||
}
|
||||
|
||||
if (!outputHeaders.includes("Address 1"))
|
||||
return { error: 'Mapping must include "Address 1"' };
|
||||
if (!outputHeaders.includes("postcode"))
|
||||
return { error: 'Mapping must include "postcode"' };
|
||||
|
||||
const outputRows = rows.map((row) => {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [src, renamed] of Object.entries(sourceToOutput)) {
|
||||
out[renamed] = row[src] ?? "";
|
||||
}
|
||||
return out;
|
||||
});
|
||||
|
||||
const outSheet = XLSX.utils.json_to_sheet(outputRows, { header: outputHeaders });
|
||||
return { csv: XLSX.utils.sheet_to_csv(outSheet) };
|
||||
}
|
||||
|
||||
export async function POST(
|
||||
request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
|
||||
) {
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
|
||||
const { portfolioId, uploadId } = await params;
|
||||
|
||||
const guarded = await loadForAddressMatching(uploadId);
|
||||
switch (guarded.kind) {
|
||||
case "not_found":
|
||||
return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
case "wrong_state":
|
||||
return NextResponse.json(
|
||||
{ error: `Upload not ready for onboarding (state: ${guarded.current})` },
|
||||
{ status: 409 }
|
||||
);
|
||||
case "missing_mapping":
|
||||
return NextResponse.json({ error: "Column mapping missing" }, { status: 422 });
|
||||
}
|
||||
const upload = guarded.upload;
|
||||
|
||||
const s3 = createS3Client();
|
||||
const outputS3 = createRetrofitDataS3Client();
|
||||
const outputBucket = retrofitDataS3Bucket();
|
||||
|
||||
let fileBuffer: Buffer;
|
||||
try {
|
||||
const obj = await s3
|
||||
.getObject({ Bucket: upload.s3Bucket, Key: upload.s3Key })
|
||||
.promise();
|
||||
fileBuffer = Buffer.from(obj.Body as Uint8Array);
|
||||
} catch (err) {
|
||||
console.error("Failed to read source file from S3:", err);
|
||||
return NextResponse.json({ error: "Failed to read source file" }, { status: 500 });
|
||||
}
|
||||
|
||||
const transformed = transformFile(fileBuffer, upload.columnMapping!);
|
||||
if (transformed.error)
|
||||
return NextResponse.json({ error: transformed.error }, { status: 422 });
|
||||
|
||||
const transformedKey = `bulk_onboarding_inputs/${portfolioId}/${uploadId}.csv`;
|
||||
try {
|
||||
await outputS3
|
||||
.putObject({
|
||||
Bucket: outputBucket,
|
||||
Key: transformedKey,
|
||||
Body: transformed.csv,
|
||||
ContentType: "text/csv",
|
||||
})
|
||||
.promise();
|
||||
} catch (err) {
|
||||
console.error("Failed to upload transformed CSV:", err);
|
||||
return NextResponse.json({ error: "Failed to store transformed file" }, { status: 500 });
|
||||
}
|
||||
|
||||
const s3Uri = `s3://${outputBucket}/${transformedKey}`;
|
||||
|
||||
const trigger = await triggerAddressMatching({
|
||||
uploadId,
|
||||
s3Uri,
|
||||
sessionToken: readSessionToken(request),
|
||||
});
|
||||
if (trigger.kind === "trigger_failed")
|
||||
return NextResponse.json({ error: trigger.message }, { status: trigger.status });
|
||||
|
||||
return NextResponse.json({ taskId: trigger.taskId }, { status: 200 });
|
||||
}
|
||||
17
src/app/api/portfolio/[portfolioId]/bulk-uploads/route.ts
Normal file
17
src/app/api/portfolio/[portfolioId]/bulk-uploads/route.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import { listForPortfolio } from "@/lib/bulkUpload/server";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<{ portfolioId: string }> }
|
||||
) {
|
||||
const { portfolioId } = await params;
|
||||
|
||||
try {
|
||||
const uploads = await listForPortfolio(portfolioId);
|
||||
return NextResponse.json(uploads, { status: 200 });
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch bulk uploads:", error);
|
||||
return NextResponse.json({ error: "Internal server error" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
40
src/app/api/tasks/[taskId]/summary/route.ts
Normal file
40
src/app/api/tasks/[taskId]/summary/route.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { tasks } from "@/app/db/schema/tasks/tasks";
|
||||
import { subTasks } from "@/app/db/schema/tasks/subtask";
|
||||
import { eq, count, sql } from "drizzle-orm";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<{ taskId: string }> }
|
||||
) {
|
||||
const { taskId } = await params;
|
||||
|
||||
try {
|
||||
const [row] = await db
|
||||
.select({
|
||||
id: tasks.id,
|
||||
taskSource: tasks.taskSource,
|
||||
status: tasks.status,
|
||||
service: tasks.service,
|
||||
jobStarted: tasks.jobStarted,
|
||||
jobCompleted: tasks.jobCompleted,
|
||||
updatedAt: tasks.updatedAt,
|
||||
totalSubtasks: count(subTasks.id),
|
||||
completedSubtasks: sql<number>`count(case when lower(${subTasks.status}) in ('completed', 'complete') then 1 end)::int`,
|
||||
failedSubtasks: sql<number>`count(case when lower(${subTasks.status}) in ('failed', 'failure', 'error') then 1 end)::int`,
|
||||
})
|
||||
.from(tasks)
|
||||
.leftJoin(subTasks, eq(subTasks.taskId, tasks.id))
|
||||
.where(eq(tasks.id, taskId))
|
||||
.groupBy(tasks.id)
|
||||
.limit(1);
|
||||
|
||||
if (!row) return NextResponse.json({ error: "Not found" }, { status: 404 });
|
||||
|
||||
return NextResponse.json(row);
|
||||
} catch (error) {
|
||||
console.error("Error fetching task summary:", error);
|
||||
return NextResponse.json({ error: "Failed to fetch task summary" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
55
src/app/api/upload/bulk-addresses/confirm/route.ts
Normal file
55
src/app/api/upload/bulk-addresses/confirm/route.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
|
||||
const BodySchema = z.object({
|
||||
fileKey: z.string(),
|
||||
filename: z.string(),
|
||||
portfolioId: z.string(),
|
||||
userId: z.string(),
|
||||
sourceHeaders: z
|
||||
.array(z.union([z.string(), z.null(), z.undefined()]))
|
||||
.default([])
|
||||
.transform((arr) =>
|
||||
arr.filter((h): h is string => typeof h === "string" && h.trim().length > 0)
|
||||
),
|
||||
});
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let body;
|
||||
try {
|
||||
body = BodySchema.parse(await request.json());
|
||||
} catch (error) {
|
||||
console.error("Invalid input:", error);
|
||||
return NextResponse.json({ error: "Invalid input" }, { status: 400 });
|
||||
}
|
||||
|
||||
const bucket = process.env.RETROFIT_PLAN_INPUT_BUCKET_NAME;
|
||||
if (!bucket) {
|
||||
console.error("RETROFIT_PLAN_INPUT_BUCKET_NAME not set");
|
||||
return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 });
|
||||
}
|
||||
|
||||
try {
|
||||
const [record] = await db
|
||||
.insert(bulkAddressUploads)
|
||||
.values({
|
||||
portfolioId: body.portfolioId,
|
||||
userId: body.userId,
|
||||
s3Bucket: bucket,
|
||||
s3Key: body.fileKey,
|
||||
filename: body.filename,
|
||||
sourceHeaders: body.sourceHeaders,
|
||||
})
|
||||
.returning();
|
||||
|
||||
return NextResponse.json(
|
||||
{ id: record.id, s3Key: record.s3Key, s3Bucket: record.s3Bucket, status: record.status },
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Failed to record upload:", error);
|
||||
return NextResponse.json({ error: "Internal server error" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
36
src/app/api/upload/bulk-addresses/route.ts
Normal file
36
src/app/api/upload/bulk-addresses/route.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { createS3Client } from "@/app/utils/s3";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
|
||||
const BodySchema = z.object({
|
||||
userId: z.string(),
|
||||
portfolioId: z.string(),
|
||||
fileKey: z.string(),
|
||||
contentType: z.string(),
|
||||
});
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let body;
|
||||
try {
|
||||
body = BodySchema.parse(await request.json());
|
||||
} catch (error) {
|
||||
console.error("Invalid input:", error);
|
||||
return NextResponse.json({ error: "Invalid input" }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const s3 = createS3Client();
|
||||
|
||||
const preSignedUrl = await s3.getSignedUrlPromise("putObject", {
|
||||
Bucket: process.env.RETROFIT_PLAN_INPUT_BUCKET_NAME,
|
||||
Key: body.fileKey,
|
||||
ContentType: body.contentType,
|
||||
Expires: 5 * 60,
|
||||
});
|
||||
|
||||
return NextResponse.json({ url: preSignedUrl }, { status: 200 });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return NextResponse.json({ error: "Internal server error" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -35,6 +35,16 @@ export default function AddNew({
|
|||
router.push(`/portfolio/${portfolioId}/remote-assessment`);
|
||||
}
|
||||
|
||||
function handleBulkUploadClick() {
|
||||
const pw = window.prompt("Enter password to access bulk upload");
|
||||
if (pw === null) return;
|
||||
if (pw === "domnatechteamonly") {
|
||||
setIsBulkUploadOpen(true);
|
||||
} else {
|
||||
window.alert("Incorrect password");
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<BulkUploadComingSoonModal
|
||||
|
|
@ -118,7 +128,7 @@ export default function AddNew({
|
|||
<MenuItem>
|
||||
{({ active }) => (
|
||||
<button
|
||||
onClick={() => setIsBulkUploadOpen(true)}
|
||||
onClick={handleBulkUploadClick}
|
||||
className={cn(
|
||||
"w-full p-3 rounded-lg text-left flex gap-3 transition-colors",
|
||||
active && "bg-gray-100"
|
||||
|
|
|
|||
|
|
@ -4,11 +4,33 @@ import {
|
|||
Dialog,
|
||||
DialogBackdrop,
|
||||
DialogPanel,
|
||||
DialogTitle,
|
||||
Transition,
|
||||
TransitionChild,
|
||||
} from "@headlessui/react";
|
||||
import { Fragment } from "react";
|
||||
import { XMarkIcon, RectangleStackIcon } from "@heroicons/react/24/outline";
|
||||
import { Fragment, useRef, useState, DragEvent } from "react";
|
||||
import * as XLSX from "xlsx";
|
||||
import {
|
||||
XMarkIcon,
|
||||
DocumentTextIcon,
|
||||
ArrowDownTrayIcon,
|
||||
CloudArrowUpIcon,
|
||||
InformationCircleIcon,
|
||||
ArrowRightIcon,
|
||||
CheckCircleIcon,
|
||||
ExclamationCircleIcon,
|
||||
} from "@heroicons/react/24/outline";
|
||||
import { useSession } from "next-auth/react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useCreateBulkUpload } from "@/lib/bulkUpload/client";
|
||||
|
||||
const MAX_FILE_SIZE_MB = 50;
|
||||
const ALLOWED_EXTENSIONS = [".csv", ".xlsx", ".xls"];
|
||||
const CONTENT_TYPES: Record<string, string> = {
|
||||
".csv": "text/csv",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".xls": "application/vnd.ms-excel",
|
||||
};
|
||||
|
||||
interface BulkUploadComingSoonModalProps {
|
||||
isOpen: boolean;
|
||||
|
|
@ -16,13 +38,180 @@ interface BulkUploadComingSoonModalProps {
|
|||
portfolioId: string;
|
||||
}
|
||||
|
||||
function downloadTemplate() {
|
||||
const ws = XLSX.utils.aoa_to_sheet([["Internal Reference (Optional)", "Address", "Postcode"]]);
|
||||
const wb = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(wb, ws, "Properties");
|
||||
XLSX.writeFile(wb, "bulk_upload_template.xlsx");
|
||||
}
|
||||
|
||||
function getFileExtension(filename: string): string {
|
||||
return filename.slice(filename.lastIndexOf(".")).toLowerCase();
|
||||
}
|
||||
|
||||
function generateS3Key(userId: string, portfolioId: string, ext: string): string {
|
||||
const timestamp = new Date().toISOString().replace(/[:.-]/g, "");
|
||||
return `bulk-addresses/${userId}/${portfolioId}/${timestamp}/addresses${ext}`;
|
||||
}
|
||||
|
||||
function validateFile(file: File): string | null {
|
||||
const sizeMB = file.size / (1024 * 1024);
|
||||
if (sizeMB > MAX_FILE_SIZE_MB) {
|
||||
return `File too large. Max ${MAX_FILE_SIZE_MB}MB.`;
|
||||
}
|
||||
const ext = getFileExtension(file.name);
|
||||
if (!ALLOWED_EXTENSIONS.includes(ext)) {
|
||||
return "Only CSV or Excel files allowed.";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function validateHeaders(file: File): Promise<{ error: string | null; headers: string[] }> {
|
||||
const ext = getFileExtension(file.name);
|
||||
let headers: string[] = [];
|
||||
|
||||
if (ext === ".csv") {
|
||||
const text = await file.text();
|
||||
const firstLine = text.split(/\r?\n/)[0] ?? "";
|
||||
headers = firstLine.split(",").map((h) => h.trim().replace(/^["']|["']$/g, ""));
|
||||
} else {
|
||||
const buffer = await file.arrayBuffer();
|
||||
const wb = XLSX.read(buffer, { sheetRows: 1 });
|
||||
const sheet = wb.Sheets[wb.SheetNames[0]];
|
||||
const rows = XLSX.utils.sheet_to_json<unknown[]>(sheet, { header: 1, defval: "" });
|
||||
headers = ((rows[0] as unknown[]) ?? []).map((h) => String(h ?? "").trim());
|
||||
}
|
||||
|
||||
headers = headers.filter((h) => h.length > 0);
|
||||
|
||||
const normalised = headers.map((h) => h.toLowerCase());
|
||||
const hasAddress = normalised.some((h) => h.startsWith("address"));
|
||||
const hasPostcode = normalised.some((h) => h === "postcode");
|
||||
|
||||
if (!hasAddress && !hasPostcode) {
|
||||
return { error: "Missing required columns: Address and Postcode.", headers };
|
||||
}
|
||||
if (!hasAddress) {
|
||||
return { error: "Missing required column: Address (or Address 1, Address 2, etc.).", headers };
|
||||
}
|
||||
if (!hasPostcode) {
|
||||
return { error: 'Missing required column: "Postcode".', headers };
|
||||
}
|
||||
return { error: null, headers };
|
||||
}
|
||||
|
||||
export default function BulkUploadComingSoonModal({
|
||||
isOpen,
|
||||
onClose,
|
||||
portfolioId,
|
||||
}: BulkUploadComingSoonModalProps) {
|
||||
const session = useSession();
|
||||
const router = useRouter();
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const createUpload = useCreateBulkUpload();
|
||||
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [selectedFile, setSelectedFile] = useState<File | null>(null);
|
||||
const [sourceHeaders, setSourceHeaders] = useState<string[]>([]);
|
||||
const [validationError, setValidationError] = useState<string | null>(null);
|
||||
const [validating, setValidating] = useState(false);
|
||||
const [uploadProgress, setUploadProgress] = useState<number | null>(null);
|
||||
|
||||
const uploading = createUpload.isPending;
|
||||
const uploadError = createUpload.error
|
||||
? "Upload failed. Please try again, or contact a Domna representative if the issue persists."
|
||||
: null;
|
||||
|
||||
async function handleFile(file: File) {
|
||||
createUpload.reset();
|
||||
setSelectedFile(null);
|
||||
setValidationError(null);
|
||||
|
||||
const sizeOrTypeError = validateFile(file);
|
||||
if (sizeOrTypeError) {
|
||||
setValidationError(sizeOrTypeError);
|
||||
return;
|
||||
}
|
||||
|
||||
setValidating(true);
|
||||
const { error: headerError, headers } = await validateHeaders(file);
|
||||
setValidating(false);
|
||||
|
||||
if (headerError) {
|
||||
setValidationError(headerError);
|
||||
return;
|
||||
}
|
||||
|
||||
setSourceHeaders(headers);
|
||||
setSelectedFile(file);
|
||||
}
|
||||
|
||||
function handleDragOver(e: DragEvent<HTMLDivElement>) {
|
||||
e.preventDefault();
|
||||
setIsDragging(true);
|
||||
}
|
||||
|
||||
function handleDragLeave() {
|
||||
setIsDragging(false);
|
||||
}
|
||||
|
||||
function handleDrop(e: DragEvent<HTMLDivElement>) {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
const file = e.dataTransfer.files[0];
|
||||
if (file) handleFile(file);
|
||||
}
|
||||
|
||||
function handleInputChange(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
if (file) handleFile(file);
|
||||
}
|
||||
|
||||
function handleClose() {
|
||||
setSelectedFile(null);
|
||||
setSourceHeaders([]);
|
||||
setValidationError(null);
|
||||
setValidating(false);
|
||||
setUploadProgress(null);
|
||||
createUpload.reset();
|
||||
onClose();
|
||||
}
|
||||
|
||||
function handleUpload() {
|
||||
const userId = String(session.data?.user?.dbId ?? "");
|
||||
if (!selectedFile || !userId) return;
|
||||
|
||||
const ext = getFileExtension(selectedFile.name);
|
||||
const contentType = CONTENT_TYPES[ext] ?? "application/octet-stream";
|
||||
const fileKey = generateS3Key(userId, portfolioId, ext);
|
||||
setUploadProgress(0);
|
||||
|
||||
createUpload.mutate(
|
||||
{
|
||||
file: selectedFile,
|
||||
portfolioId,
|
||||
userId,
|
||||
sourceHeaders,
|
||||
contentType,
|
||||
fileKey,
|
||||
onProgress: setUploadProgress,
|
||||
},
|
||||
{
|
||||
onSuccess: ({ id: uploadId }) => {
|
||||
router.push(`/portfolio/${portfolioId}/bulk-upload/${uploadId}/map-columns`);
|
||||
onClose();
|
||||
},
|
||||
onSettled: () => setUploadProgress(null),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const canUpload = !!selectedFile && !uploading && !validating;
|
||||
|
||||
return (
|
||||
<Transition show={isOpen} as={Fragment}>
|
||||
<Dialog as="div" className="relative z-50" onClose={onClose}>
|
||||
<Transition appear show={isOpen} as={Fragment}>
|
||||
<Dialog as="div" className="relative z-[9999]" onClose={handleClose}>
|
||||
{/* Backdrop */}
|
||||
<TransitionChild
|
||||
as={Fragment}
|
||||
enter="ease-out duration-200"
|
||||
|
|
@ -32,52 +221,204 @@ export default function BulkUploadComingSoonModal({
|
|||
leaveFrom="opacity-100"
|
||||
leaveTo="opacity-0"
|
||||
>
|
||||
<DialogBackdrop className="fixed inset-0 bg-black/30 backdrop-blur-sm" />
|
||||
<DialogBackdrop className="fixed inset-0 bg-gray-900/40 backdrop-blur-sm" />
|
||||
</TransitionChild>
|
||||
|
||||
{/* Panel */}
|
||||
<div className="fixed inset-0 flex items-center justify-center p-4">
|
||||
<TransitionChild
|
||||
as={Fragment}
|
||||
enter="ease-out duration-200"
|
||||
enterFrom="opacity-0 scale-95"
|
||||
enterTo="opacity-100 scale-100"
|
||||
enterFrom="opacity-0 scale-95 translate-y-2"
|
||||
enterTo="opacity-100 scale-100 translate-y-0"
|
||||
leave="ease-in duration-150"
|
||||
leaveFrom="opacity-100 scale-100"
|
||||
leaveTo="opacity-0 scale-95"
|
||||
leaveFrom="opacity-100 scale-100 translate-y-0"
|
||||
leaveTo="opacity-0 scale-95 translate-y-2"
|
||||
>
|
||||
<DialogPanel className="w-full max-w-md bg-white rounded-2xl shadow-xl p-8 relative">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="absolute top-4 right-4 p-1.5 rounded-lg text-gray-400 hover:text-gray-700 hover:bg-gray-100 transition-colors"
|
||||
>
|
||||
<XMarkIcon className="h-5 w-5" />
|
||||
</button>
|
||||
|
||||
<div className="flex flex-col items-center text-center gap-4">
|
||||
<div className="w-14 h-14 rounded-2xl bg-amber-50 flex items-center justify-center">
|
||||
<RectangleStackIcon className="h-7 w-7 text-amber-500" />
|
||||
</div>
|
||||
<DialogPanel className="w-full max-w-2xl bg-white rounded-2xl shadow-[0_40px_60px_-15px_rgba(21,29,33,0.15)] overflow-hidden flex flex-col">
|
||||
|
||||
{/* Header */}
|
||||
<div className="px-10 pt-10 pb-6 flex justify-between items-start">
|
||||
<div>
|
||||
<span className="text-[11px] font-semibold text-amber-700 bg-amber-100 px-2 py-0.5 rounded-full">
|
||||
Coming Soon
|
||||
</span>
|
||||
<h2 className="mt-3 text-2xl font-extrabold text-gray-900 tracking-tight">
|
||||
Bulk Address Upload
|
||||
</h2>
|
||||
<p className="mt-2 text-sm text-gray-500 leading-relaxed">
|
||||
Upload multiple addresses in one go. This feature is currently in development
|
||||
and will be available soon.
|
||||
<DialogTitle className="text-2xl font-extrabold text-gray-900 tracking-tight mb-2">
|
||||
Bulk Upload: New Properties
|
||||
</DialogTitle>
|
||||
<p className="text-sm text-gray-500 leading-relaxed max-w-md">
|
||||
This workflow is designed for adding new residential or commercial
|
||||
assets to your portfolio. Upload your dataset to begin the
|
||||
transformation.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="mt-2 px-6 py-2.5 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold hover:opacity-90 transition-opacity"
|
||||
onClick={handleClose}
|
||||
className="p-2 hover:bg-gray-100 rounded-full transition-colors ml-4 shrink-0"
|
||||
>
|
||||
Got it
|
||||
<XMarkIcon className="h-5 w-5 text-gray-400" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="px-10 py-2 space-y-5">
|
||||
|
||||
{/* Template section */}
|
||||
<div className="bg-gray-50 p-5 rounded-xl flex items-center justify-between">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="w-11 h-11 rounded-lg bg-gray-100 flex items-center justify-center shrink-0">
|
||||
<DocumentTextIcon className="h-6 w-6 text-midblue" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-gray-900">Required Template Format</p>
|
||||
<p className="text-xs text-gray-500 mt-0.5">
|
||||
Must contain:{" "}
|
||||
<span className="font-medium text-midblue">
|
||||
Address, Postcode
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={downloadTemplate}
|
||||
className="flex items-center gap-1.5 text-xs font-semibold text-midblue hover:text-gray-900 transition-colors shrink-0 ml-4"
|
||||
>
|
||||
<ArrowDownTrayIcon className="h-4 w-4" />
|
||||
Download template
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Dropzone */}
|
||||
<div
|
||||
onClick={() => !uploading && fileInputRef.current?.click()}
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
className={`border-2 border-dashed rounded-2xl p-12 flex flex-col items-center justify-center transition-colors ${
|
||||
uploading || validating
|
||||
? "border-gray-200 bg-gray-50 cursor-default"
|
||||
: validationError
|
||||
? "border-red-300 bg-red-50 cursor-pointer"
|
||||
: isDragging
|
||||
? "border-midblue bg-blue-50 cursor-copy"
|
||||
: selectedFile
|
||||
? "border-green-400 bg-green-50 cursor-pointer"
|
||||
: "border-gray-200 hover:border-gray-300 hover:bg-gray-50 cursor-pointer"
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".csv,.xlsx,.xls"
|
||||
className="hidden"
|
||||
onChange={handleInputChange}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
/>
|
||||
|
||||
{validating ? (
|
||||
<>
|
||||
<div className="w-14 h-14 rounded-full bg-gray-100 flex items-center justify-center mb-4">
|
||||
<span className="h-6 w-6 rounded-full border-2 border-midblue border-t-transparent animate-spin" />
|
||||
</div>
|
||||
<p className="text-base font-bold text-gray-900 mb-1">Checking headers…</p>
|
||||
<p className="text-xs text-gray-400">Validating column structure</p>
|
||||
</>
|
||||
) : uploading ? (
|
||||
<>
|
||||
<div className="w-14 h-14 rounded-full bg-gray-100 flex items-center justify-center mb-4">
|
||||
<CloudArrowUpIcon className="h-7 w-7 text-midblue" />
|
||||
</div>
|
||||
<p className="text-base font-bold text-gray-900 mb-1">Uploading…</p>
|
||||
<p className="text-xs text-gray-400 mb-4">{selectedFile?.name}</p>
|
||||
<div className="w-full max-w-xs bg-gray-200 rounded-full h-2">
|
||||
<div
|
||||
className="bg-midblue h-2 rounded-full transition-all duration-200"
|
||||
style={{ width: `${uploadProgress ?? 0}%` }}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-xs text-gray-400 mt-1">{uploadProgress ?? 0}%</p>
|
||||
</>
|
||||
) : validationError ? (
|
||||
<>
|
||||
<ExclamationCircleIcon className="h-14 w-14 text-red-400 mb-4" />
|
||||
<p className="text-base font-bold text-red-600 mb-1">{validationError}</p>
|
||||
<p className="text-xs text-gray-400">Click to choose a different file</p>
|
||||
</>
|
||||
) : selectedFile ? (
|
||||
<>
|
||||
<CheckCircleIcon className="h-14 w-14 text-green-400 mb-4" />
|
||||
<p className="text-base font-bold text-gray-900 mb-1">{selectedFile.name}</p>
|
||||
<p className="text-xs text-gray-400">Click to change file</p>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<div className="w-14 h-14 rounded-full bg-gray-100 flex items-center justify-center mb-4">
|
||||
<CloudArrowUpIcon className="h-7 w-7 text-midblue" />
|
||||
</div>
|
||||
<p className="text-base font-bold text-gray-900 mb-1">
|
||||
Drag and drop CSV or XLSX
|
||||
</p>
|
||||
<p className="text-xs text-gray-400">
|
||||
or <span className="text-midblue font-semibold">click to browse</span> · Max {MAX_FILE_SIZE_MB}MB
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Upload error */}
|
||||
{uploadError && (
|
||||
<p className="text-xs text-red-500 flex items-center gap-1.5">
|
||||
<ExclamationCircleIcon className="h-4 w-4 shrink-0" />
|
||||
{uploadError}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Info strip */}
|
||||
<div className="flex items-center gap-3 text-xs text-gray-500 bg-gray-50 px-4 py-3 rounded-lg">
|
||||
<InformationCircleIcon className="h-4 w-4 text-midblue shrink-0" />
|
||||
<span>
|
||||
Properties will be automatically validated against national
|
||||
architectural databases.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="px-10 py-7 mt-4 flex items-center justify-between bg-gray-50/50">
|
||||
<div className="flex items-center gap-4">
|
||||
<button
|
||||
onClick={handleClose}
|
||||
className="text-sm font-semibold text-gray-400 hover:text-gray-700 transition-colors"
|
||||
>
|
||||
Cancel and Exit
|
||||
</button>
|
||||
<button
|
||||
onClick={() => { handleClose(); router.push(`/portfolio/${portfolioId}/bulk-upload`); }}
|
||||
className="text-sm font-semibold text-midblue hover:text-gray-900 transition-colors"
|
||||
>
|
||||
View previous uploads
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<button
|
||||
onClick={handleUpload}
|
||||
disabled={!canUpload}
|
||||
className={`flex items-center gap-2 px-7 py-2.5 rounded-2xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold transition-opacity ${
|
||||
canUpload ? "opacity-100 hover:opacity-90" : "opacity-40 cursor-not-allowed"
|
||||
}`}
|
||||
>
|
||||
{uploading ? (
|
||||
<>
|
||||
<span className="h-4 w-4 rounded-full border-2 border-white border-t-transparent animate-spin" />
|
||||
Uploading…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Upload File
|
||||
<ArrowRightIcon className="h-4 w-4" />
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</DialogPanel>
|
||||
</TransitionChild>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
ALTER TABLE "bulk_address_uploads" ADD COLUMN "task_id" uuid;--> statement-breakpoint
|
||||
ALTER TABLE "bulk_address_uploads" ADD COLUMN "combined_output_s3_uri" text;
|
||||
|
|
@ -303,6 +303,18 @@
|
|||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"task_id": {
|
||||
"name": "task_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"combined_output_s3_uri": {
|
||||
"name": "combined_output_s3_uri",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
|
|
@ -6467,8 +6479,8 @@
|
|||
"schema": "public",
|
||||
"values": [
|
||||
"none",
|
||||
"cladded with “sufficient space to fill the wall”",
|
||||
"cladded with “insufficient space to fill the wall”"
|
||||
"cladded with \u201csufficient space to fill the wall\u201d",
|
||||
"cladded with \u201cinsufficient space to fill the wall\u201d"
|
||||
]
|
||||
},
|
||||
"public.inspections_insulation_material": {
|
||||
|
|
@ -6495,8 +6507,8 @@
|
|||
"schema": "public",
|
||||
"values": [
|
||||
"no render",
|
||||
"rendered with “insufficient” space between dpc and render",
|
||||
"rendered with “sufficient” space between dpc and render"
|
||||
"rendered with \u201cinsufficient\u201d space between dpc and render",
|
||||
"rendered with \u201csufficient\u201d space between dpc and render"
|
||||
]
|
||||
},
|
||||
"public.inspections_roof_orientation": {
|
||||
|
|
@ -6850,13 +6862,13 @@
|
|||
"schema": "public",
|
||||
"values": [
|
||||
"1",
|
||||
"2–5",
|
||||
"6–20",
|
||||
"2\u20135",
|
||||
"6\u201320",
|
||||
"21+",
|
||||
"1–50",
|
||||
"51–100",
|
||||
"101–300",
|
||||
"301–1000",
|
||||
"1\u201350",
|
||||
"51\u2013100",
|
||||
"101\u2013300",
|
||||
"301\u20131000",
|
||||
"1000+"
|
||||
]
|
||||
},
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -386,6 +386,7 @@ export interface PropertyWithRelations extends Record<string, unknown> {
|
|||
totalFloorArea: number | null;
|
||||
co2Emissions: number | null;
|
||||
mainfuel: string | null;
|
||||
lexiscore: number | null;
|
||||
}
|
||||
|
||||
export type NonIntrusiveSurveyNotes = InferModel<
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
"use client";
|
||||
|
||||
import { useRouter } from "next/navigation";
|
||||
import Link from "next/link";
|
||||
import { ArrowRightIcon } from "@heroicons/react/24/outline";
|
||||
import {
|
||||
useBulkUploadProgress,
|
||||
useFinalize,
|
||||
useRequestCombine,
|
||||
} from "@/lib/bulkUpload/client";
|
||||
|
||||
interface Props {
|
||||
portfolioSlug: string;
|
||||
portfolioId: string;
|
||||
uploadId: string;
|
||||
isDomnaUser: boolean;
|
||||
}
|
||||
|
||||
const TASK_TERMINAL_STATUSES = new Set(["complete", "completed", "failed", "failure", "error"]);
|
||||
const TASK_FAILED_STATUSES = new Set(["failed", "failure", "error"]);
|
||||
|
||||
export default function OnboardingProgress({
|
||||
portfolioSlug,
|
||||
portfolioId,
|
||||
uploadId,
|
||||
isDomnaUser,
|
||||
}: Props) {
|
||||
const router = useRouter();
|
||||
const progress = useBulkUploadProgress(portfolioId, uploadId);
|
||||
const combine = useRequestCombine(portfolioId, uploadId);
|
||||
const finalize = useFinalize(portfolioId, uploadId);
|
||||
|
||||
if (progress.isError) return null;
|
||||
if (!progress.data) {
|
||||
return (
|
||||
<div className="mt-4 flex items-center gap-2 text-sm text-gray-400">
|
||||
<span className="w-4 h-4 rounded-full border-2 border-gray-300 border-t-transparent animate-spin" />
|
||||
Loading progress…
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const { task, upload } = progress.data;
|
||||
const total = task?.totalSubtasks ?? 0;
|
||||
const completedSubtasks = task?.completedSubtasks ?? 0;
|
||||
const failedSubtasks = task?.failedSubtasks ?? 0;
|
||||
const percent = total > 0 ? Math.round((completedSubtasks / total) * 100) : 0;
|
||||
|
||||
const taskStatus = task?.status.toLowerCase() ?? "";
|
||||
const taskDone = TASK_TERMINAL_STATUSES.has(taskStatus);
|
||||
const taskFailed = TASK_FAILED_STATUSES.has(taskStatus);
|
||||
const isCombining = upload.status === "combining";
|
||||
const isImporting = upload.status === "awaiting_review";
|
||||
|
||||
const canRunCombiner = taskDone && !taskFailed && upload.status === "processing";
|
||||
const canFinalize = upload.status === "awaiting_review";
|
||||
|
||||
return (
|
||||
<div className="mt-6 space-y-3">
|
||||
<div className="w-full bg-gray-100 rounded-full h-2 overflow-hidden">
|
||||
<div
|
||||
className={`h-2 rounded-full transition-all duration-500 ${taskFailed ? "bg-red-400" : "bg-[#14163d]"}`}
|
||||
style={{ width: total > 0 ? `${percent}%` : "4%" }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-4 text-xs text-gray-500">
|
||||
{total > 0 && (
|
||||
<span>
|
||||
<span className="font-semibold text-gray-700">{completedSubtasks}</span> / {total} batches complete
|
||||
</span>
|
||||
)}
|
||||
{failedSubtasks > 0 && (
|
||||
<span className="flex items-center gap-1 text-red-500 font-semibold">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-red-400" />
|
||||
{failedSubtasks} failed
|
||||
</span>
|
||||
)}
|
||||
{!taskDone && (
|
||||
<span className="flex items-center gap-1 text-blue-500">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-blue-400 animate-pulse" />
|
||||
Running
|
||||
</span>
|
||||
)}
|
||||
{isCombining && (
|
||||
<span className="flex items-center gap-1 text-blue-500">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-blue-400 animate-pulse" />
|
||||
Combining results…
|
||||
</span>
|
||||
)}
|
||||
{isImporting && (
|
||||
<span className="flex items-center gap-1 text-blue-500">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-blue-400 animate-pulse" />
|
||||
Awaiting import
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{(canRunCombiner || canFinalize) && (
|
||||
<div className="flex flex-col gap-2 pt-2">
|
||||
{canRunCombiner && (
|
||||
<StageButton
|
||||
label="Run Combiner"
|
||||
activeLabel="Starting combiner…"
|
||||
isPending={combine.isPending}
|
||||
onClick={() => combine.mutate()}
|
||||
/>
|
||||
)}
|
||||
{canFinalize && (
|
||||
<StageButton
|
||||
label="Finalise"
|
||||
activeLabel="Finalising…"
|
||||
isPending={finalize.isPending}
|
||||
onClick={() =>
|
||||
finalize.mutate(undefined, { onSuccess: () => router.refresh() })
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{combine.error && (
|
||||
<p className="text-xs text-red-500">{combine.error.message}</p>
|
||||
)}
|
||||
{finalize.error && (
|
||||
<div className="rounded-lg border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700">
|
||||
<p className="font-semibold">Import failed</p>
|
||||
<p className="text-red-600 mt-0.5 break-words">{finalize.error.message}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isDomnaUser && (
|
||||
<Link
|
||||
href={`/portfolio/${portfolioSlug}/settings/logs`}
|
||||
className="text-xs text-gray-400 hover:text-gray-700 underline underline-offset-2 transition-colors"
|
||||
>
|
||||
View detailed logs
|
||||
</Link>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StageButton({
|
||||
label,
|
||||
activeLabel,
|
||||
isPending,
|
||||
onClick,
|
||||
}: {
|
||||
label: string;
|
||||
activeLabel: string;
|
||||
isPending: boolean;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
disabled={isPending}
|
||||
className={`inline-flex items-center gap-2 self-start px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold transition-opacity ${
|
||||
isPending ? "opacity-50 cursor-not-allowed" : "hover:opacity-90"
|
||||
}`}
|
||||
>
|
||||
{isPending ? (
|
||||
<>
|
||||
<span className="w-4 h-4 rounded-full border-2 border-white border-t-transparent animate-spin" />
|
||||
{activeLabel}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
{label}
|
||||
<ArrowRightIcon className="h-4 w-4" />
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
"use client";
|
||||
|
||||
import { useRouter } from "next/navigation";
|
||||
import { ArrowRightIcon } from "@heroicons/react/24/outline";
|
||||
import { useStartAddressMatching } from "@/lib/bulkUpload/client";
|
||||
|
||||
interface Props {
|
||||
portfolioId: string;
|
||||
uploadId: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
export default function StartAddressMatchingButton({ portfolioId, uploadId }: Props) {
|
||||
const router = useRouter();
|
||||
const { mutate, isPending, error } = useStartAddressMatching(portfolioId, uploadId);
|
||||
|
||||
return (
|
||||
<div className="mt-4">
|
||||
<button
|
||||
onClick={() => mutate(undefined, { onSuccess: () => router.refresh() })}
|
||||
disabled={isPending}
|
||||
className={`inline-flex items-center gap-2 px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold transition-opacity ${
|
||||
isPending ? "opacity-50 cursor-not-allowed" : "hover:opacity-90"
|
||||
}`}
|
||||
>
|
||||
{isPending ? (
|
||||
<>
|
||||
<span className="w-4 h-4 rounded-full border-2 border-white border-t-transparent animate-spin" />
|
||||
Starting…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Start Onboarding
|
||||
<ArrowRightIcon className="h-4 w-4" />
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
{error && (
|
||||
<p className="mt-2 text-xs text-red-500">
|
||||
{error.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
ArrowLeftIcon,
|
||||
ArrowRightIcon,
|
||||
TableCellsIcon,
|
||||
ArrowsRightLeftIcon,
|
||||
} from "@heroicons/react/24/outline";
|
||||
import { useSetColumnMapping } from "@/lib/bulkUpload/client";
|
||||
|
||||
const INTERNAL_FIELDS = [
|
||||
{ value: "address_1", label: "Address 1", required: true },
|
||||
{ value: "address_2", label: "Address 2", required: false },
|
||||
{ value: "address_3", label: "Address 3", required: false },
|
||||
{ value: "postcode", label: "Postcode", required: true },
|
||||
{ value: "internal_reference", label: "Internal Reference (Optional)", required: false },
|
||||
{ value: "skip", label: "Skip this column", required: false },
|
||||
];
|
||||
|
||||
const REQUIRED_VALUES = ["address_1", "postcode"];
|
||||
|
||||
function autoDetect(header: string): string {
|
||||
const h = header.toLowerCase().replace(/[\s_\-]/g, "");
|
||||
if (/^(address|addr)(line)?(1|one)?$/.test(h)) return "address_1";
|
||||
if (/^(address|addr)(line)?(2|two)|^street$/.test(h)) return "address_2";
|
||||
if (/^(address|addr)(line)?(3|three)|^locality$|^town$|^city$/.test(h)) return "address_3";
|
||||
if (/^post(al)?code$|^postcode$|^pcode$/.test(h)) return "postcode";
|
||||
if (/^(internal)?ref(erence)?$|^id$/.test(h)) return "internal_reference";
|
||||
return "skip";
|
||||
}
|
||||
|
||||
function buildInitialMapping(
|
||||
headers: string[],
|
||||
existing?: Record<string, string>
|
||||
): Record<string, string> {
|
||||
const mapping: Record<string, string> = {};
|
||||
for (const h of headers) {
|
||||
mapping[h] = existing?.[h] ?? autoDetect(h);
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
portfolioId: string;
|
||||
uploadId: string;
|
||||
filename: string;
|
||||
sourceHeaders: string[];
|
||||
existingMapping?: Record<string, string>;
|
||||
}
|
||||
|
||||
export default function MapColumnsClient({
|
||||
portfolioId,
|
||||
uploadId,
|
||||
filename,
|
||||
sourceHeaders,
|
||||
existingMapping,
|
||||
}: Props) {
|
||||
const router = useRouter();
|
||||
const [mapping, setMapping] = useState<Record<string, string>>(
|
||||
buildInitialMapping(sourceHeaders, existingMapping)
|
||||
);
|
||||
const setMappingMutation = useSetColumnMapping(portfolioId, uploadId);
|
||||
|
||||
const mappedValues = Object.values(mapping).filter((v) => v !== "skip");
|
||||
const missingRequired = REQUIRED_VALUES.filter((r) => !mappedValues.includes(r));
|
||||
const submitting = setMappingMutation.isPending;
|
||||
const error = setMappingMutation.error?.message ?? null;
|
||||
const canSubmit = missingRequired.length === 0 && !submitting;
|
||||
|
||||
function setField(header: string, value: string) {
|
||||
setMapping((prev) => ({ ...prev, [header]: value }));
|
||||
}
|
||||
|
||||
function handleSubmit() {
|
||||
if (!canSubmit) return;
|
||||
setMappingMutation.mutate(
|
||||
{ columnMapping: mapping },
|
||||
{
|
||||
onSuccess: () => {
|
||||
router.push(`/portfolio/${portfolioId}/bulk-upload/${uploadId}`);
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-4xl mx-auto px-6 py-10">
|
||||
{/* Breadcrumb + step */}
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<p className="text-xs font-semibold text-gray-400 uppercase tracking-widest">
|
||||
Bulk Uploads › Column Remapper
|
||||
</p>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
Step 2 of 3
|
||||
</span>
|
||||
<div className="flex gap-1 ml-2">
|
||||
{[1, 2, 3].map((s) => (
|
||||
<div
|
||||
key={s}
|
||||
className={`h-1.5 rounded-full ${
|
||||
s <= 2 ? "w-8 bg-[#14163d]" : "w-8 bg-gray-200"
|
||||
}`}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Header */}
|
||||
<div className="mb-8">
|
||||
<h1 className="text-3xl font-extrabold text-gray-900 tracking-tight mb-1">
|
||||
Column Remapper
|
||||
</h1>
|
||||
<p className="text-sm text-gray-500 max-w-lg">
|
||||
Align your spreadsheet headers with our internal property data structure to
|
||||
ensure accurate address processing.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Table */}
|
||||
<div className="bg-white border border-gray-100 rounded-2xl overflow-hidden shadow-sm mb-6">
|
||||
{/* Column headers */}
|
||||
<div className="grid grid-cols-12 items-center px-6 py-3 bg-gray-50 border-b border-gray-100">
|
||||
<span className="col-span-4 text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
Spreadsheet Header
|
||||
</span>
|
||||
<span className="col-span-1" />
|
||||
<span className="col-span-5 text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
Internal Field Mapping
|
||||
</span>
|
||||
<span className="col-span-2 text-xs font-semibold text-gray-400 uppercase tracking-wider text-right">
|
||||
Status
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{sourceHeaders.length === 0 ? (
|
||||
<div className="px-6 py-12 text-center text-sm text-gray-400">
|
||||
No headers found in this file.
|
||||
</div>
|
||||
) : (
|
||||
<div className="divide-y divide-gray-50">
|
||||
{sourceHeaders.map((header) => {
|
||||
const value = mapping[header] ?? "skip";
|
||||
const isMapped = value !== "skip";
|
||||
return (
|
||||
<div
|
||||
key={header}
|
||||
className="grid grid-cols-12 items-center px-6 py-4 hover:bg-gray-50/50 transition-colors"
|
||||
>
|
||||
{/* Source header */}
|
||||
<div className="col-span-4 flex items-center gap-3">
|
||||
<div className="w-8 h-8 rounded-lg bg-gray-100 flex items-center justify-center shrink-0">
|
||||
<TableCellsIcon className="h-4 w-4 text-gray-400" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-gray-900">{header}</p>
|
||||
<p className="text-xs text-gray-400">Source column</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Arrow */}
|
||||
<div className="col-span-1 flex justify-center">
|
||||
<ArrowsRightLeftIcon className="h-4 w-4 text-gray-300" />
|
||||
</div>
|
||||
|
||||
{/* Dropdown */}
|
||||
<div className="col-span-5">
|
||||
<select
|
||||
value={value}
|
||||
onChange={(e) => setField(header, e.target.value)}
|
||||
className="w-full text-sm border border-gray-200 rounded-lg px-3 py-2 bg-white text-gray-800 focus:outline-none focus:ring-2 focus:ring-[#14163d]/20 focus:border-[#14163d]"
|
||||
>
|
||||
{INTERNAL_FIELDS.map((f) => (
|
||||
<option key={f.value} value={f.value}>
|
||||
{f.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{/* Status badge */}
|
||||
<div className="col-span-2 flex justify-end">
|
||||
<span
|
||||
className={`inline-flex items-center gap-1 px-2.5 py-1 rounded-full text-xs font-semibold ${
|
||||
isMapped
|
||||
? "bg-amber-50 text-amber-700"
|
||||
: "bg-gray-100 text-gray-400"
|
||||
}`}
|
||||
>
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-current opacity-70" />
|
||||
{isMapped ? "Mapped" : "Skipped"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Validation error */}
|
||||
{missingRequired.length > 0 && (
|
||||
<p className="text-xs text-amber-600 mb-4">
|
||||
Required fields not yet mapped:{" "}
|
||||
{missingRequired
|
||||
.map((r) => INTERNAL_FIELDS.find((f) => f.value === r)?.label)
|
||||
.join(", ")}
|
||||
</p>
|
||||
)}
|
||||
{error && <p className="text-xs text-red-500 mb-4">{error}</p>}
|
||||
|
||||
{/* Footer */}
|
||||
<div className="flex items-center justify-between">
|
||||
<Link
|
||||
href={`/portfolio/${portfolioId}/bulk-upload`}
|
||||
className="flex items-center gap-1.5 text-sm font-semibold text-gray-400 hover:text-gray-700 transition-colors"
|
||||
>
|
||||
<ArrowLeftIcon className="h-4 w-4" />
|
||||
Back
|
||||
</Link>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<Link
|
||||
href={`/portfolio/${portfolioId}/bulk-upload`}
|
||||
className="text-sm font-semibold text-gray-400 hover:text-gray-700 transition-colors"
|
||||
>
|
||||
Cancel
|
||||
</Link>
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
disabled={!canSubmit}
|
||||
className={`flex items-center gap-2 px-7 py-2.5 rounded-2xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold transition-opacity ${
|
||||
canSubmit ? "opacity-100 hover:opacity-90" : "opacity-40 cursor-not-allowed"
|
||||
}`}
|
||||
>
|
||||
{submitting ? "Saving…" : "Process Mapping"}
|
||||
{!submitting && <ArrowRightIcon className="h-4 w-4" />}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Pro tip */}
|
||||
<div className="mt-10 bg-gray-50 rounded-2xl p-6">
|
||||
<p className="text-xs font-semibold text-midblue uppercase tracking-wider mb-2">
|
||||
Pro Tip
|
||||
</p>
|
||||
<p className="text-sm text-gray-500 italic">
|
||||
“Ensure your source file doesn't have blank headers. Any column mapped to
|
||||
“Skip” will be ignored during import.”
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
import { redirect, notFound } from "next/navigation";
|
||||
import MapColumnsClient from "./MapColumnsClient";
|
||||
|
||||
export default async function MapColumnsPage(props: {
|
||||
params: Promise<{ slug: string; uploadId: string }>;
|
||||
}) {
|
||||
const { slug, uploadId } = await props.params;
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) redirect("/login");
|
||||
|
||||
const [upload] = await db
|
||||
.select()
|
||||
.from(bulkAddressUploads)
|
||||
.where(eq(bulkAddressUploads.id, uploadId))
|
||||
.limit(1);
|
||||
|
||||
if (!upload) notFound();
|
||||
|
||||
return (
|
||||
<MapColumnsClient
|
||||
portfolioId={slug}
|
||||
uploadId={uploadId}
|
||||
filename={upload.filename}
|
||||
sourceHeaders={upload.sourceHeaders}
|
||||
existingMapping={upload.columnMapping ?? undefined}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
import { redirect, notFound } from "next/navigation";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
ArrowLeftIcon,
|
||||
ArrowRightIcon,
|
||||
CheckCircleIcon,
|
||||
ClockIcon,
|
||||
ExclamationCircleIcon,
|
||||
ArrowPathIcon,
|
||||
} from "@heroicons/react/24/outline";
|
||||
import StartAddressMatchingButton from "./StartAddressMatchingButton";
|
||||
import OnboardingProgress from "./OnboardingProgress";
|
||||
|
||||
function formatDate(date: Date) {
|
||||
return new Intl.DateTimeFormat("en-GB", {
|
||||
day: "2-digit",
|
||||
month: "short",
|
||||
year: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
}).format(date);
|
||||
}
|
||||
|
||||
const STATUS_CONFIG = {
|
||||
ready_for_processing: {
|
||||
icon: ClockIcon,
|
||||
iconBg: "bg-amber-50",
|
||||
iconColor: "text-amber-500",
|
||||
title: "Awaiting column mapping",
|
||||
body: "Map your spreadsheet columns to our internal fields before processing can begin.",
|
||||
cta: true,
|
||||
},
|
||||
mapping_complete: {
|
||||
icon: CheckCircleIcon,
|
||||
iconBg: "bg-blue-50",
|
||||
iconColor: "text-blue-500",
|
||||
title: "Mapping complete",
|
||||
body: "Column mapping saved. Start onboarding to begin matching your addresses to UPRNs.",
|
||||
cta: true,
|
||||
},
|
||||
processing: {
|
||||
icon: ArrowPathIcon,
|
||||
iconBg: "bg-blue-50",
|
||||
iconColor: "text-blue-500",
|
||||
title: "Processing…",
|
||||
body: "Your file is currently being processed. This may take a few minutes.",
|
||||
cta: false,
|
||||
},
|
||||
combining: {
|
||||
icon: ArrowPathIcon,
|
||||
iconBg: "bg-blue-50",
|
||||
iconColor: "text-blue-500",
|
||||
title: "Combining results…",
|
||||
body: "Your matched addresses are being assembled. Almost ready for review.",
|
||||
cta: false,
|
||||
},
|
||||
awaiting_review: {
|
||||
icon: ArrowPathIcon,
|
||||
iconBg: "bg-blue-50",
|
||||
iconColor: "text-blue-500",
|
||||
title: "Importing addresses…",
|
||||
body: "Matches ready, writing into your portfolio.",
|
||||
cta: false,
|
||||
},
|
||||
complete: {
|
||||
icon: CheckCircleIcon,
|
||||
iconBg: "bg-green-50",
|
||||
iconColor: "text-green-500",
|
||||
title: "Processing complete",
|
||||
body: "All addresses have been imported into your portfolio.",
|
||||
cta: false,
|
||||
},
|
||||
failed: {
|
||||
icon: ExclamationCircleIcon,
|
||||
iconBg: "bg-red-50",
|
||||
iconColor: "text-red-500",
|
||||
title: "Processing failed",
|
||||
body: "Something went wrong during processing. Contact a Domna representative for assistance.",
|
||||
cta: false,
|
||||
},
|
||||
} as const;
|
||||
|
||||
export default async function BulkUploadDetailPage(props: {
|
||||
params: Promise<{ slug: string; uploadId: string }>;
|
||||
}) {
|
||||
const { slug, uploadId } = await props.params;
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) redirect("/login");
|
||||
const isDomnaUser = !!session.user?.email?.endsWith("@domna.homes");
|
||||
|
||||
const [upload] = await db
|
||||
.select()
|
||||
.from(bulkAddressUploads)
|
||||
.where(eq(bulkAddressUploads.id, uploadId))
|
||||
.limit(1);
|
||||
|
||||
if (!upload) notFound();
|
||||
|
||||
const statusKey = upload.status as keyof typeof STATUS_CONFIG;
|
||||
const config = STATUS_CONFIG[statusKey] ?? STATUS_CONFIG.ready_for_processing;
|
||||
const Icon = config.icon;
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto px-6 py-10">
|
||||
{/* Back */}
|
||||
<Link
|
||||
href={`/portfolio/${slug}/bulk-upload`}
|
||||
className="inline-flex items-center gap-1.5 text-sm text-gray-400 hover:text-gray-700 transition-colors mb-8"
|
||||
>
|
||||
<ArrowLeftIcon className="h-4 w-4" />
|
||||
Back to uploads
|
||||
</Link>
|
||||
|
||||
{/* Header */}
|
||||
<div className="mb-8">
|
||||
<p className="text-xs font-semibold text-gray-400 uppercase tracking-widest mb-1">
|
||||
Bulk Upload
|
||||
</p>
|
||||
<h1 className="text-3xl font-extrabold text-gray-900 tracking-tight mb-1">
|
||||
{upload.filename}
|
||||
</h1>
|
||||
<p className="text-sm text-gray-500">Uploaded {formatDate(upload.createdAt)}</p>
|
||||
</div>
|
||||
|
||||
{/* Status card */}
|
||||
<div className="bg-white border border-gray-100 rounded-2xl p-6 shadow-sm mb-4">
|
||||
<div className="flex items-start gap-4">
|
||||
<div className={`w-11 h-11 rounded-xl flex items-center justify-center shrink-0 ${config.iconBg}`}>
|
||||
<Icon className={`h-6 w-6 ${config.iconColor}`} />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<p className="text-sm font-semibold text-gray-900 mb-1">{config.title}</p>
|
||||
<p className="text-sm text-gray-500 leading-relaxed">{config.body}</p>
|
||||
|
||||
{statusKey === "ready_for_processing" && (
|
||||
<Link
|
||||
href={`/portfolio/${slug}/bulk-upload/${uploadId}/map-columns`}
|
||||
className="mt-4 inline-flex items-center gap-2 px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold hover:opacity-90 transition-opacity"
|
||||
>
|
||||
Map Columns
|
||||
<ArrowRightIcon className="h-4 w-4" />
|
||||
</Link>
|
||||
)}
|
||||
|
||||
{statusKey === "mapping_complete" && (
|
||||
<div className="mt-4 flex flex-col gap-3">
|
||||
<Link
|
||||
href={`/portfolio/${slug}/bulk-upload/${uploadId}/map-columns`}
|
||||
className="inline-flex items-center gap-1.5 text-sm text-gray-400 hover:text-gray-700 transition-colors"
|
||||
>
|
||||
Edit column mapping
|
||||
<ArrowRightIcon className="h-3.5 w-3.5" />
|
||||
</Link>
|
||||
<StartAddressMatchingButton
|
||||
portfolioId={upload.portfolioId}
|
||||
uploadId={uploadId}
|
||||
filename={upload.filename}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(statusKey === "processing" ||
|
||||
statusKey === "combining" ||
|
||||
statusKey === "awaiting_review" ||
|
||||
statusKey === "complete" ||
|
||||
statusKey === "failed") &&
|
||||
upload.taskId && (
|
||||
<OnboardingProgress
|
||||
portfolioSlug={slug}
|
||||
portfolioId={upload.portfolioId}
|
||||
uploadId={uploadId}
|
||||
isDomnaUser={isDomnaUser}
|
||||
/>
|
||||
)}
|
||||
|
||||
{statusKey === "complete" && (
|
||||
<a
|
||||
href={`/portfolio/${slug}`}
|
||||
className="mt-4 inline-flex items-center gap-2 px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold hover:opacity-90 transition-opacity"
|
||||
>
|
||||
Open properties
|
||||
<ArrowRightIcon className="h-4 w-4" />
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
);
|
||||
}
|
||||
141
src/app/portfolio/[slug]/(portfolio)/bulk-upload/page.tsx
Normal file
141
src/app/portfolio/[slug]/(portfolio)/bulk-upload/page.tsx
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
import { eq, desc } from "drizzle-orm";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
|
||||
import { redirect } from "next/navigation";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
ArrowRightIcon,
|
||||
DocumentTextIcon,
|
||||
CloudArrowUpIcon,
|
||||
} from "@heroicons/react/24/outline";
|
||||
|
||||
const STATUS_LABELS: Record<string, { label: string; classes: string }> = {
|
||||
ready_for_processing: { label: "Ready", classes: "bg-amber-100 text-amber-700" },
|
||||
processing: { label: "Processing", classes: "bg-blue-100 text-blue-700" },
|
||||
complete: { label: "Complete", classes: "bg-green-100 text-green-700" },
|
||||
failed: { label: "Failed", classes: "bg-red-100 text-red-700" },
|
||||
};
|
||||
|
||||
function formatDate(date: Date) {
|
||||
return new Intl.DateTimeFormat("en-GB", {
|
||||
day: "2-digit",
|
||||
month: "short",
|
||||
year: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
}).format(date);
|
||||
}
|
||||
|
||||
export default async function BulkUploadListPage(props: {
|
||||
params: Promise<{ slug: string }>;
|
||||
}) {
|
||||
const { slug } = await props.params;
|
||||
const session = await getServerSession(AuthOptions);
|
||||
if (!session) redirect("/login");
|
||||
|
||||
const uploads = await db
|
||||
.select()
|
||||
.from(bulkAddressUploads)
|
||||
.where(eq(bulkAddressUploads.portfolioId, slug))
|
||||
.orderBy(desc(bulkAddressUploads.createdAt));
|
||||
|
||||
return (
|
||||
<div className="max-w-4xl mx-auto px-6 py-10">
|
||||
{/* Header */}
|
||||
<div className="mb-8">
|
||||
<p className="text-xs font-semibold text-gray-400 uppercase tracking-widest mb-1">
|
||||
Portfolio › Bulk Uploads
|
||||
</p>
|
||||
<h1 className="text-3xl font-extrabold text-gray-900 tracking-tight">
|
||||
Batch Uploads
|
||||
</h1>
|
||||
<p className="text-sm text-gray-500 mt-1">
|
||||
Select an upload to continue processing, or start a new import.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{uploads.length === 0 ? (
|
||||
/* Empty state */
|
||||
<div className="flex flex-col items-center justify-center py-24 border-2 border-dashed border-gray-200 rounded-2xl text-center">
|
||||
<div className="w-14 h-14 rounded-full bg-gray-100 flex items-center justify-center mb-4">
|
||||
<CloudArrowUpIcon className="h-7 w-7 text-gray-400" />
|
||||
</div>
|
||||
<p className="text-base font-semibold text-gray-700 mb-1">No uploads yet</p>
|
||||
<p className="text-sm text-gray-400">
|
||||
Use the Bulk Upload button on your portfolio to get started.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
/* Upload list */
|
||||
<div className="space-y-3">
|
||||
{/* Column headers */}
|
||||
<div className="grid grid-cols-12 px-4 pb-1">
|
||||
<span className="col-span-6 text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
File
|
||||
</span>
|
||||
<span className="col-span-3 text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
Uploaded
|
||||
</span>
|
||||
<span className="col-span-2 text-xs font-semibold text-gray-400 uppercase tracking-wider">
|
||||
Status
|
||||
</span>
|
||||
<span className="col-span-1" />
|
||||
</div>
|
||||
|
||||
{uploads.map((upload) => {
|
||||
const status = STATUS_LABELS[upload.status] ?? {
|
||||
label: upload.status,
|
||||
classes: "bg-gray-100 text-gray-600",
|
||||
};
|
||||
return (
|
||||
<Link
|
||||
key={upload.id}
|
||||
href={`/portfolio/${slug}/bulk-upload/${upload.id}`}
|
||||
className="grid grid-cols-12 items-center bg-white border border-gray-100 rounded-xl px-4 py-4 hover:border-gray-300 hover:shadow-sm transition-all group"
|
||||
>
|
||||
{/* Filename */}
|
||||
<div className="col-span-6 flex items-center gap-3 min-w-0">
|
||||
<div className="w-9 h-9 rounded-lg bg-gray-50 border border-gray-100 flex items-center justify-center shrink-0">
|
||||
<DocumentTextIcon className="h-5 w-5 text-midblue" />
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<p className="text-sm font-semibold text-gray-900 truncate">
|
||||
{upload.filename}
|
||||
</p>
|
||||
<p className="text-xs text-gray-400 truncate font-mono">
|
||||
{upload.s3Key.split("/").pop()}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Date */}
|
||||
<div className="col-span-3">
|
||||
<p className="text-sm text-gray-500">
|
||||
{formatDate(upload.createdAt)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Status badge */}
|
||||
<div className="col-span-2">
|
||||
<span
|
||||
className={`inline-flex items-center gap-1 px-2.5 py-1 rounded-full text-xs font-semibold ${status.classes}`}
|
||||
>
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-current opacity-70" />
|
||||
{status.label}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Arrow */}
|
||||
<div className="col-span-1 flex justify-end">
|
||||
<ArrowRightIcon className="h-4 w-4 text-gray-300 group-hover:text-gray-600 transition-colors" />
|
||||
</div>
|
||||
</Link>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -710,7 +710,8 @@ export async function getProperties(
|
|||
epc.is_expired AS "epcIsExpired",
|
||||
epc.total_floor_area AS "totalFloorArea",
|
||||
epc.co2_emissions AS "co2Emissions",
|
||||
epc.mainfuel AS mainfuel
|
||||
epc.mainfuel AS mainfuel,
|
||||
p.lexiscore AS lexiscore
|
||||
FROM property p
|
||||
LEFT JOIN property_targets t
|
||||
ON t.property_id = p.id
|
||||
|
|
@ -751,7 +752,8 @@ export async function getProperties(
|
|||
epc.is_expired,
|
||||
epc.total_floor_area,
|
||||
epc.co2_emissions,
|
||||
epc.mainfuel
|
||||
epc.mainfuel,
|
||||
p.lexiscore
|
||||
LIMIT ${limit} OFFSET ${offset};
|
||||
`);
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,20 @@ export function createS3Client(config?: S3Config) {
|
|||
});
|
||||
}
|
||||
|
||||
export function createRetrofitDataS3Client() {
|
||||
return new S3({
|
||||
region: process.env.RETROFIT_DATA_DEV_REGION,
|
||||
accessKeyId: process.env.RETROFIT_DATA_DEV_ACCESS_KEY,
|
||||
secretAccessKey: process.env.RETROFIT_DATA_DEV_SECRET_KEY,
|
||||
});
|
||||
}
|
||||
|
||||
export function retrofitDataS3Bucket(): string {
|
||||
const bucket = process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME;
|
||||
if (!bucket) throw new Error("RETROFIT_DATA_DEV_S3_BUCKET_NAME not set");
|
||||
return bucket;
|
||||
}
|
||||
|
||||
|
||||
// Get presigned url from s3
|
||||
export type PresignGetOptions = {
|
||||
|
|
|
|||
|
|
@ -16,19 +16,20 @@ const sqsClient = new SQSClient({
|
|||
},
|
||||
});
|
||||
|
||||
let cachedQueueUrl: string | null = null;
|
||||
const queueUrlCache = new Map<string, string>();
|
||||
|
||||
// Export if you want to reuse elsewhere
|
||||
export async function getQueueUrl(queueName: string): Promise<string> {
|
||||
if (cachedQueueUrl) return cachedQueueUrl;
|
||||
const cached = queueUrlCache.get(queueName);
|
||||
if (cached) return cached;
|
||||
|
||||
const resp = await sqsClient.send(
|
||||
new GetQueueUrlCommand({ QueueName: queueName })
|
||||
);
|
||||
if (!resp.QueueUrl)
|
||||
throw new Error(`Could not resolve SQS URL for queue: ${queueName}`);
|
||||
cachedQueueUrl = resp.QueueUrl;
|
||||
return cachedQueueUrl;
|
||||
queueUrlCache.set(queueName, resp.QueueUrl);
|
||||
return resp.QueueUrl;
|
||||
}
|
||||
|
||||
type SendOptions = {
|
||||
|
|
|
|||
164
src/lib/bulkUpload/client.ts
Normal file
164
src/lib/bulkUpload/client.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"use client";
|
||||
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { bulkUploadKeys } from "./keys";
|
||||
import { isTerminalStatus, type BulkUpload, type ProgressView } from "./types";
|
||||
|
||||
async function parseError(res: Response, fallback: string): Promise<Error> {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
return new Error(body?.error ?? fallback);
|
||||
}
|
||||
|
||||
export type CreateBulkUploadInput = {
|
||||
file: File;
|
||||
portfolioId: string;
|
||||
userId: string;
|
||||
sourceHeaders: string[];
|
||||
contentType: string;
|
||||
fileKey: string;
|
||||
onProgress?: (percent: number) => void;
|
||||
};
|
||||
|
||||
export type CreateBulkUploadResult = {
|
||||
id: string;
|
||||
s3Key: string;
|
||||
s3Bucket: string;
|
||||
status: string;
|
||||
};
|
||||
|
||||
export function useCreateBulkUpload() {
|
||||
return useMutation<CreateBulkUploadResult, Error, CreateBulkUploadInput>({
|
||||
mutationFn: async (input) => {
|
||||
const presignRes = await fetch("/api/upload/bulk-addresses", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
userId: input.userId,
|
||||
portfolioId: input.portfolioId,
|
||||
fileKey: input.fileKey,
|
||||
contentType: input.contentType,
|
||||
}),
|
||||
});
|
||||
if (!presignRes.ok) throw await parseError(presignRes, "Failed to generate upload URL.");
|
||||
const { url: presignedUrl } = await presignRes.json();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("PUT", presignedUrl);
|
||||
xhr.setRequestHeader("Content-Type", input.contentType);
|
||||
if (input.onProgress) {
|
||||
xhr.upload.addEventListener("progress", (e) => {
|
||||
if (e.lengthComputable) {
|
||||
input.onProgress!(Math.round((e.loaded / e.total) * 100));
|
||||
}
|
||||
});
|
||||
}
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) resolve();
|
||||
else reject(new Error(`S3 upload failed: ${xhr.status}`));
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("Network error during upload"));
|
||||
xhr.send(input.file);
|
||||
});
|
||||
|
||||
const confirmRes = await fetch("/api/upload/bulk-addresses/confirm", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
fileKey: input.fileKey,
|
||||
filename: input.file.name,
|
||||
portfolioId: input.portfolioId,
|
||||
userId: input.userId,
|
||||
sourceHeaders: input.sourceHeaders,
|
||||
}),
|
||||
});
|
||||
if (!confirmRes.ok) throw await parseError(confirmRes, "Failed to record upload.");
|
||||
return confirmRes.json();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useSetColumnMapping(portfolioId: string, uploadId: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation<BulkUpload, Error, { columnMapping: Record<string, string> }>({
|
||||
mutationFn: async (input) => {
|
||||
const res = await fetch(`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
if (!res.ok) throw await parseError(res, "Failed to save mapping.");
|
||||
return res.json();
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: bulkUploadKeys.progress(uploadId) });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useStartAddressMatching(portfolioId: string, uploadId: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation<{ taskId: string }, Error, void>({
|
||||
mutationFn: async () => {
|
||||
const res = await fetch(
|
||||
`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/start-address-matching`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
if (!res.ok) throw await parseError(res, "Failed to start address matching.");
|
||||
return res.json();
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: bulkUploadKeys.progress(uploadId) });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useBulkUploadProgress(portfolioId: string, uploadId: string) {
|
||||
return useQuery<ProgressView, Error>({
|
||||
queryKey: bulkUploadKeys.progress(uploadId),
|
||||
queryFn: async () => {
|
||||
const res = await fetch(
|
||||
`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/progress`,
|
||||
);
|
||||
if (!res.ok) throw await parseError(res, "Failed to load progress.");
|
||||
return res.json();
|
||||
},
|
||||
refetchInterval: (data) => {
|
||||
const status = data?.upload.status;
|
||||
return status && isTerminalStatus(status) ? false : 3000;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useRequestCombine(portfolioId: string, uploadId: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation<unknown, Error, void>({
|
||||
mutationFn: async () => {
|
||||
const res = await fetch(
|
||||
`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/combine`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
if (!res.ok) throw await parseError(res, "Failed to start combiner.");
|
||||
return res.json();
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: bulkUploadKeys.progress(uploadId) });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useFinalize(portfolioId: string, uploadId: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation<void, Error, void>({
|
||||
mutationFn: async () => {
|
||||
const res = await fetch(
|
||||
`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/finalize`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
if (!res.ok) throw await parseError(res, "Finalize failed.");
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: bulkUploadKeys.progress(uploadId) });
|
||||
},
|
||||
});
|
||||
}
|
||||
4
src/lib/bulkUpload/keys.ts
Normal file
4
src/lib/bulkUpload/keys.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export const bulkUploadKeys = {
|
||||
all: ["bulkUpload"] as const,
|
||||
progress: (uploadId: string) => ["bulkUpload", uploadId, "progress"] as const,
|
||||
};
|
||||
273
src/lib/bulkUpload/server.ts
Normal file
273
src/lib/bulkUpload/server.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
import { db } from "@/app/db/db";
|
||||
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
import { tasks } from "@/app/db/schema/tasks/tasks";
|
||||
import { subTasks } from "@/app/db/schema/tasks/subtask";
|
||||
import { count, desc, eq, sql } from "drizzle-orm";
|
||||
import type { BulkUpload, BulkUploadStatus, ProgressView, TaskSummary } from "./types";
|
||||
|
||||
const REMAP_ALLOWED: ReadonlySet<BulkUploadStatus> = new Set([
|
||||
"ready_for_processing",
|
||||
"mapping_complete",
|
||||
]);
|
||||
|
||||
type FastApiTriggerArgs = {
|
||||
endpoint: string;
|
||||
payload: Record<string, unknown>;
|
||||
sessionToken: string | undefined;
|
||||
};
|
||||
|
||||
type FastApiTriggerResult = { ok: true } | { ok: false; status: number; message: string };
|
||||
|
||||
async function triggerFastApiPipeline(args: FastApiTriggerArgs): Promise<FastApiTriggerResult> {
|
||||
const url = process.env.FASTAPI_API_URL;
|
||||
const key = process.env.FASTAPI_API_KEY;
|
||||
if (!url || !key) {
|
||||
console.error("FASTAPI_API_URL or FASTAPI_API_KEY not set");
|
||||
return { ok: false, status: 500, message: "Server misconfiguration" };
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(`${url}${args.endpoint}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": key,
|
||||
Authorization: `Bearer ${args.sessionToken}`,
|
||||
},
|
||||
body: JSON.stringify(args.payload),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const errText = await res.text().catch(() => "");
|
||||
console.error(`FastAPI ${args.endpoint} failed:`, res.status, errText);
|
||||
return { ok: false, status: 502, message: "Pipeline trigger failed" };
|
||||
}
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
console.error(`Failed to reach FastAPI ${args.endpoint}:`, err);
|
||||
return { ok: false, status: 502, message: "Pipeline trigger failed" };
|
||||
}
|
||||
}
|
||||
|
||||
async function loadById(uploadId: string): Promise<BulkUpload | null> {
|
||||
const [row] = await db
|
||||
.select()
|
||||
.from(bulkAddressUploads)
|
||||
.where(eq(bulkAddressUploads.id, uploadId))
|
||||
.limit(1);
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
export async function listForPortfolio(portfolioId: string): Promise<BulkUpload[]> {
|
||||
return db
|
||||
.select()
|
||||
.from(bulkAddressUploads)
|
||||
.where(eq(bulkAddressUploads.portfolioId, portfolioId))
|
||||
.orderBy(desc(bulkAddressUploads.createdAt));
|
||||
}
|
||||
|
||||
async function loadTaskSummary(taskId: string): Promise<TaskSummary | null> {
|
||||
const [row] = await db
|
||||
.select({
|
||||
id: tasks.id,
|
||||
taskSource: tasks.taskSource,
|
||||
status: tasks.status,
|
||||
service: tasks.service,
|
||||
jobStarted: tasks.jobStarted,
|
||||
jobCompleted: tasks.jobCompleted,
|
||||
updatedAt: tasks.updatedAt,
|
||||
totalSubtasks: count(subTasks.id),
|
||||
completedSubtasks: sql<number>`count(case when lower(${subTasks.status}) in ('completed', 'complete') then 1 end)::int`,
|
||||
failedSubtasks: sql<number>`count(case when lower(${subTasks.status}) in ('failed', 'failure', 'error') then 1 end)::int`,
|
||||
})
|
||||
.from(tasks)
|
||||
.leftJoin(subTasks, eq(subTasks.taskId, tasks.id))
|
||||
.where(eq(tasks.id, taskId))
|
||||
.groupBy(tasks.id)
|
||||
.limit(1);
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
export async function getProgressView(uploadId: string): Promise<ProgressView | null> {
|
||||
const upload = await loadById(uploadId);
|
||||
if (!upload) return null;
|
||||
const task = upload.taskId ? await loadTaskSummary(upload.taskId) : null;
|
||||
return { upload, task };
|
||||
}
|
||||
|
||||
function validateMapping(mapping: Record<string, string>): string | null {
|
||||
const values = Object.values(mapping);
|
||||
if (!values.includes("address_1")) return "Mapping must include address_1.";
|
||||
if (!values.includes("postcode")) return "Mapping must include postcode.";
|
||||
return null;
|
||||
}
|
||||
|
||||
export type SetMappingOutcome =
|
||||
| { kind: "ok"; upload: BulkUpload }
|
||||
| { kind: "not_found" }
|
||||
| { kind: "invalid_status"; current: string }
|
||||
| { kind: "invalid_mapping"; reason: string };
|
||||
|
||||
export async function setColumnMapping(
|
||||
uploadId: string,
|
||||
mapping: Record<string, string>,
|
||||
): Promise<SetMappingOutcome> {
|
||||
const upload = await loadById(uploadId);
|
||||
if (!upload) return { kind: "not_found" };
|
||||
if (!REMAP_ALLOWED.has(upload.status as BulkUploadStatus))
|
||||
return { kind: "invalid_status", current: upload.status };
|
||||
|
||||
const reason = validateMapping(mapping);
|
||||
if (reason) return { kind: "invalid_mapping", reason };
|
||||
|
||||
const [updated] = await db
|
||||
.update(bulkAddressUploads)
|
||||
.set({ columnMapping: mapping, status: "mapping_complete" })
|
||||
.where(eq(bulkAddressUploads.id, uploadId))
|
||||
.returning();
|
||||
if (!updated) return { kind: "not_found" };
|
||||
return { kind: "ok", upload: updated };
|
||||
}
|
||||
|
||||
export type LoadForAddressMatchingOutcome =
|
||||
| { kind: "ok"; upload: BulkUpload }
|
||||
| { kind: "not_found" }
|
||||
| { kind: "wrong_state"; current: string }
|
||||
| { kind: "missing_mapping" };
|
||||
|
||||
export async function loadForAddressMatching(
|
||||
uploadId: string,
|
||||
): Promise<LoadForAddressMatchingOutcome> {
|
||||
const upload = await loadById(uploadId);
|
||||
if (!upload) return { kind: "not_found" };
|
||||
if (upload.status !== "mapping_complete")
|
||||
return { kind: "wrong_state", current: upload.status };
|
||||
if (!upload.columnMapping) return { kind: "missing_mapping" };
|
||||
return { kind: "ok", upload };
|
||||
}
|
||||
|
||||
export type TriggerAddressMatchingOutcome =
|
||||
| { kind: "ok"; taskId: string }
|
||||
| { kind: "trigger_failed"; status: number; message: string };
|
||||
|
||||
export async function triggerAddressMatching(args: {
|
||||
uploadId: string;
|
||||
s3Uri: string;
|
||||
sessionToken: string | undefined;
|
||||
}): Promise<TriggerAddressMatchingOutcome> {
|
||||
const upload = await loadById(args.uploadId);
|
||||
if (!upload) return { kind: "trigger_failed", status: 404, message: "Upload not found" };
|
||||
|
||||
const now = new Date();
|
||||
const [task] = await db
|
||||
.insert(tasks)
|
||||
.values({
|
||||
taskSource: `Address Onboarding – ${upload.filename}`,
|
||||
service: "address2uprn",
|
||||
source: "portfolio_id",
|
||||
sourceId: upload.portfolioId,
|
||||
status: "waiting",
|
||||
jobStarted: now,
|
||||
})
|
||||
.returning();
|
||||
const [subTask] = await db
|
||||
.insert(subTasks)
|
||||
.values({
|
||||
taskId: task.id,
|
||||
status: "waiting",
|
||||
inputs: JSON.stringify({ bulk_upload_id: args.uploadId }),
|
||||
})
|
||||
.returning();
|
||||
|
||||
const payload = {
|
||||
task_id: task.id,
|
||||
sub_task_id: subTask.id,
|
||||
s3_uri: args.s3Uri,
|
||||
};
|
||||
|
||||
const trigger = await triggerFastApiPipeline({
|
||||
endpoint: "/v1/bulk-uploads/trigger-postcode-splitter",
|
||||
payload,
|
||||
sessionToken: args.sessionToken,
|
||||
});
|
||||
if (!trigger.ok)
|
||||
return { kind: "trigger_failed", status: trigger.status, message: trigger.message };
|
||||
|
||||
await Promise.all([
|
||||
db
|
||||
.update(bulkAddressUploads)
|
||||
.set({ status: "processing", taskId: task.id })
|
||||
.where(eq(bulkAddressUploads.id, args.uploadId)),
|
||||
db
|
||||
.update(tasks)
|
||||
.set({ status: "in progress" })
|
||||
.where(eq(tasks.id, task.id)),
|
||||
db
|
||||
.update(subTasks)
|
||||
.set({ inputs: JSON.stringify(payload) })
|
||||
.where(eq(subTasks.id, subTask.id)),
|
||||
]);
|
||||
return { kind: "ok", taskId: task.id };
|
||||
}
|
||||
|
||||
export type CombineRetriggerOutcome =
|
||||
| { kind: "triggered"; taskId: string; subTaskId: string }
|
||||
| { kind: "already_combined" }
|
||||
| { kind: "not_found" }
|
||||
| { kind: "missing_task" }
|
||||
| { kind: "trigger_failed"; status: number; message: string };
|
||||
|
||||
export async function requestCombineRetrigger(args: {
|
||||
uploadId: string;
|
||||
sessionToken: string | undefined;
|
||||
}): Promise<CombineRetriggerOutcome> {
|
||||
const upload = await loadById(args.uploadId);
|
||||
if (!upload) return { kind: "not_found" };
|
||||
if (!upload.taskId) return { kind: "missing_task" };
|
||||
if (upload.combinedOutputS3Uri) return { kind: "already_combined" };
|
||||
|
||||
const [subTask] = await db
|
||||
.insert(subTasks)
|
||||
.values({ taskId: upload.taskId, status: "waiting" })
|
||||
.returning();
|
||||
|
||||
const payload = { task_id: upload.taskId, sub_task_id: subTask.id };
|
||||
|
||||
const trigger = await triggerFastApiPipeline({
|
||||
endpoint: "/v1/bulk-uploads/trigger-combiner",
|
||||
payload,
|
||||
sessionToken: args.sessionToken,
|
||||
});
|
||||
if (!trigger.ok)
|
||||
return { kind: "trigger_failed", status: trigger.status, message: trigger.message };
|
||||
|
||||
await db
|
||||
.update(subTasks)
|
||||
.set({ inputs: JSON.stringify(payload) })
|
||||
.where(eq(subTasks.id, subTask.id));
|
||||
|
||||
return { kind: "triggered", taskId: upload.taskId, subTaskId: subTask.id };
|
||||
}
|
||||
|
||||
export type LoadForFinalizeOutcome =
|
||||
| { kind: "ready"; upload: BulkUpload }
|
||||
| { kind: "already_finalized" }
|
||||
| { kind: "not_found" }
|
||||
| { kind: "not_yet_combined" }
|
||||
| { kind: "wrong_state"; current: string };
|
||||
|
||||
export async function loadForFinalize(uploadId: string): Promise<LoadForFinalizeOutcome> {
|
||||
const upload = await loadById(uploadId);
|
||||
if (!upload) return { kind: "not_found" };
|
||||
if (upload.status === "complete") return { kind: "already_finalized" };
|
||||
if (upload.status !== "awaiting_review")
|
||||
return { kind: "wrong_state", current: upload.status };
|
||||
if (!upload.combinedOutputS3Uri) return { kind: "not_yet_combined" };
|
||||
return { kind: "ready", upload };
|
||||
}
|
||||
|
||||
export async function markFinalized(uploadId: string): Promise<void> {
|
||||
await db
|
||||
.update(bulkAddressUploads)
|
||||
.set({ status: "complete" })
|
||||
.where(eq(bulkAddressUploads.id, uploadId));
|
||||
}
|
||||
42
src/lib/bulkUpload/types.ts
Normal file
42
src/lib/bulkUpload/types.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import type { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
|
||||
|
||||
export const BULK_UPLOAD_STATUSES = [
|
||||
"ready_for_processing",
|
||||
"mapping_complete",
|
||||
"processing",
|
||||
"combining",
|
||||
"awaiting_review",
|
||||
"complete",
|
||||
"failed",
|
||||
] as const;
|
||||
|
||||
export type BulkUploadStatus = (typeof BULK_UPLOAD_STATUSES)[number];
|
||||
|
||||
export type BulkUpload = typeof bulkAddressUploads.$inferSelect;
|
||||
|
||||
export type TaskSummary = {
|
||||
id: string;
|
||||
taskSource: string;
|
||||
status: string;
|
||||
service: string | null;
|
||||
jobStarted: Date | null;
|
||||
jobCompleted: Date | null;
|
||||
updatedAt: Date;
|
||||
totalSubtasks: number;
|
||||
completedSubtasks: number;
|
||||
failedSubtasks: number;
|
||||
};
|
||||
|
||||
export type ProgressView = {
|
||||
upload: BulkUpload;
|
||||
task: TaskSummary | null;
|
||||
};
|
||||
|
||||
const TERMINAL_UPLOAD_STATUSES: ReadonlySet<BulkUploadStatus> = new Set([
|
||||
"complete",
|
||||
"failed",
|
||||
]);
|
||||
|
||||
export function isTerminalStatus(status: string): boolean {
|
||||
return TERMINAL_UPLOAD_STATUSES.has(status as BulkUploadStatus);
|
||||
}
|
||||
8
src/lib/session.ts
Normal file
8
src/lib/session.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
import { NextRequest } from "next/server";
|
||||
|
||||
export function readSessionToken(request: NextRequest): string | undefined {
|
||||
return (
|
||||
request.cookies.get("__Secure-next-auth.session-token")?.value ??
|
||||
request.cookies.get("next-auth.session-token")?.value
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue