address2uprn onboaridng poc

This commit is contained in:
Jun-te Kim 2026-04-21 20:24:26 +00:00
parent 119a800995
commit bee57a56b5
20 changed files with 467 additions and 396 deletions

View file

@ -5,7 +5,16 @@
"Bash(backlog mcp *)",
"Read(//home/vscode/.config/nvim/**)",
"Read(//home/vscode/.config/nvim/lua/plugins/**)",
"Bash(npx tsc *)"
"Bash(npx tsc *)",
"Read(//workspaces/home/github/Model/backend/**)",
"Read(//workspaces/home/github/Model/etl/**)",
"mcp__backlog__task_create",
"mcp__backlog__task_view",
"mcp__backlog__task_edit",
"Read(//workspaces/home/github/Model/**)",
"Bash(pytest backend/tests/test_bulk_combiner_status.py -v --no-cov)",
"Bash(echo \"EXIT: $?\")",
"mcp__backlog__task_list"
]
},
"enabledMcpjsonServers": [

View file

@ -17,6 +17,7 @@ services:
- SSH_AUTH_SOCK=${SSH_AUTH_SOCK:-}
networks:
- frontend-net
- shared-dev
pgadmin:
image: dpage/pgadmin4
@ -32,3 +33,5 @@ services:
networks:
frontend-net:
driver: bridge
shared-dev:
external: true

2
.gitignore vendored
View file

@ -37,3 +37,5 @@ cypress.env.json
# typescript
*.tsbuildinfo
next-env.d.ts
backlog/*

View file

@ -1,31 +0,0 @@
---
id: TASK-10
title: Redirect to confirm-matches when combined_output_s3_uri populated
status: To Do
assignee: []
created_date: '2026-04-20'
updated_date: '2026-04-20'
labels:
- frontend
- bulk-upload
- ui
dependencies:
- TASK-8
priority: medium
ordinal: 5000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
`OnboardingProgress.tsx` currently fires client-side combine POST when task terminal. Once backend auto-chains (backend-task-5) OR frontend triggers via backend route (task-7), the polling should watch for `bulk_address_uploads.combined_output_s3_uri` to be set. When present, show "Review matches →" CTA (or auto-redirect to confirm-matches page, task-8).
May require a new GET endpoint that returns `{status, combined_output_s3_uri}` for polling. Or extend existing task summary with upload fields.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 Polling stops once combined_output_s3_uri populated
- [ ] #2 UI surfaces a clear CTA to review matches
- [ ] #3 No duplicate combiner fires across refreshes
<!-- AC:END -->

View file

@ -1,36 +0,0 @@
---
id: TASK-4
title: Smoke-test full bulk upload flow end-to-end on dev
status: To Do
assignee:
- Jun-te Kim
created_date: '2026-04-18 19:02'
updated_date: '2026-04-20'
labels:
- qa
- bulk-upload
dependencies:
- TASK-6
- TASK-7
- TASK-8
- TASK-9
- TASK-10
priority: medium
ordinal: 9000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
After frontend + backend refactor ships, run the full flow on dev: upload xlsx → map columns → start onboarding → poll task progress → combiner fires → combined_output_s3_uri populated → review matches page renders → confirm rows → addresses appear in portfolio.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 Frontend no longer needs POSTCODE_SPLITTER_QUEUE_NAME or BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME env vars
- [ ] #2 Backend logs show both splitter + combiner triggered via HTTP route
- [ ] #3 bulk_final_outputs/{task_id}/combined_<timestamp>.csv exists in retrofit_sap_data bucket
- [ ] #4 bulk_address_uploads.combined_output_s3_uri populated
- [ ] #5 Confirm-matches page renders with match rows
- [ ] #6 After confirm submit, addresses persist into portfolio
<!-- AC:END -->

View file

@ -1,28 +0,0 @@
---
id: TASK-5
title: >-
Squash migrations 0178 (DROP) + 0179 (re-ADD) next time bulk_address_uploads
touched
status: Done
assignee:
- Jun-te Kim
created_date: '2026-04-18 19:02'
updated_date: '2026-04-18 19:06'
labels:
- tech-debt
- db
dependencies: []
priority: low
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
0178 DROPs task_id + combined_output_s3_uri; 0179 re-ADDs them. Net-zero on live, wasted churn on fresh envs. Collapse to single migration next time schema changes in this area.
<!-- SECTION:DESCRIPTION:END -->
## Implementation Notes
<!-- SECTION:NOTES:BEGIN -->
Squashed in-place: deleted 0179.sql + 0179_snapshot.json, removed from _journal.json, patched 0178_snapshot.json to include task_id + combined_output_s3_uri cols. Orphan row may remain in live __drizzle_migrations but drizzle tolerates it.
<!-- SECTION:NOTES:END -->

View file

@ -1,105 +0,0 @@
---
id: TASK-6
title: Refactor onboard route to call backend trigger-splitter endpoint
status: In Progress
assignee: []
created_date: '2026-04-20'
updated_date: '2026-04-20 12:55'
labels:
- frontend
- bulk-upload
- refactor
dependencies:
- BACKEND-TASK-1
priority: high
ordinal: 1000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
Currently `src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts` builds an SQS message and sends it to `POSTCODE_SPLITTER_QUEUE_NAME`. Move the SQS send to the backend. Frontend still transforms XLSX → CSV + uploads to S3, then calls backend HTTP `POST /v1/bulk-uploads/trigger-splitter`. Drop `POSTCODE_SPLITTER_QUEUE_NAME`, `sendToQueue`, and SQS IAM dependency.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [x] #1 `onboard/route.ts` no longer imports `sendToQueue` or reads `POSTCODE_SPLITTER_QUEUE_NAME`
- [x] #2 Transformed CSV still uploaded to `bulk_onboarding_inputs/{portfolioId}/{uploadId}.csv`
- [x] #3 Backend trigger-splitter endpoint called with correct payload
- [ ] #4 DB updates (bulk_address_uploads.status="processing", tasks.status, subTasks.inputs) still happen on success
- [ ] #5 4xx/5xx from backend → return 502 to client with useful message
<!-- AC:END -->
## Implementation Plan
<!-- SECTION:PLAN:BEGIN -->
<!-- SECTION:NOTES:BEGIN -->
### File changed
`src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts`
### What stays the same
- Auth check via `getServerSession`
- Upload record + status validation
- `transformFile()` XLSX → CSV
- S3 upload of transformed CSV to `bulk_onboarding_inputs/{portfolioId}/{uploadId}.csv`
- DB writes: `bulkAddressUploads.status = "processing"`, `tasks.status = "in progress"`, `subTasks.inputs`
### What changes
**Remove:**
```ts
import { sendToQueue } from "@/app/utils/sqs";
// POSTCODE_SPLITTER_QUEUE_NAME env var check
```
**Add — call backend trigger-splitter:**
```ts
const fastapiUrl = process.env.FASTAPI_API_URL;
const fastapiKey = process.env.FASTAPI_API_KEY;
if (!fastapiUrl || !fastapiKey) {
return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 });
}
const sessionToken =
request.cookies.get("__Secure-next-auth.session-token")?.value ??
request.cookies.get("next-auth.session-token")?.value;
const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-splitter`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"x-api-key": fastapiKey,
Authorization: `Bearer ${sessionToken}`,
},
body: JSON.stringify({
task_id: body.taskId,
sub_task_id: body.subTaskId,
s3_uri: s3Uri,
}),
});
if (!triggerRes.ok) {
const errText = await triggerRes.text().catch(() => "");
console.error("Backend trigger-splitter failed:", triggerRes.status, errText);
return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 });
}
```
**Order of ops** (no change to structure, just swap):
1. Validate body + upload record
2. Read source file from S3
3. Transform XLSX → CSV
4. Upload transformed CSV to S3
5. ~~sendToQueue~~ → fetch backend trigger-splitter
6. DB updates (status, taskId, subTask inputs) — only after step 5 succeeds
### Env vars required
- `FASTAPI_API_URL` — already set, already used in `plan/trigger/route.ts`
- `FASTAPI_API_KEY` — already set
### Env vars removed
- `POSTCODE_SPLITTER_QUEUE_NAME` — no longer read by frontend (can remove from .env.local + staging/prod)
<!-- SECTION:NOTES:END -->
<!-- SECTION:PLAN:END -->

View file

@ -1,32 +0,0 @@
---
id: TASK-7
title: Refactor combine route to call backend trigger-combiner endpoint
status: To Do
assignee: []
created_date: '2026-04-20'
updated_date: '2026-04-20'
labels:
- frontend
- bulk-upload
- refactor
dependencies:
- BACKEND-TASK-2
priority: high
ordinal: 2000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
`src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts` sends SQS directly to `BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME`. Replace with call to backend `POST /bulk-uploads/{task_id}/combine`. Drop queue name env var + SendMessage IAM dependency on frontend.
If backend auto-chains combiner on splitter completion (backend-task-5), this route may simply proxy a manual "re-combine" action or be removed entirely. Decide during implementation.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 `combine/route.ts` no longer imports `sendToQueue` or reads `BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME`
- [ ] #2 Backend trigger-combiner endpoint called with task_id
- [ ] #3 Frontend still updates subTasks row with inputs on success (or delegates to backend)
- [ ] #4 Decision logged: proxy vs delete-after-auto-chain
<!-- AC:END -->

View file

@ -1,34 +0,0 @@
---
id: TASK-8
title: Add confirm-matches page for bulk upload address→UPRN review
status: To Do
assignee: []
created_date: '2026-04-20'
updated_date: '2026-04-20'
labels:
- frontend
- bulk-upload
- ui
dependencies:
- TASK-9
- BACKEND-TASK-3
priority: high
ordinal: 3000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
New route `/portfolio/{slug}/bulk-upload/{uploadId}/confirm-matches`. Loads combined CSV from backend (via frontend proxy route, see task-9) and renders review table: original address input | matched UPRN | matched address | confidence. User can accept/reject per row, then POST confirmed rows to backend to persist into portfolio `addresses`.
Status transitions: `complete` (combiner done) → show "Review matches" CTA on upload detail page → confirm-matches page → on submit, move upload to `confirmed` or similar.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 Page renders match rows in a scrollable table
- [ ] #2 Row actions: accept (default on), reject
- [ ] #3 Submit posts accepted rows to backend confirm-matches route
- [ ] #4 After submit, redirect to portfolio addresses list
- [ ] #5 No useEffect/useMemo (per CLAUDE.md) — use Server Components + Route Handlers where possible
<!-- AC:END -->

View file

@ -1,35 +0,0 @@
---
id: TASK-9
title: Add proxy API routes for combined-results and confirm-matches
status: To Do
assignee: []
created_date: '2026-04-20'
updated_date: '2026-04-20'
labels:
- frontend
- bulk-upload
- api
dependencies:
- BACKEND-TASK-3
- BACKEND-TASK-4
priority: high
ordinal: 4000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
Two Next.js route handlers that proxy to backend:
- `GET /api/portfolio/{portfolioId}/bulk-uploads/{uploadId}/combined-results` → backend GET `/bulk-uploads/{uploadId}/combined-results`. Returns parsed match rows for the confirm UI.
- `POST /api/portfolio/{portfolioId}/bulk-uploads/{uploadId}/confirm-matches` → backend POST `/bulk-uploads/{uploadId}/confirm-matches`. Body: accepted rows.
Both must: check session, pass through portfolio scope, translate backend errors to sane frontend responses.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 Both routes auth-gated via getServerSession
- [ ] #2 Params typed as Promise per Next.js 15 convention
- [ ] #3 Backend 4xx/5xx surfaced with appropriate HTTP code + message
- [ ] #4 Upload id is validated against bulk_address_uploads row before proxying
<!-- AC:END -->

View file

@ -5,10 +5,9 @@ import { eq } from "drizzle-orm";
import { NextRequest, NextResponse } from "next/server";
import { getServerSession } from "next-auth";
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
import { sendToQueue } from "@/app/utils/sqs";
export async function POST(
_request: NextRequest,
request: NextRequest,
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
) {
const session = await getServerSession(AuthOptions);
@ -28,9 +27,10 @@ export async function POST(
if (upload.combinedOutputS3Uri)
return NextResponse.json({ alreadyCombined: true }, { status: 200 });
const queueName = process.env.BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME;
if (!queueName) {
console.error("BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME not set");
const fastapiUrl = process.env.FASTAPI_API_URL;
const fastapiKey = process.env.FASTAPI_API_KEY;
if (!fastapiUrl || !fastapiKey) {
console.error("FASTAPI_API_URL or FASTAPI_API_KEY not set");
return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 });
}
@ -44,11 +44,29 @@ export async function POST(
const messageBody = { task_id: upload.taskId, sub_task_id: subTask.id };
const sessionToken =
request.cookies.get("__Secure-next-auth.session-token")?.value ??
request.cookies.get("next-auth.session-token")?.value;
try {
await sendToQueue(messageBody, { queueName });
const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-combiner`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"x-api-key": fastapiKey,
Authorization: `Bearer ${sessionToken}`,
},
body: JSON.stringify(messageBody),
});
if (!triggerRes.ok) {
const errText = await triggerRes.text().catch(() => "");
console.error("Backend trigger-combiner failed:", triggerRes.status, errText);
return NextResponse.json({ error: "Failed to trigger combiner" }, { status: 502 });
}
} catch (err) {
console.error("Failed to send combiner SQS message:", err);
return NextResponse.json({ error: "Failed to queue combiner job" }, { status: 500 });
console.error("Failed to reach backend trigger-combiner:", err);
return NextResponse.json({ error: "Failed to trigger combiner" }, { status: 502 });
}
await db

View file

@ -1,70 +0,0 @@
import { db } from "@/app/db/db";
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
import { eq } from "drizzle-orm";
import { NextRequest, NextResponse } from "next/server";
import { getServerSession } from "next-auth";
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
export async function POST(
request: NextRequest,
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
) {
const session = await getServerSession(AuthOptions);
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
const { uploadId } = await params;
const [upload] = await db
.select({ taskId: bulkAddressUploads.taskId })
.from(bulkAddressUploads)
.where(eq(bulkAddressUploads.id, uploadId))
.limit(1);
if (!upload) return NextResponse.json({ error: "Not found" }, { status: 404 });
if (!upload.taskId) return NextResponse.json({ error: "Task not started" }, { status: 409 });
const fastapiUrl = process.env.FASTAPI_API_URL;
const fastapiKey = process.env.FASTAPI_API_KEY;
if (!fastapiUrl || !fastapiKey) {
console.error("FASTAPI_API_URL or FASTAPI_API_KEY not set");
return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 });
}
const sessionToken =
request.cookies.get("__Secure-next-auth.session-token")?.value ??
request.cookies.get("next-auth.session-token")?.value;
let body;
try {
body = await request.json();
} catch {
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
}
try {
const res = await fetch(
`${fastapiUrl}/v1/bulk-uploads/${upload.taskId}/confirm-matches`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
"x-api-key": fastapiKey,
Authorization: `Bearer ${sessionToken}`,
},
body: JSON.stringify(body),
}
);
if (!res.ok) {
const errText = await res.text().catch(() => "");
console.error("Backend confirm-matches failed:", res.status, errText);
return NextResponse.json({ error: "Failed to confirm matches" }, { status: 502 });
}
const data = await res.json();
return NextResponse.json(data, { status: 200 });
} catch (err) {
console.error("Failed to reach backend confirm-matches:", err);
return NextResponse.json({ error: "Failed to confirm matches" }, { status: 502 });
}
}

View file

@ -143,7 +143,7 @@ export async function POST(
request.cookies.get("next-auth.session-token")?.value;
try {
const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-splitter`, {
const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-postcode-splitter`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@ -159,11 +159,11 @@ export async function POST(
if (!triggerRes.ok) {
const errText = await triggerRes.text().catch(() => "");
console.error("Backend trigger-splitter failed:", triggerRes.status, errText);
console.error("Backend trigger-postcode-splitter failed:", triggerRes.status, errText);
return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 });
}
} catch (err) {
console.error("Failed to reach backend trigger-splitter:", err);
console.error("Failed to reach backend trigger-postcode-splitter:", err);
return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 });
}

View file

@ -2,12 +2,37 @@ import { db } from "@/app/db/db";
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
import { eq } from "drizzle-orm";
import { NextRequest, NextResponse } from "next/server";
import { getServerSession } from "next-auth";
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
import { z } from "zod";
const PatchSchema = z.object({
columnMapping: z.record(z.string(), z.string()),
});
export async function GET(
_request: NextRequest,
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }
) {
const session = await getServerSession(AuthOptions);
if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
const { uploadId } = await params;
const [upload] = await db
.select({
status: bulkAddressUploads.status,
combinedOutputS3Uri: bulkAddressUploads.combinedOutputS3Uri,
})
.from(bulkAddressUploads)
.where(eq(bulkAddressUploads.id, uploadId))
.limit(1);
if (!upload) return NextResponse.json({ error: "Not found" }, { status: 404 });
return NextResponse.json(upload, { status: 200 });
}
export async function PATCH(
request: NextRequest,
{ params }: { params: Promise<{ portfolioId: string; uploadId: string }> }

View file

@ -8,7 +8,12 @@ const BodySchema = z.object({
filename: z.string(),
portfolioId: z.string(),
userId: z.string(),
sourceHeaders: z.array(z.string()).default([]),
sourceHeaders: z
.array(z.union([z.string(), z.null(), z.undefined()]))
.default([])
.transform((arr) =>
arr.filter((h): h is string => typeof h === "string" && h.trim().length > 0)
),
});
export async function POST(request: NextRequest) {

View file

@ -77,10 +77,12 @@ async function validateHeaders(file: File): Promise<{ error: string | null; head
const buffer = await file.arrayBuffer();
const wb = XLSX.read(buffer, { sheetRows: 1 });
const sheet = wb.Sheets[wb.SheetNames[0]];
const rows = XLSX.utils.sheet_to_json<string[]>(sheet, { header: 1 });
headers = ((rows[0] as string[]) ?? []).map((h) => String(h ?? "").trim());
const rows = XLSX.utils.sheet_to_json<unknown[]>(sheet, { header: 1, defval: "" });
headers = ((rows[0] as unknown[]) ?? []).map((h) => String(h ?? "").trim());
}
headers = headers.filter((h) => h.length > 0);
const normalised = headers.map((h) => h.toLowerCase());
const hasAddress = normalised.some((h) => h.startsWith("address"));
const hasPostcode = normalised.some((h) => h === "postcode");

View file

@ -1,6 +1,7 @@
"use client";
import { useEffect, useState, useRef } from "react";
import { useRouter } from "next/navigation";
import Link from "next/link";
interface TaskData {
@ -12,6 +13,11 @@ interface TaskData {
failedSubtasks: number;
}
interface UploadStatus {
status: string;
combinedOutputS3Uri: string | null;
}
interface Props {
taskId: string;
portfolioSlug: string;
@ -30,10 +36,13 @@ export default function OnboardingProgress({
uploadId,
isDomnaUser,
}: Props) {
const router = useRouter();
const [data, setData] = useState<TaskData | null>(null);
const [uploadStatus, setUploadStatus] = useState<UploadStatus | null>(null);
const [fetchError, setFetchError] = useState(false);
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const combineFiredRef = useRef(false);
const redirectedRef = useRef(false);
useEffect(() => {
async function poll() {
@ -43,14 +52,30 @@ export default function OnboardingProgress({
const json: TaskData = await res.json();
setData(json);
const status = json.status.toLowerCase();
if (TERMINAL_STATUSES.has(status)) {
if (intervalRef.current) clearInterval(intervalRef.current);
if (!FAILED_STATUSES.has(status) && !combineFiredRef.current) {
combineFiredRef.current = true;
fetch(`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/combine`, {
method: "POST",
}).catch((err) => console.error("Failed to trigger combiner:", err));
}
const uploadRes = await fetch(
`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}`
);
if (uploadRes.ok) {
const upload: UploadStatus = await uploadRes.json();
setUploadStatus(upload);
if (upload.status === "awaiting_review" && !redirectedRef.current) {
redirectedRef.current = true;
if (intervalRef.current) clearInterval(intervalRef.current);
router.push(
`/portfolio/${portfolioSlug}/bulk-upload/${uploadId}/confirm-matches`
);
return;
}
}
}
} catch {
setFetchError(true);
@ -60,7 +85,7 @@ export default function OnboardingProgress({
poll();
intervalRef.current = setInterval(poll, 3000);
return () => { if (intervalRef.current) clearInterval(intervalRef.current); };
}, [taskId, portfolioId, uploadId]);
}, [taskId, portfolioId, portfolioSlug, uploadId, router]);
if (fetchError) return null;
if (!data) {
@ -76,12 +101,15 @@ export default function OnboardingProgress({
const complete = data.completedSubtasks;
const failed = data.failedSubtasks;
const percent = total > 0 ? Math.round((complete / total) * 100) : 0;
const isDone = TERMINAL_STATUSES.has(data.status.toLowerCase());
const isFailed = ["failed", "failure", "error"].includes(data.status.toLowerCase());
const taskDone = TERMINAL_STATUSES.has(data.status.toLowerCase());
const isFailed = FAILED_STATUSES.has(data.status.toLowerCase());
const isCombining =
taskDone && !isFailed && uploadStatus?.status === "combining";
const isAwaitingReview =
taskDone && !isFailed && uploadStatus?.status === "awaiting_review";
return (
<div className="mt-6 space-y-3">
{/* Progress bar */}
<div className="w-full bg-gray-100 rounded-full h-2 overflow-hidden">
<div
className={`h-2 rounded-full transition-all duration-500 ${isFailed ? "bg-red-400" : "bg-[#14163d]"}`}
@ -89,7 +117,6 @@ export default function OnboardingProgress({
/>
</div>
{/* Counts */}
<div className="flex items-center gap-4 text-xs text-gray-500">
{total > 0 && (
<span>
@ -102,20 +129,35 @@ export default function OnboardingProgress({
{failed} failed
</span>
)}
{!isDone && (
{!taskDone && (
<span className="flex items-center gap-1 text-blue-500">
<span className="w-1.5 h-1.5 rounded-full bg-blue-400 animate-pulse" />
Running
</span>
)}
{isDone && !isFailed && (
{isCombining && (
<span className="flex items-center gap-1 text-blue-500">
<span className="w-1.5 h-1.5 rounded-full bg-blue-400 animate-pulse" />
Combining results
</span>
)}
{isAwaitingReview && (
<span className="flex items-center gap-1 text-green-600 font-semibold">
<span className="w-1.5 h-1.5 rounded-full bg-green-500" />
Complete
Ready for review
</span>
)}
</div>
{isAwaitingReview && (
<Link
href={`/portfolio/${portfolioSlug}/bulk-upload/${uploadId}/confirm-matches`}
className="inline-flex items-center gap-2 px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold hover:opacity-90 transition-opacity"
>
Review matches
</Link>
)}
{isDomnaUser && (
<Link
href={`/portfolio/${portfolioSlug}/settings/logs`}

View file

@ -0,0 +1,197 @@
"use client";
import Link from "next/link";
export interface CombinedResultRow {
row_index: number;
input_address: string;
internal_reference: string | null;
uprn: string | null;
matched_address: string | null;
lexiscore: number | null;
score_bucket: "high" | "med" | "low" | null;
flags: ("duplicate" | "missing")[];
}
export interface CombinedResultsResponse {
task_id: string;
total: number;
offset: number;
limit: number;
flags_summary: { duplicates: number; missing: number; matched: number };
rows: CombinedResultRow[];
}
interface Props {
data: CombinedResultsResponse;
slug: string;
uploadId: string;
filter: "all" | "missing" | "duplicate";
offset: number;
limit: number;
}
function scoreChipClasses(bucket: CombinedResultRow["score_bucket"]): string {
if (bucket === "high") return "bg-green-50 text-green-700 border-green-200";
if (bucket === "med") return "bg-amber-50 text-amber-700 border-amber-200";
if (bucket === "low") return "bg-red-50 text-red-700 border-red-200";
return "bg-gray-50 text-gray-400 border-gray-200";
}
function scoreChipLabel(bucket: CombinedResultRow["score_bucket"]): string {
if (bucket === "high") return "High";
if (bucket === "med") return "Medium";
if (bucket === "low") return "Low";
return "—";
}
function flagPillClasses(flag: "duplicate" | "missing"): string {
return flag === "missing"
? "bg-red-50 text-red-700 border border-red-200"
: "bg-amber-50 text-amber-700 border border-amber-200";
}
function tabClasses(active: boolean): string {
return active
? "px-4 py-2 rounded-xl text-sm font-semibold bg-[#14163d] text-white"
: "px-4 py-2 rounded-xl text-sm font-medium text-gray-500 hover:bg-gray-100";
}
export default function ConfirmMatchesClient({
data,
slug,
uploadId,
filter,
offset,
limit,
}: Props) {
const rows =
filter === "all"
? data.rows
: data.rows.filter((r) => r.flags.includes(filter));
const basePath = `/portfolio/${slug}/bulk-upload/${uploadId}/confirm-matches`;
const pageStart = data.total === 0 ? 0 : offset + 1;
const pageEnd = Math.min(offset + data.rows.length, data.total);
const hasPrev = offset > 0;
const hasNext = offset + limit < data.total;
const prevOffset = Math.max(0, offset - limit);
const nextOffset = offset + limit;
return (
<div className="space-y-4">
<div className="flex items-center gap-2">
<Link href={`${basePath}?offset=${offset}&limit=${limit}`} className={tabClasses(filter === "all")}>
All ({data.total})
</Link>
<Link
href={`${basePath}?offset=0&limit=${limit}&filter=missing`}
className={tabClasses(filter === "missing")}
>
Missing ({data.flags_summary.missing})
</Link>
<Link
href={`${basePath}?offset=0&limit=${limit}&filter=duplicate`}
className={tabClasses(filter === "duplicate")}
>
Duplicates ({data.flags_summary.duplicates})
</Link>
</div>
<div className="bg-white border border-gray-100 rounded-2xl overflow-hidden shadow-sm">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-xs text-gray-500 uppercase tracking-wider">
<tr>
<th className="text-left px-4 py-3">Internal Ref</th>
<th className="text-left px-4 py-3">Input Address</th>
<th className="text-left px-4 py-3">UPRN</th>
<th className="text-left px-4 py-3">Matched Address</th>
<th className="text-left px-4 py-3">Score</th>
<th className="text-left px-4 py-3">Flags</th>
<th className="text-left px-4 py-3">Actions</th>
</tr>
</thead>
<tbody>
{rows.length === 0 && (
<tr>
<td colSpan={7} className="px-4 py-8 text-center text-sm text-gray-400">
No rows match this filter.
</td>
</tr>
)}
{rows.map((row) => (
<tr key={row.row_index} className="border-t border-gray-100">
<td className="px-4 py-3 text-gray-600">{row.internal_reference ?? "—"}</td>
<td className="px-4 py-3 text-gray-900">{row.input_address || "—"}</td>
<td className="px-4 py-3 font-mono text-xs text-gray-700">{row.uprn ?? "—"}</td>
<td className="px-4 py-3 text-gray-600">{row.matched_address ?? "—"}</td>
<td className="px-4 py-3">
<span
className={`inline-flex items-center px-2.5 py-1 rounded-full text-xs font-semibold border ${scoreChipClasses(row.score_bucket)}`}
>
{scoreChipLabel(row.score_bucket)}
</span>
</td>
<td className="px-4 py-3">
<div className="flex flex-wrap gap-1">
{row.flags.map((f) => (
<span
key={f}
className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium ${flagPillClasses(f)}`}
>
{f === "missing" ? "Missing" : "Duplicate"}
</span>
))}
{row.flags.length === 0 && <span className="text-xs text-gray-400"></span>}
</div>
</td>
<td className="px-4 py-3">
<button
type="button"
disabled
title="Coming soon"
className="px-3 py-1.5 rounded-lg text-xs font-semibold bg-gray-100 text-gray-400 cursor-not-allowed"
>
Advanced ARA search
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
<div className="flex items-center justify-between text-sm text-gray-500">
<span>
Showing {pageStart}{pageEnd} of {data.total}
</span>
<div className="flex items-center gap-2">
{hasPrev ? (
<Link
href={`${basePath}?offset=${prevOffset}&limit=${limit}${filter !== "all" ? `&filter=${filter}` : ""}`}
className="px-3 py-1.5 rounded-lg bg-white border border-gray-200 text-sm font-medium hover:bg-gray-50"
>
Prev
</Link>
) : (
<span className="px-3 py-1.5 rounded-lg bg-gray-50 border border-gray-100 text-sm text-gray-300">
Prev
</span>
)}
{hasNext ? (
<Link
href={`${basePath}?offset=${nextOffset}&limit=${limit}${filter !== "all" ? `&filter=${filter}` : ""}`}
className="px-3 py-1.5 rounded-lg bg-[#14163d] text-white text-sm font-semibold hover:opacity-90"
>
Next
</Link>
) : (
<span className="px-3 py-1.5 rounded-lg bg-gray-50 border border-gray-100 text-sm text-gray-300">
Next
</span>
)}
</div>
</div>
</div>
);
}

View file

@ -0,0 +1,109 @@
"use server";
import { db } from "@/app/db/db";
import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads";
import { eq } from "drizzle-orm";
import { getServerSession } from "next-auth";
import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions";
import { redirect, notFound } from "next/navigation";
import { cookies, headers } from "next/headers";
import Link from "next/link";
import { ArrowLeftIcon } from "@heroicons/react/24/outline";
import ConfirmMatchesClient, {
CombinedResultsResponse,
} from "./ConfirmMatchesClient";
const DEFAULT_LIMIT = 100;
export default async function ConfirmMatchesPage(props: {
params: Promise<{ slug: string; uploadId: string }>;
searchParams: Promise<{ offset?: string; limit?: string; filter?: string }>;
}) {
const { slug, uploadId } = await props.params;
const search = await props.searchParams;
const session = await getServerSession(AuthOptions);
if (!session) redirect("/login");
const [upload] = await db
.select()
.from(bulkAddressUploads)
.where(eq(bulkAddressUploads.id, uploadId))
.limit(1);
if (!upload) notFound();
if (upload.status !== "awaiting_review") {
redirect(`/portfolio/${slug}/bulk-upload/${uploadId}`);
}
const offset = Math.max(0, parseInt(search.offset ?? "0", 10) || 0);
const limit = Math.max(1, Math.min(500, parseInt(search.limit ?? `${DEFAULT_LIMIT}`, 10) || DEFAULT_LIMIT));
const filter = search.filter === "missing" || search.filter === "duplicate" ? search.filter : "all";
const h = await headers();
const host = h.get("host");
const proto = h.get("x-forwarded-proto") ?? "http";
const cookieStore = await cookies();
const cookieHeader = cookieStore.getAll().map((c) => `${c.name}=${c.value}`).join("; ");
const url = `${proto}://${host}/api/portfolio/${upload.portfolioId}/bulk-uploads/${uploadId}/combined-results?offset=${offset}&limit=${limit}`;
let data: CombinedResultsResponse | null = null;
let fetchError: string | null = null;
try {
const res = await fetch(url, { headers: { Cookie: cookieHeader }, cache: "no-store" });
if (!res.ok) {
fetchError = `Failed to load results (${res.status})`;
} else {
data = (await res.json()) as CombinedResultsResponse;
}
} catch (err) {
console.error("Failed to fetch combined-results:", err);
fetchError = "Failed to load results";
}
return (
<div className="max-w-6xl mx-auto px-6 py-10">
<Link
href={`/portfolio/${slug}/bulk-upload/${uploadId}`}
className="inline-flex items-center gap-1.5 text-sm text-gray-400 hover:text-gray-700 transition-colors mb-8"
>
<ArrowLeftIcon className="h-4 w-4" />
Back to upload
</Link>
<div className="mb-8">
<p className="text-xs font-semibold text-gray-400 uppercase tracking-widest mb-1">
Review matches
</p>
<h1 className="text-3xl font-extrabold text-gray-900 tracking-tight mb-1">
{upload.filename}
</h1>
{data && (
<p className="text-sm text-gray-500">
{data.total} addresses ·{" "}
<span className="text-amber-600 font-semibold">{data.flags_summary.duplicates}</span> duplicates ·{" "}
<span className="text-red-600 font-semibold">{data.flags_summary.missing}</span> missing ·{" "}
<span className="text-green-600 font-semibold">{data.flags_summary.matched}</span> matched
</p>
)}
</div>
{fetchError && (
<div className="bg-red-50 border border-red-100 rounded-xl p-4 text-sm text-red-700">
{fetchError}
</div>
)}
{data && (
<ConfirmMatchesClient
data={data}
slug={slug}
uploadId={uploadId}
filter={filter}
offset={offset}
limit={limit}
/>
)}
</div>
);
}

View file

@ -53,6 +53,22 @@ const STATUS_CONFIG = {
body: "Your file is currently being processed. This may take a few minutes.",
cta: false,
},
combining: {
icon: ArrowPathIcon,
iconBg: "bg-blue-50",
iconColor: "text-blue-500",
title: "Combining results…",
body: "Your matched addresses are being assembled. Almost ready for review.",
cta: false,
},
awaiting_review: {
icon: CheckCircleIcon,
iconBg: "bg-green-50",
iconColor: "text-green-500",
title: "Ready for review",
body: "Your matches are ready. Review and confirm before finalising onboarding.",
cta: false,
},
complete: {
icon: CheckCircleIcon,
iconBg: "bg-green-50",
@ -150,7 +166,11 @@ export default async function BulkUploadDetailPage(props: {
</div>
)}
{(statusKey === "processing" || statusKey === "complete" || statusKey === "failed") &&
{(statusKey === "processing" ||
statusKey === "combining" ||
statusKey === "awaiting_review" ||
statusKey === "complete" ||
statusKey === "failed") &&
upload.taskId && (
<OnboardingProgress
taskId={upload.taskId}
@ -160,6 +180,16 @@ export default async function BulkUploadDetailPage(props: {
isDomnaUser={isDomnaUser}
/>
)}
{statusKey === "awaiting_review" && (
<Link
href={`/portfolio/${slug}/bulk-upload/${uploadId}/confirm-matches`}
className="mt-4 inline-flex items-center gap-2 px-5 py-2 rounded-xl bg-gradient-to-br from-[#14163d] to-[#15173e] text-white text-sm font-bold hover:opacity-90 transition-opacity"
>
Review matches
<ArrowRightIcon className="h-4 w-4" />
</Link>
)}
</div>
</div>
</div>