From bee57a56b52fb5c9ed2bd7dea3737b77752b1a20 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 21 Apr 2026 20:24:26 +0000 Subject: [PATCH] address2uprn onboaridng poc --- .claude/settings.local.json | 11 +- .devcontainer/docker-compose.yml | 3 + .gitignore | 2 + ...s-when-combined_output_s3_uri-populated.md | 31 --- ...- Smoke-test-combiner-end-to-end-on-dev.md | 36 ---- ...-next-time-bulk_address_uploads-touched.md | 28 --- ...-call-backend-trigger-splitter-endpoint.md | 105 ---------- ...-route-to-call-backend-trigger-combiner.md | 32 --- ...irm-matches-page-for-bulk-upload-review.md | 34 --- ...or-combined-results-and-confirm-matches.md | 35 ---- .../bulk-uploads/[uploadId]/combine/route.ts | 34 ++- .../[uploadId]/confirm-matches/route.ts | 70 ------- .../bulk-uploads/[uploadId]/onboard/route.ts | 6 +- .../bulk-uploads/[uploadId]/route.ts | 25 +++ .../upload/bulk-addresses/confirm/route.ts | 7 +- .../portfolio/BulkUploadComingSoonModal.tsx | 6 +- .../[uploadId]/OnboardingProgress.tsx | 60 +++++- .../confirm-matches/ConfirmMatchesClient.tsx | 197 ++++++++++++++++++ .../[uploadId]/confirm-matches/page.tsx | 109 ++++++++++ .../bulk-upload/[uploadId]/page.tsx | 32 ++- 20 files changed, 467 insertions(+), 396 deletions(-) delete mode 100644 backlog/tasks/task-10 - Redirect-to-confirm-matches-when-combined_output_s3_uri-populated.md delete mode 100644 backlog/tasks/task-4 - Smoke-test-combiner-end-to-end-on-dev.md delete mode 100644 backlog/tasks/task-5 - Squash-migrations-0178-DROP-0179-re-ADD-next-time-bulk_address_uploads-touched.md delete mode 100644 backlog/tasks/task-6 - Refactor-onboard-route-to-call-backend-trigger-splitter-endpoint.md delete mode 100644 backlog/tasks/task-7 - Refactor-combine-route-to-call-backend-trigger-combiner.md delete mode 100644 backlog/tasks/task-8 - Add-confirm-matches-page-for-bulk-upload-review.md delete mode 100644 backlog/tasks/task-9 - Add-proxy-api-routes-for-combined-results-and-confirm-matches.md delete mode 100644 src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/confirm-matches/route.ts create mode 100644 src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/confirm-matches/ConfirmMatchesClient.tsx create mode 100644 src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/confirm-matches/page.tsx diff --git a/.claude/settings.local.json b/.claude/settings.local.json index b6b929a..dddaa7e 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -5,7 +5,16 @@ "Bash(backlog mcp *)", "Read(//home/vscode/.config/nvim/**)", "Read(//home/vscode/.config/nvim/lua/plugins/**)", - "Bash(npx tsc *)" + "Bash(npx tsc *)", + "Read(//workspaces/home/github/Model/backend/**)", + "Read(//workspaces/home/github/Model/etl/**)", + "mcp__backlog__task_create", + "mcp__backlog__task_view", + "mcp__backlog__task_edit", + "Read(//workspaces/home/github/Model/**)", + "Bash(pytest backend/tests/test_bulk_combiner_status.py -v --no-cov)", + "Bash(echo \"EXIT: $?\")", + "mcp__backlog__task_list" ] }, "enabledMcpjsonServers": [ diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index ed3c80c..2477976 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -17,6 +17,7 @@ services: - SSH_AUTH_SOCK=${SSH_AUTH_SOCK:-} networks: - frontend-net + - shared-dev pgadmin: image: dpage/pgadmin4 @@ -32,3 +33,5 @@ services: networks: frontend-net: driver: bridge + shared-dev: + external: true diff --git a/.gitignore b/.gitignore index 6fcf0a3..61bcc46 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ cypress.env.json # typescript *.tsbuildinfo next-env.d.ts + +backlog/* diff --git a/backlog/tasks/task-10 - Redirect-to-confirm-matches-when-combined_output_s3_uri-populated.md b/backlog/tasks/task-10 - Redirect-to-confirm-matches-when-combined_output_s3_uri-populated.md deleted file mode 100644 index 3dbff76..0000000 --- a/backlog/tasks/task-10 - Redirect-to-confirm-matches-when-combined_output_s3_uri-populated.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: TASK-10 -title: Redirect to confirm-matches when combined_output_s3_uri populated -status: To Do -assignee: [] -created_date: '2026-04-20' -updated_date: '2026-04-20' -labels: - - frontend - - bulk-upload - - ui -dependencies: - - TASK-8 -priority: medium -ordinal: 5000 ---- - -## Description - - -`OnboardingProgress.tsx` currently fires client-side combine POST when task terminal. Once backend auto-chains (backend-task-5) OR frontend triggers via backend route (task-7), the polling should watch for `bulk_address_uploads.combined_output_s3_uri` to be set. When present, show "Review matches →" CTA (or auto-redirect to confirm-matches page, task-8). - -May require a new GET endpoint that returns `{status, combined_output_s3_uri}` for polling. Or extend existing task summary with upload fields. - - -## Acceptance Criteria - -- [ ] #1 Polling stops once combined_output_s3_uri populated -- [ ] #2 UI surfaces a clear CTA to review matches -- [ ] #3 No duplicate combiner fires across refreshes - diff --git a/backlog/tasks/task-4 - Smoke-test-combiner-end-to-end-on-dev.md b/backlog/tasks/task-4 - Smoke-test-combiner-end-to-end-on-dev.md deleted file mode 100644 index 95911ab..0000000 --- a/backlog/tasks/task-4 - Smoke-test-combiner-end-to-end-on-dev.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -id: TASK-4 -title: Smoke-test full bulk upload flow end-to-end on dev -status: To Do -assignee: - - Jun-te Kim -created_date: '2026-04-18 19:02' -updated_date: '2026-04-20' -labels: - - qa - - bulk-upload -dependencies: - - TASK-6 - - TASK-7 - - TASK-8 - - TASK-9 - - TASK-10 -priority: medium -ordinal: 9000 ---- - -## Description - - -After frontend + backend refactor ships, run the full flow on dev: upload xlsx → map columns → start onboarding → poll task progress → combiner fires → combined_output_s3_uri populated → review matches page renders → confirm rows → addresses appear in portfolio. - - -## Acceptance Criteria - -- [ ] #1 Frontend no longer needs POSTCODE_SPLITTER_QUEUE_NAME or BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME env vars -- [ ] #2 Backend logs show both splitter + combiner triggered via HTTP route -- [ ] #3 bulk_final_outputs/{task_id}/combined_.csv exists in retrofit_sap_data bucket -- [ ] #4 bulk_address_uploads.combined_output_s3_uri populated -- [ ] #5 Confirm-matches page renders with match rows -- [ ] #6 After confirm submit, addresses persist into portfolio - diff --git a/backlog/tasks/task-5 - Squash-migrations-0178-DROP-0179-re-ADD-next-time-bulk_address_uploads-touched.md b/backlog/tasks/task-5 - Squash-migrations-0178-DROP-0179-re-ADD-next-time-bulk_address_uploads-touched.md deleted file mode 100644 index 2d64e04..0000000 --- a/backlog/tasks/task-5 - Squash-migrations-0178-DROP-0179-re-ADD-next-time-bulk_address_uploads-touched.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -id: TASK-5 -title: >- - Squash migrations 0178 (DROP) + 0179 (re-ADD) next time bulk_address_uploads - touched -status: Done -assignee: - - Jun-te Kim -created_date: '2026-04-18 19:02' -updated_date: '2026-04-18 19:06' -labels: - - tech-debt - - db -dependencies: [] -priority: low ---- - -## Description - - -0178 DROPs task_id + combined_output_s3_uri; 0179 re-ADDs them. Net-zero on live, wasted churn on fresh envs. Collapse to single migration next time schema changes in this area. - - -## Implementation Notes - - -Squashed in-place: deleted 0179.sql + 0179_snapshot.json, removed from _journal.json, patched 0178_snapshot.json to include task_id + combined_output_s3_uri cols. Orphan row may remain in live __drizzle_migrations but drizzle tolerates it. - diff --git a/backlog/tasks/task-6 - Refactor-onboard-route-to-call-backend-trigger-splitter-endpoint.md b/backlog/tasks/task-6 - Refactor-onboard-route-to-call-backend-trigger-splitter-endpoint.md deleted file mode 100644 index 310c2fa..0000000 --- a/backlog/tasks/task-6 - Refactor-onboard-route-to-call-backend-trigger-splitter-endpoint.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -id: TASK-6 -title: Refactor onboard route to call backend trigger-splitter endpoint -status: In Progress -assignee: [] -created_date: '2026-04-20' -updated_date: '2026-04-20 12:55' -labels: - - frontend - - bulk-upload - - refactor -dependencies: - - BACKEND-TASK-1 -priority: high -ordinal: 1000 ---- - -## Description - - -Currently `src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts` builds an SQS message and sends it to `POSTCODE_SPLITTER_QUEUE_NAME`. Move the SQS send to the backend. Frontend still transforms XLSX → CSV + uploads to S3, then calls backend HTTP `POST /v1/bulk-uploads/trigger-splitter`. Drop `POSTCODE_SPLITTER_QUEUE_NAME`, `sendToQueue`, and SQS IAM dependency. - - -## Acceptance Criteria - -- [x] #1 `onboard/route.ts` no longer imports `sendToQueue` or reads `POSTCODE_SPLITTER_QUEUE_NAME` -- [x] #2 Transformed CSV still uploaded to `bulk_onboarding_inputs/{portfolioId}/{uploadId}.csv` -- [x] #3 Backend trigger-splitter endpoint called with correct payload -- [ ] #4 DB updates (bulk_address_uploads.status="processing", tasks.status, subTasks.inputs) still happen on success -- [ ] #5 4xx/5xx from backend → return 502 to client with useful message - - -## Implementation Plan - - - - -### File changed -`src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts` - -### What stays the same -- Auth check via `getServerSession` -- Upload record + status validation -- `transformFile()` XLSX → CSV -- S3 upload of transformed CSV to `bulk_onboarding_inputs/{portfolioId}/{uploadId}.csv` -- DB writes: `bulkAddressUploads.status = "processing"`, `tasks.status = "in progress"`, `subTasks.inputs` - -### What changes - -**Remove:** -```ts -import { sendToQueue } from "@/app/utils/sqs"; -// POSTCODE_SPLITTER_QUEUE_NAME env var check -``` - -**Add — call backend trigger-splitter:** -```ts -const fastapiUrl = process.env.FASTAPI_API_URL; -const fastapiKey = process.env.FASTAPI_API_KEY; -if (!fastapiUrl || !fastapiKey) { - return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 }); -} - -const sessionToken = - request.cookies.get("__Secure-next-auth.session-token")?.value ?? - request.cookies.get("next-auth.session-token")?.value; - -const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-splitter`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": fastapiKey, - Authorization: `Bearer ${sessionToken}`, - }, - body: JSON.stringify({ - task_id: body.taskId, - sub_task_id: body.subTaskId, - s3_uri: s3Uri, - }), -}); - -if (!triggerRes.ok) { - const errText = await triggerRes.text().catch(() => ""); - console.error("Backend trigger-splitter failed:", triggerRes.status, errText); - return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 }); -} -``` - -**Order of ops** (no change to structure, just swap): -1. Validate body + upload record -2. Read source file from S3 -3. Transform XLSX → CSV -4. Upload transformed CSV to S3 -5. ~~sendToQueue~~ → fetch backend trigger-splitter -6. DB updates (status, taskId, subTask inputs) — only after step 5 succeeds - -### Env vars required -- `FASTAPI_API_URL` — already set, already used in `plan/trigger/route.ts` -- `FASTAPI_API_KEY` — already set - -### Env vars removed -- `POSTCODE_SPLITTER_QUEUE_NAME` — no longer read by frontend (can remove from .env.local + staging/prod) - - - diff --git a/backlog/tasks/task-7 - Refactor-combine-route-to-call-backend-trigger-combiner.md b/backlog/tasks/task-7 - Refactor-combine-route-to-call-backend-trigger-combiner.md deleted file mode 100644 index 19ad491..0000000 --- a/backlog/tasks/task-7 - Refactor-combine-route-to-call-backend-trigger-combiner.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -id: TASK-7 -title: Refactor combine route to call backend trigger-combiner endpoint -status: To Do -assignee: [] -created_date: '2026-04-20' -updated_date: '2026-04-20' -labels: - - frontend - - bulk-upload - - refactor -dependencies: - - BACKEND-TASK-2 -priority: high -ordinal: 2000 ---- - -## Description - - -`src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts` sends SQS directly to `BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME`. Replace with call to backend `POST /bulk-uploads/{task_id}/combine`. Drop queue name env var + SendMessage IAM dependency on frontend. - -If backend auto-chains combiner on splitter completion (backend-task-5), this route may simply proxy a manual "re-combine" action or be removed entirely. Decide during implementation. - - -## Acceptance Criteria - -- [ ] #1 `combine/route.ts` no longer imports `sendToQueue` or reads `BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME` -- [ ] #2 Backend trigger-combiner endpoint called with task_id -- [ ] #3 Frontend still updates subTasks row with inputs on success (or delegates to backend) -- [ ] #4 Decision logged: proxy vs delete-after-auto-chain - diff --git a/backlog/tasks/task-8 - Add-confirm-matches-page-for-bulk-upload-review.md b/backlog/tasks/task-8 - Add-confirm-matches-page-for-bulk-upload-review.md deleted file mode 100644 index 1334ca3..0000000 --- a/backlog/tasks/task-8 - Add-confirm-matches-page-for-bulk-upload-review.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -id: TASK-8 -title: Add confirm-matches page for bulk upload address→UPRN review -status: To Do -assignee: [] -created_date: '2026-04-20' -updated_date: '2026-04-20' -labels: - - frontend - - bulk-upload - - ui -dependencies: - - TASK-9 - - BACKEND-TASK-3 -priority: high -ordinal: 3000 ---- - -## Description - - -New route `/portfolio/{slug}/bulk-upload/{uploadId}/confirm-matches`. Loads combined CSV from backend (via frontend proxy route, see task-9) and renders review table: original address input | matched UPRN | matched address | confidence. User can accept/reject per row, then POST confirmed rows to backend to persist into portfolio `addresses`. - -Status transitions: `complete` (combiner done) → show "Review matches" CTA on upload detail page → confirm-matches page → on submit, move upload to `confirmed` or similar. - - -## Acceptance Criteria - -- [ ] #1 Page renders match rows in a scrollable table -- [ ] #2 Row actions: accept (default on), reject -- [ ] #3 Submit posts accepted rows to backend confirm-matches route -- [ ] #4 After submit, redirect to portfolio addresses list -- [ ] #5 No useEffect/useMemo (per CLAUDE.md) — use Server Components + Route Handlers where possible - diff --git a/backlog/tasks/task-9 - Add-proxy-api-routes-for-combined-results-and-confirm-matches.md b/backlog/tasks/task-9 - Add-proxy-api-routes-for-combined-results-and-confirm-matches.md deleted file mode 100644 index bcd2145..0000000 --- a/backlog/tasks/task-9 - Add-proxy-api-routes-for-combined-results-and-confirm-matches.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -id: TASK-9 -title: Add proxy API routes for combined-results and confirm-matches -status: To Do -assignee: [] -created_date: '2026-04-20' -updated_date: '2026-04-20' -labels: - - frontend - - bulk-upload - - api -dependencies: - - BACKEND-TASK-3 - - BACKEND-TASK-4 -priority: high -ordinal: 4000 ---- - -## Description - - -Two Next.js route handlers that proxy to backend: -- `GET /api/portfolio/{portfolioId}/bulk-uploads/{uploadId}/combined-results` → backend GET `/bulk-uploads/{uploadId}/combined-results`. Returns parsed match rows for the confirm UI. -- `POST /api/portfolio/{portfolioId}/bulk-uploads/{uploadId}/confirm-matches` → backend POST `/bulk-uploads/{uploadId}/confirm-matches`. Body: accepted rows. - -Both must: check session, pass through portfolio scope, translate backend errors to sane frontend responses. - - -## Acceptance Criteria - -- [ ] #1 Both routes auth-gated via getServerSession -- [ ] #2 Params typed as Promise per Next.js 15 convention -- [ ] #3 Backend 4xx/5xx surfaced with appropriate HTTP code + message -- [ ] #4 Upload id is validated against bulk_address_uploads row before proxying - diff --git a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts index 1ebb20b..673e2c5 100644 --- a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts +++ b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/combine/route.ts @@ -5,10 +5,9 @@ import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { getServerSession } from "next-auth"; import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; -import { sendToQueue } from "@/app/utils/sqs"; export async function POST( - _request: NextRequest, + request: NextRequest, { params }: { params: Promise<{ portfolioId: string; uploadId: string }> } ) { const session = await getServerSession(AuthOptions); @@ -28,9 +27,10 @@ export async function POST( if (upload.combinedOutputS3Uri) return NextResponse.json({ alreadyCombined: true }, { status: 200 }); - const queueName = process.env.BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME; - if (!queueName) { - console.error("BULK_ADDRESS2UPRN_COMBINER_QUEUE_NAME not set"); + const fastapiUrl = process.env.FASTAPI_API_URL; + const fastapiKey = process.env.FASTAPI_API_KEY; + if (!fastapiUrl || !fastapiKey) { + console.error("FASTAPI_API_URL or FASTAPI_API_KEY not set"); return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 }); } @@ -44,11 +44,29 @@ export async function POST( const messageBody = { task_id: upload.taskId, sub_task_id: subTask.id }; + const sessionToken = + request.cookies.get("__Secure-next-auth.session-token")?.value ?? + request.cookies.get("next-auth.session-token")?.value; + try { - await sendToQueue(messageBody, { queueName }); + const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-combiner`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": fastapiKey, + Authorization: `Bearer ${sessionToken}`, + }, + body: JSON.stringify(messageBody), + }); + + if (!triggerRes.ok) { + const errText = await triggerRes.text().catch(() => ""); + console.error("Backend trigger-combiner failed:", triggerRes.status, errText); + return NextResponse.json({ error: "Failed to trigger combiner" }, { status: 502 }); + } } catch (err) { - console.error("Failed to send combiner SQS message:", err); - return NextResponse.json({ error: "Failed to queue combiner job" }, { status: 500 }); + console.error("Failed to reach backend trigger-combiner:", err); + return NextResponse.json({ error: "Failed to trigger combiner" }, { status: 502 }); } await db diff --git a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/confirm-matches/route.ts b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/confirm-matches/route.ts deleted file mode 100644 index 53fb143..0000000 --- a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/confirm-matches/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { db } from "@/app/db/db"; -import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads"; -import { eq } from "drizzle-orm"; -import { NextRequest, NextResponse } from "next/server"; -import { getServerSession } from "next-auth"; -import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; - -export async function POST( - request: NextRequest, - { params }: { params: Promise<{ portfolioId: string; uploadId: string }> } -) { - const session = await getServerSession(AuthOptions); - if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - - const { uploadId } = await params; - - const [upload] = await db - .select({ taskId: bulkAddressUploads.taskId }) - .from(bulkAddressUploads) - .where(eq(bulkAddressUploads.id, uploadId)) - .limit(1); - - if (!upload) return NextResponse.json({ error: "Not found" }, { status: 404 }); - if (!upload.taskId) return NextResponse.json({ error: "Task not started" }, { status: 409 }); - - const fastapiUrl = process.env.FASTAPI_API_URL; - const fastapiKey = process.env.FASTAPI_API_KEY; - if (!fastapiUrl || !fastapiKey) { - console.error("FASTAPI_API_URL or FASTAPI_API_KEY not set"); - return NextResponse.json({ error: "Server misconfiguration" }, { status: 500 }); - } - - const sessionToken = - request.cookies.get("__Secure-next-auth.session-token")?.value ?? - request.cookies.get("next-auth.session-token")?.value; - - let body; - try { - body = await request.json(); - } catch { - return NextResponse.json({ error: "Invalid JSON" }, { status: 400 }); - } - - try { - const res = await fetch( - `${fastapiUrl}/v1/bulk-uploads/${upload.taskId}/confirm-matches`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": fastapiKey, - Authorization: `Bearer ${sessionToken}`, - }, - body: JSON.stringify(body), - } - ); - - if (!res.ok) { - const errText = await res.text().catch(() => ""); - console.error("Backend confirm-matches failed:", res.status, errText); - return NextResponse.json({ error: "Failed to confirm matches" }, { status: 502 }); - } - - const data = await res.json(); - return NextResponse.json(data, { status: 200 }); - } catch (err) { - console.error("Failed to reach backend confirm-matches:", err); - return NextResponse.json({ error: "Failed to confirm matches" }, { status: 502 }); - } -} diff --git a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts index 65c4edb..e5b77b2 100644 --- a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts +++ b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/onboard/route.ts @@ -143,7 +143,7 @@ export async function POST( request.cookies.get("next-auth.session-token")?.value; try { - const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-splitter`, { + const triggerRes = await fetch(`${fastapiUrl}/v1/bulk-uploads/trigger-postcode-splitter`, { method: "POST", headers: { "Content-Type": "application/json", @@ -159,11 +159,11 @@ export async function POST( if (!triggerRes.ok) { const errText = await triggerRes.text().catch(() => ""); - console.error("Backend trigger-splitter failed:", triggerRes.status, errText); + console.error("Backend trigger-postcode-splitter failed:", triggerRes.status, errText); return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 }); } } catch (err) { - console.error("Failed to reach backend trigger-splitter:", err); + console.error("Failed to reach backend trigger-postcode-splitter:", err); return NextResponse.json({ error: "Failed to trigger address matching" }, { status: 502 }); } diff --git a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/route.ts b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/route.ts index b61e05d..f51bd73 100644 --- a/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/route.ts +++ b/src/app/api/portfolio/[portfolioId]/bulk-uploads/[uploadId]/route.ts @@ -2,12 +2,37 @@ import { db } from "@/app/db/db"; import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; import { z } from "zod"; const PatchSchema = z.object({ columnMapping: z.record(z.string(), z.string()), }); +export async function GET( + _request: NextRequest, + { params }: { params: Promise<{ portfolioId: string; uploadId: string }> } +) { + const session = await getServerSession(AuthOptions); + if (!session) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + const { uploadId } = await params; + + const [upload] = await db + .select({ + status: bulkAddressUploads.status, + combinedOutputS3Uri: bulkAddressUploads.combinedOutputS3Uri, + }) + .from(bulkAddressUploads) + .where(eq(bulkAddressUploads.id, uploadId)) + .limit(1); + + if (!upload) return NextResponse.json({ error: "Not found" }, { status: 404 }); + + return NextResponse.json(upload, { status: 200 }); +} + export async function PATCH( request: NextRequest, { params }: { params: Promise<{ portfolioId: string; uploadId: string }> } diff --git a/src/app/api/upload/bulk-addresses/confirm/route.ts b/src/app/api/upload/bulk-addresses/confirm/route.ts index edc6357..8edaab4 100644 --- a/src/app/api/upload/bulk-addresses/confirm/route.ts +++ b/src/app/api/upload/bulk-addresses/confirm/route.ts @@ -8,7 +8,12 @@ const BodySchema = z.object({ filename: z.string(), portfolioId: z.string(), userId: z.string(), - sourceHeaders: z.array(z.string()).default([]), + sourceHeaders: z + .array(z.union([z.string(), z.null(), z.undefined()])) + .default([]) + .transform((arr) => + arr.filter((h): h is string => typeof h === "string" && h.trim().length > 0) + ), }); export async function POST(request: NextRequest) { diff --git a/src/app/components/portfolio/BulkUploadComingSoonModal.tsx b/src/app/components/portfolio/BulkUploadComingSoonModal.tsx index 1cd47bf..a416f00 100644 --- a/src/app/components/portfolio/BulkUploadComingSoonModal.tsx +++ b/src/app/components/portfolio/BulkUploadComingSoonModal.tsx @@ -77,10 +77,12 @@ async function validateHeaders(file: File): Promise<{ error: string | null; head const buffer = await file.arrayBuffer(); const wb = XLSX.read(buffer, { sheetRows: 1 }); const sheet = wb.Sheets[wb.SheetNames[0]]; - const rows = XLSX.utils.sheet_to_json(sheet, { header: 1 }); - headers = ((rows[0] as string[]) ?? []).map((h) => String(h ?? "").trim()); + const rows = XLSX.utils.sheet_to_json(sheet, { header: 1, defval: "" }); + headers = ((rows[0] as unknown[]) ?? []).map((h) => String(h ?? "").trim()); } + headers = headers.filter((h) => h.length > 0); + const normalised = headers.map((h) => h.toLowerCase()); const hasAddress = normalised.some((h) => h.startsWith("address")); const hasPostcode = normalised.some((h) => h === "postcode"); diff --git a/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/OnboardingProgress.tsx b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/OnboardingProgress.tsx index a4afb30..7effd7f 100644 --- a/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/OnboardingProgress.tsx +++ b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/OnboardingProgress.tsx @@ -1,6 +1,7 @@ "use client"; import { useEffect, useState, useRef } from "react"; +import { useRouter } from "next/navigation"; import Link from "next/link"; interface TaskData { @@ -12,6 +13,11 @@ interface TaskData { failedSubtasks: number; } +interface UploadStatus { + status: string; + combinedOutputS3Uri: string | null; +} + interface Props { taskId: string; portfolioSlug: string; @@ -30,10 +36,13 @@ export default function OnboardingProgress({ uploadId, isDomnaUser, }: Props) { + const router = useRouter(); const [data, setData] = useState(null); + const [uploadStatus, setUploadStatus] = useState(null); const [fetchError, setFetchError] = useState(false); const intervalRef = useRef | null>(null); const combineFiredRef = useRef(false); + const redirectedRef = useRef(false); useEffect(() => { async function poll() { @@ -43,14 +52,30 @@ export default function OnboardingProgress({ const json: TaskData = await res.json(); setData(json); const status = json.status.toLowerCase(); + if (TERMINAL_STATUSES.has(status)) { - if (intervalRef.current) clearInterval(intervalRef.current); if (!FAILED_STATUSES.has(status) && !combineFiredRef.current) { combineFiredRef.current = true; fetch(`/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}/combine`, { method: "POST", }).catch((err) => console.error("Failed to trigger combiner:", err)); } + + const uploadRes = await fetch( + `/api/portfolio/${portfolioId}/bulk-uploads/${uploadId}` + ); + if (uploadRes.ok) { + const upload: UploadStatus = await uploadRes.json(); + setUploadStatus(upload); + if (upload.status === "awaiting_review" && !redirectedRef.current) { + redirectedRef.current = true; + if (intervalRef.current) clearInterval(intervalRef.current); + router.push( + `/portfolio/${portfolioSlug}/bulk-upload/${uploadId}/confirm-matches` + ); + return; + } + } } } catch { setFetchError(true); @@ -60,7 +85,7 @@ export default function OnboardingProgress({ poll(); intervalRef.current = setInterval(poll, 3000); return () => { if (intervalRef.current) clearInterval(intervalRef.current); }; - }, [taskId, portfolioId, uploadId]); + }, [taskId, portfolioId, portfolioSlug, uploadId, router]); if (fetchError) return null; if (!data) { @@ -76,12 +101,15 @@ export default function OnboardingProgress({ const complete = data.completedSubtasks; const failed = data.failedSubtasks; const percent = total > 0 ? Math.round((complete / total) * 100) : 0; - const isDone = TERMINAL_STATUSES.has(data.status.toLowerCase()); - const isFailed = ["failed", "failure", "error"].includes(data.status.toLowerCase()); + const taskDone = TERMINAL_STATUSES.has(data.status.toLowerCase()); + const isFailed = FAILED_STATUSES.has(data.status.toLowerCase()); + const isCombining = + taskDone && !isFailed && uploadStatus?.status === "combining"; + const isAwaitingReview = + taskDone && !isFailed && uploadStatus?.status === "awaiting_review"; return (
- {/* Progress bar */}
- {/* Counts */}
{total > 0 && ( @@ -102,20 +129,35 @@ export default function OnboardingProgress({ {failed} failed )} - {!isDone && ( + {!taskDone && ( Running )} - {isDone && !isFailed && ( + {isCombining && ( + + + Combining results… + + )} + {isAwaitingReview && ( - Complete + Ready for review )}
+ {isAwaitingReview && ( + + Review matches + + )} + {isDomnaUser && ( r.flags.includes(filter)); + + const basePath = `/portfolio/${slug}/bulk-upload/${uploadId}/confirm-matches`; + const pageStart = data.total === 0 ? 0 : offset + 1; + const pageEnd = Math.min(offset + data.rows.length, data.total); + const hasPrev = offset > 0; + const hasNext = offset + limit < data.total; + const prevOffset = Math.max(0, offset - limit); + const nextOffset = offset + limit; + + return ( +
+
+ + All ({data.total}) + + + Missing ({data.flags_summary.missing}) + + + Duplicates ({data.flags_summary.duplicates}) + +
+ +
+ + + + + + + + + + + + + + {rows.length === 0 && ( + + + + )} + {rows.map((row) => ( + + + + + + + + + + ))} + +
Internal RefInput AddressUPRNMatched AddressScoreFlagsActions
+ No rows match this filter. +
{row.internal_reference ?? "—"}{row.input_address || "—"}{row.uprn ?? "—"}{row.matched_address ?? "—"} + + {scoreChipLabel(row.score_bucket)} + + +
+ {row.flags.map((f) => ( + + {f === "missing" ? "Missing" : "Duplicate"} + + ))} + {row.flags.length === 0 && —} +
+ 		+ +
+
+ +
+ + Showing {pageStart}–{pageEnd} of {data.total} + +
+ {hasPrev ? ( + + Prev + + ) : ( + + Prev + + )} + {hasNext ? ( + + Next + + ) : ( + + Next + + )} +
+
+
+ ); +} diff --git a/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/confirm-matches/page.tsx b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/confirm-matches/page.tsx new file mode 100644 index 0000000..9cfac38 --- /dev/null +++ b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/confirm-matches/page.tsx @@ -0,0 +1,109 @@ +"use server"; + +import { db } from "@/app/db/db"; +import { bulkAddressUploads } from "@/app/db/schema/bulk_address_uploads"; +import { eq } from "drizzle-orm"; +import { getServerSession } from "next-auth"; +import { AuthOptions } from "@/app/api/auth/[...nextauth]/authOptions"; +import { redirect, notFound } from "next/navigation"; +import { cookies, headers } from "next/headers"; +import Link from "next/link"; +import { ArrowLeftIcon } from "@heroicons/react/24/outline"; +import ConfirmMatchesClient, { + CombinedResultsResponse, +} from "./ConfirmMatchesClient"; + +const DEFAULT_LIMIT = 100; + +export default async function ConfirmMatchesPage(props: { + params: Promise<{ slug: string; uploadId: string }>; + searchParams: Promise<{ offset?: string; limit?: string; filter?: string }>; +}) { + const { slug, uploadId } = await props.params; + const search = await props.searchParams; + const session = await getServerSession(AuthOptions); + if (!session) redirect("/login"); + + const [upload] = await db + .select() + .from(bulkAddressUploads) + .where(eq(bulkAddressUploads.id, uploadId)) + .limit(1); + + if (!upload) notFound(); + if (upload.status !== "awaiting_review") { + redirect(`/portfolio/${slug}/bulk-upload/${uploadId}`); + } + + const offset = Math.max(0, parseInt(search.offset ?? "0", 10) || 0); + const limit = Math.max(1, Math.min(500, parseInt(search.limit ?? `${DEFAULT_LIMIT}`, 10) || DEFAULT_LIMIT)); + const filter = search.filter === "missing" || search.filter === "duplicate" ? search.filter : "all"; + + const h = await headers(); + const host = h.get("host"); + const proto = h.get("x-forwarded-proto") ?? "http"; + const cookieStore = await cookies(); + const cookieHeader = cookieStore.getAll().map((c) => `${c.name}=${c.value}`).join("; "); + + const url = `${proto}://${host}/api/portfolio/${upload.portfolioId}/bulk-uploads/${uploadId}/combined-results?offset=${offset}&limit=${limit}`; + + let data: CombinedResultsResponse | null = null; + let fetchError: string | null = null; + try { + const res = await fetch(url, { headers: { Cookie: cookieHeader }, cache: "no-store" }); + if (!res.ok) { + fetchError = `Failed to load results (${res.status})`; + } else { + data = (await res.json()) as CombinedResultsResponse; + } + } catch (err) { + console.error("Failed to fetch combined-results:", err); + fetchError = "Failed to load results"; + } + + return ( +
+ + + Back to upload + + +
+

+ Review matches +

+

+ {upload.filename} +

+ {data && ( +

+ {data.total} addresses ·{" "} + {data.flags_summary.duplicates} duplicates ·{" "} + {data.flags_summary.missing} missing ·{" "} + {data.flags_summary.matched} matched +

+ )} +
+ + {fetchError && ( +
+ {fetchError} +
+ )} + + {data && ( + + )} +
+ ); +} diff --git a/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/page.tsx b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/page.tsx index 5f7ca9b..438b9ad 100644 --- a/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/page.tsx +++ b/src/app/portfolio/[slug]/(portfolio)/bulk-upload/[uploadId]/page.tsx @@ -53,6 +53,22 @@ const STATUS_CONFIG = { body: "Your file is currently being processed. This may take a few minutes.", cta: false, }, + combining: { + icon: ArrowPathIcon, + iconBg: "bg-blue-50", + iconColor: "text-blue-500", + title: "Combining results…", + body: "Your matched addresses are being assembled. Almost ready for review.", + cta: false, + }, + awaiting_review: { + icon: CheckCircleIcon, + iconBg: "bg-green-50", + iconColor: "text-green-500", + title: "Ready for review", + body: "Your matches are ready. Review and confirm before finalising onboarding.", + cta: false, + }, complete: { icon: CheckCircleIcon, iconBg: "bg-green-50", @@ -150,7 +166,11 @@ export default async function BulkUploadDetailPage(props: {
)} - {(statusKey === "processing" || statusKey === "complete" || statusKey === "failed") && + {(statusKey === "processing" || + statusKey === "combining" || + statusKey === "awaiting_review" || + statusKey === "complete" || + statusKey === "failed") && upload.taskId && ( )} + + {statusKey === "awaiting_review" && ( + + Review matches + + + )}