added changes for potential upload fix past 1mb size limit

This commit is contained in:
Jun-te Kim 2025-11-14 11:22:17 +00:00
parent 27d88f1e9a
commit c101b5b13a
6 changed files with 332 additions and 182 deletions

View file

@ -7,8 +7,6 @@ services:
context: ..
dockerfile: .devcontainer/Dockerfile
command: sleep infinity
ports:
- "3000:3000"
volumes:
- ..:/workspaces/assessment-model
networks:

View file

@ -1 +1 @@
npm run dev
npm run dev -- -p 3000

View file

@ -1,11 +1,13 @@
export const runtime = "nodejs";
import { NextRequest, NextResponse } from "next/server";
import { S3Client, PutObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3";
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { db } from "@/app/db/db";
import { filesFromSurveyor } from "@/app/db/schema/files_from_surveyor";
import { and, eq } from "drizzle-orm";
// ✅ Initialize S3 Client (AWS SDK v3)
// ✅ Initialize S3 Client
const s3 = new S3Client({
region: process.env.RETROFIT_DATA_DEV_REGION,
credentials: {
@ -14,90 +16,18 @@ const s3 = new S3Client({
},
});
// ✅ Helper: safely convert BigInt fields to string
// ✅ Helper to safely convert BigInts
function serializeBigInt<T extends Record<string, any>>(obj: T): T {
return Object.fromEntries(
Object.entries(obj).map(([key, value]) => [
key,
typeof value === "bigint" ? value.toString() : value,
Object.entries(obj).map(([k, v]) => [
k,
typeof v === "bigint" ? v.toString() : v,
])
) as T;
}
// -------------------
// 📤 POST — Upload files to S3 + Save to DB
// -------------------
export async function POST(request: NextRequest) {
try {
const formData = await request.formData();
const files = formData.getAll("files") as File[];
const portfolioIdStr = formData.get("portfolioId") as string;
const propertyIdStr = formData.get("propertyId") as string;
if (!portfolioIdStr || !propertyIdStr) {
return NextResponse.json(
{ msg: "Missing portfolioId or propertyId" },
{ status: 400 }
);
}
const portfolioId = BigInt(portfolioIdStr);
const propertyId = BigInt(propertyIdStr);
if (!files || files.length === 0) {
return NextResponse.json({ msg: "No files uploaded" }, { status: 400 });
}
const uploadedFiles = await Promise.all(
files.map(async (file) => {
const timestamp = Date.now();
const safeName = file.name.replace(/\s+/g, "_");
const fileKey = `files_from_surveyor/${portfolioId}/${propertyId}/${timestamp}-${safeName}`;
const buffer = Buffer.from(await file.arrayBuffer());
await s3.send(
new PutObjectCommand({
Bucket: process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME!,
Key: fileKey,
Body: buffer,
ContentType: file.type,
})
);
const fileUrl = `https://${process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME}.s3.${process.env.RETROFIT_DATA_DEV_REGION}.amazonaws.com/${fileKey}`;
return { fileKey, fileUrl };
})
);
const inserted = await db
.insert(filesFromSurveyor)
.values(
uploadedFiles.map((file) => ({
portfolioId,
propertyId,
s3JsonUrl: file.fileUrl,
}))
)
.returning();
// ✅ Serialize BigInts before sending JSON
const safeInserted = inserted.map(serializeBigInt);
return NextResponse.json({ files: safeInserted }, { status: 200 });
} catch (error) {
console.error("❌ S3 or DB Error:", error);
return NextResponse.json(
{ msg: "File upload failed", error: String(error) },
{ status: 500 }
);
}
}
// -------------------
// 📥 GET — Fetch uploaded files + Presigned URLs
// 📥 GET — Fetch files from DB + temporary S3 view URLs
// -------------------
export async function GET(request: NextRequest) {
try {
@ -115,7 +45,6 @@ export async function GET(request: NextRequest) {
const portfolioId = BigInt(portfolioIdStr);
const propertyId = BigInt(propertyIdStr);
// ✅ Correct Drizzle where syntax
const files = await db
.select()
.from(filesFromSurveyor)
@ -126,18 +55,23 @@ export async function GET(request: NextRequest) {
)
);
// ✅ Add 30-min presigned URLs
const withSignedUrls = await Promise.all(
// ✅ Add temporary presigned GET URLs (30-min expiry)
const filesWithSignedUrls = await Promise.all(
files.map(async (file) => {
const key = file.s3JsonUrl.split(".amazonaws.com/")[1];
const presignedUrl = await getSignedUrl(
s3,
new GetObjectCommand({
Bucket: process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME!,
Key: key,
}),
{ expiresIn: 60 * 30 } // 30 minutes
);
let presignedUrl: string | null = null;
try {
const key = file.s3JsonUrl.split(".amazonaws.com/")[1];
presignedUrl = await getSignedUrl(
s3,
new GetObjectCommand({
Bucket: process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME!,
Key: key,
}),
{ expiresIn: 60 * 30 } // 30 minutes
);
} catch (err) {
console.warn("⚠️ Failed to create presigned URL:", err);
}
return serializeBigInt({
...file,
@ -146,11 +80,11 @@ export async function GET(request: NextRequest) {
})
);
return NextResponse.json({ files: withSignedUrls }, { status: 200 });
} catch (error) {
console.error("❌ Fetch error:", error);
return NextResponse.json({ files: filesWithSignedUrls }, { status: 200 });
} catch (err) {
console.error("❌ retrofit-data GET error:", err);
return NextResponse.json(
{ msg: "Error fetching files", error: String(error) },
{ msg: "Error fetching files", error: String(err) },
{ status: 500 }
);
}

View file

@ -0,0 +1,57 @@
export const runtime = "nodejs";
import { NextRequest, NextResponse } from "next/server";
import { db } from "@/app/db/db";
import { filesFromSurveyor } from "@/app/db/schema/files_from_surveyor";
// ✅ Helper: safely convert BigInt fields to string
function serializeBigInt<T extends Record<string, any>>(obj: T): T {
return Object.fromEntries(
Object.entries(obj).map(([key, value]) => [
key,
typeof value === "bigint" ? value.toString() : value,
])
) as T;
}
// -------------------
// 📤 POST — Save uploaded file URLs (from S3) to DB
// -------------------
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const { portfolioId, propertyId, s3JsonUrl } = body;
if (!portfolioId || !propertyId || !s3JsonUrl) {
return NextResponse.json(
{ msg: "Missing portfolioId, propertyId, or s3JsonUrl" },
{ status: 400 }
);
}
const portfolioIdBigInt = BigInt(portfolioId);
const propertyIdBigInt = BigInt(propertyId);
const inserted = await db
.insert(filesFromSurveyor)
.values({
portfolioId: portfolioIdBigInt,
propertyId: propertyIdBigInt,
s3JsonUrl,
})
.returning();
const safeInserted = inserted.map(serializeBigInt);
return NextResponse.json(
{ msg: "✅ File record saved", files: safeInserted },
{ status: 200 }
);
} catch (error) {
console.error("❌ DB insert error:", error);
return NextResponse.json(
{ msg: "Failed to save file record", error: String(error) },
{ status: 500 }
);
}
}

View file

@ -0,0 +1,52 @@
// ✅ Force Node.js runtime (not Edge)
export const runtime = "nodejs";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { NextResponse } from "next/server";
// ✅ Initialize S3 client with credentials
const s3 = new S3Client({
region: process.env.RETROFIT_DATA_DEV_REGION,
credentials: {
accessKeyId: process.env.RETROFIT_DATA_DEV_ACCESS_KEY!,
secretAccessKey: process.env.RETROFIT_DATA_DEV_SECRET_KEY!,
},
});
export async function POST(req: Request) {
try {
const { fileName, fileType, portfolioId, propertyId } = await req.json();
if (!fileName || !fileType || !portfolioId || !propertyId) {
return NextResponse.json(
{ msg: "Missing required fields" },
{ status: 400 }
);
}
const safeName = fileName.replace(/\s+/g, "_");
const key = `files_from_surveyor/${portfolioId}/${propertyId}/${Date.now()}-${safeName}`;
// ✅ Create S3 PutObjectCommand (no upload yet — just sign it)
const command = new PutObjectCommand({
Bucket: process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME!,
Key: key,
ContentType: fileType,
});
// ✅ Generate presigned upload URL (valid 5 minutes)
const uploadUrl = await getSignedUrl(s3, command, { expiresIn: 60 * 5 });
// ✅ Public file URL once uploaded
const publicUrl = `https://${process.env.RETROFIT_DATA_DEV_S3_BUCKET_NAME}.s3.${process.env.RETROFIT_DATA_DEV_REGION}.amazonaws.com/${key}`;
return NextResponse.json({ uploadUrl, publicUrl }, { status: 200 });
} catch (err) {
console.error("❌ Error generating presigned URL:", err);
return NextResponse.json(
{ msg: "Failed to generate presigned URL", error: String(err) },
{ status: 500 }
);
}
}

View file

@ -4,7 +4,7 @@ import React, { useState, useEffect } from "react";
import { useParams } from "next/navigation";
interface FileRecord {
id: number;
id?: number;
s3JsonUrl: string;
portfolioId: string;
propertyId: string;
@ -13,120 +13,229 @@ interface FileRecord {
}
const UploadPage: React.FC = () => {
const [isUploading, setIsUploading] = useState(false);
const [files, setFiles] = useState<FileRecord[]>([]);
const [isUploading, setIsUploading] = useState(false);
const [uploadProgress, setUploadProgress] = useState<number>(0);
const params = useParams();
const portfolioId = params?.slug as string;
const propertyId = params?.propertyId as string;
// -------------------
// Fetch files from DB
// -------------------
const fetchFiles = async () => {
const res = await fetch(
`/api/upload/retrofit-data?portfolioId=${portfolioId}&propertyId=${propertyId}`
);
if (res.ok) {
try {
const res = await fetch(
`/api/upload/retrofit-data?portfolioId=${portfolioId}&propertyId=${propertyId}`
);
if (!res.ok) throw new Error("Failed to fetch files");
const data = await res.json();
console.log(data, "hello");
setFiles(data.files);
setFiles(data.files || []);
} catch (err) {
console.error("❌ Fetch files error:", err);
}
};
useEffect(() => {
if (portfolioId && propertyId) {
fetchFiles();
}
if (portfolioId && propertyId) fetchFiles();
}, [portfolioId, propertyId]);
// -------------------
// Handle file upload
// -------------------
const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const files = e.target.files;
if (!files || files.length === 0) return;
const selectedFiles = e.target.files;
if (!selectedFiles || selectedFiles.length === 0) return;
const formData = new FormData();
formData.append("portfolioId", portfolioId);
formData.append("propertyId", propertyId);
Array.from(files).forEach((file) => formData.append("files", file));
setIsUploading(true);
setUploadProgress(0);
try {
setIsUploading(true);
const res = await fetch("/api/upload/retrofit-data", {
method: "POST",
body: formData,
});
if (!res.ok) throw new Error("Upload failed");
const filesArray = Array.from(selectedFiles);
for (let i = 0; i < filesArray.length; i++) {
const file = filesArray[i];
// 1⃣ Get presigned URL
const signRes = await fetch("/api/upload/sign-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
fileName: file.name,
fileType: file.type,
portfolioId,
propertyId,
}),
});
if (!signRes.ok) throw new Error("Failed to get signed URL");
const { uploadUrl, publicUrl } = await signRes.json();
// 2⃣ Upload directly to S3 (with progress)
await new Promise<void>((resolve, reject) => {
const uploadReq = new XMLHttpRequest();
uploadReq.open("PUT", uploadUrl, true);
uploadReq.setRequestHeader("Content-Type", file.type);
uploadReq.upload.onprogress = (event: ProgressEvent) => {
if (event.lengthComputable) {
const percent = Math.round((event.loaded / event.total) * 100);
setUploadProgress(percent);
}
};
uploadReq.onload = () => {
if (uploadReq.status === 200) resolve();
else reject(new Error(`S3 upload failed for ${file.name}`));
};
uploadReq.onerror = (err: ProgressEvent<EventTarget>) => {
console.error("❌ Upload network error:", err);
reject(new Error(`Network error while uploading ${file.name}`));
};
uploadReq.send(file);
});
// 3⃣ Save file record in DB
const saveRes = await fetch("/api/upload/save-to-db", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
portfolioId,
propertyId,
s3JsonUrl: publicUrl,
}),
});
if (!saveRes.ok) throw new Error("Failed to save file record");
// Update overall progress
setUploadProgress(Math.round(((i + 1) / filesArray.length) * 100));
}
await fetchFiles();
alert("✅ Files uploaded successfully!");
alert("✅ All files uploaded successfully!");
} catch (err) {
console.error(err);
alert("❌ Upload failed");
console.error("❌ Upload error:", err);
alert("❌ Upload failed. Please try again.");
} finally {
setIsUploading(false);
setUploadProgress(0);
}
};
// -------------------
// UI
// -------------------
return (
<div className="flex flex-col items-center justify-start min-h-screen gap-6 bg-gray-50 p-8">
<h1 className="text-2xl font-semibold">Upload Retrofit Data Files</h1>
<div className="text-gray-700 text-center">
<p>
<strong>Portfolio ID:</strong> {portfolioId}
<div className="min-h-screen bg-gray-50 flex flex-col items-center py-10 px-6">
<div className="w-full max-w-4xl bg-white shadow-xl rounded-2xl p-8">
<h1 className="text-3xl font-bold text-gray-900 mb-2">
Upload Retrofit Data
</h1>
<p className="text-gray-600 mb-6">
Upload files securely to our cloud storage. Each upload will
automatically link to the selected portfolio and property.
</p>
<p>
<strong>Property ID:</strong> {propertyId}
</p>
</div>
<label
htmlFor="file-upload"
className={`cursor-pointer px-5 py-3 rounded-lg transition ${
isUploading
? "bg-gray-400 cursor-not-allowed"
: "bg-blue-600 hover:bg-blue-700 text-white"
}`}
>
{isUploading ? "Uploading..." : "Choose Files"}
</label>
<div className="bg-gray-100 p-4 rounded-lg mb-6 text-gray-700">
<p>
<strong>Portfolio ID:</strong> {portfolioId}
</p>
<p>
<strong>Property ID:</strong> {propertyId}
</p>
</div>
<input
id="file-upload"
type="file"
multiple
onChange={handleFileChange}
disabled={isUploading}
className="hidden"
/>
<div className="flex items-center gap-4 mb-6">
<label
htmlFor="file-upload"
className={`px-6 py-3 rounded-lg font-medium transition-all ${
isUploading
? "bg-gray-400 cursor-not-allowed text-white"
: "bg-blue-600 hover:bg-blue-700 text-white cursor-pointer"
}`}
>
{isUploading ? "Uploading..." : "Select Files"}
</label>
<div className="w-full max-w-3xl mt-10">
<h2 className="text-lg font-medium mb-4">Uploaded Files</h2>
{files.length === 0 ? (
<p className="text-gray-500">No files uploaded yet.</p>
) : (
<table className="min-w-full border border-gray-200 bg-white rounded-lg shadow-sm">
<thead className="bg-gray-100">
<tr>
<th className="p-3 text-left">File URL</th>
<th className="p-3 text-left">Action</th>
</tr>
</thead>
<tbody>
{files.map((file) => (
<tr key={file.id} className="border-t">
<td className="p-3 text-blue-700 break-all">
{file.s3JsonUrl}
</td>
<td className="p-3">
<button
onClick={() => window.open(file.presignedUrl, "_blank")}
className="px-3 py-1 text-sm bg-green-600 text-white rounded hover:bg-green-700"
>
View
</button>
</td>
</tr>
))}
</tbody>
</table>
<input
id="file-upload"
type="file"
multiple
onChange={handleFileChange}
disabled={isUploading}
className="hidden"
/>
</div>
{isUploading && (
<div className="w-full bg-gray-200 rounded-full h-3 mb-6 overflow-hidden">
<div
className="h-3 bg-blue-600 transition-all"
style={{ width: `${uploadProgress}%` }}
/>
</div>
)}
<div className="mt-8">
<h2 className="text-lg font-semibold mb-4 text-gray-800">
Uploaded Files
</h2>
{files.length === 0 ? (
<div className="text-gray-500 text-sm italic">
No files uploaded yet.
</div>
) : (
<div className="overflow-x-auto border border-gray-200 rounded-lg">
<table className="min-w-full text-sm text-left text-gray-700">
<thead className="bg-gray-100 text-gray-800 uppercase text-xs">
<tr>
<th className="px-4 py-3">File Name</th>
<th className="px-4 py-3">Uploaded</th>
<th className="px-4 py-3 text-center">Action</th>
</tr>
</thead>
<tbody>
{files.map((file) => (
<tr
key={file.id || file.s3JsonUrl}
className="border-t hover:bg-gray-50 transition"
>
<td className="px-4 py-3 break-all text-blue-700">
{decodeURIComponent(
file.s3JsonUrl.split("/").pop() || ""
)}
</td>
<td className="px-4 py-3 text-gray-600">
{file.createdAt
? new Date(file.createdAt).toLocaleString()
: "—"}
</td>
<td className="px-4 py-3 text-center">
<button
onClick={() =>
window.open(
file.presignedUrl || file.s3JsonUrl,
"_blank"
)
}
className="px-3 py-1 bg-green-600 hover:bg-green-700 text-white rounded-md text-xs font-medium"
>
View File
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</div>
</div>
</div>
);