junte/assessment-model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/assessment-model.git synced 2026-07-19 17:02:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Merge pull request #334 from Hestia-Homes/feature/dedupe-documents

Browse source 
Dedupe documents by only fetching the most recent for each S3 key
This commit is contained in:
Daniel Roth 2026-07-01 09:27:18 +01:00 • committed by GitHub
commit 2ba084f858
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 29 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/app/api/live-tracking/property-documents/route.ts
View file

@ -40,7 +40,35 @@ export async function GET(req: Request) {
.from(uploadedFiles)
.where(condition);
const documents = rows.map((row) => ({
// Deduplicate rows that arise when the ingestion Lambda re-triggers for a deal
// that was already processed: same S3 key gets a new version, but a new DB row
// is written each time.
//
// Step 1: same s3FileKey → keep the row with the most recent upload timestamp.
const latestByS3Key = new Map<string, (typeof rows)[number]>();
for (const row of rows) {
const existing = latestByS3Key.get(row.s3FileKey);
if (!existing || row.s3UploadTimestamp > existing.s3UploadTimestamp) {
latestByS3Key.set(row.s3FileKey, row);
}
}
// Step 2: among distinct keys, same (fileType, measureName) → keep latest.
// Rows with null fileType (unclassified) are kept as-is.
const latestByDocType = new Map<string, (typeof rows)[number]>();
const unclassified: (typeof rows)[number][] = [];
for (const row of latestByS3Key.values()) {
if (!row.fileType) { unclassified.push(row); continue; }
const key = `${row.fileType}:${row.measureName ?? ""}`;
const existing = latestByDocType.get(key);
if (!existing || row.s3UploadTimestamp > existing.s3UploadTimestamp) {
latestByDocType.set(key, row);
}
}
const dedupedRows = [...latestByDocType.values(), ...unclassified];
const documents = dedupedRows.map((row) => ({
id: String(row.id),
s3FileKey: row.s3FileKey,
s3FileBucket: row.s3FileBucket,