upload file to s3 and update db after doing so

backend/ecmk_fetcher/processor.py

@@ -29,7 +29,10 @@ from backend.ecmk_fetcher.reports import (
     build_property_id,
     map_report_type_to_db_file_type,
 )
-from backend.ecmk_fetcher.sharepoint import upload_file_to_sharepoint
+from backend.ecmk_fetcher.upload import (
+    upload_file_to_s3_and_update_db,
+    upload_file_to_sharepoint,
+)
 from utils.logger import setup_logger
 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
 from utils.sharepoint.domna_sites import DomnaSites
@@ -56,7 +59,7 @@ def run_job() -> None:
 
     sharepoint_base_path: str = "/Projects/Southern Housing/SH-SURV-26-001/Assessments"
 
-    # s3_bucket: str = "retrofit-energy-assessments-dev"
+    s3_bucket: str = "retrofit-energy-assessments-dev"
 
     with sync_playwright() as p:
         browser: Browser = p.chromium.launch(headless=True)
@@ -146,6 +149,15 @@ def run_job() -> None:
                                 logger.info(
                                     f"Successfully loaded {os.path.basename(file_path)} to sharepoint for {address}"
                                 )
+
+                                # Upload to s3 and update db
+                                upload_file_to_s3_and_update_db(
+                                    bucket=s3_bucket,
+                                    file_path=file_path,
+                                    hubspot_listing_id=hubspot_listing_id,
+                                    file_type=db_file_type,
+                                )
+
                             except Exception:
                                 raise
                             finally:

backend/ecmk_fetcher/sharepoint.py

@@ -1,20 +0,0 @@
-import os
-
-from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
-
-
-def upload_file_to_sharepoint(
-    client: DomnaSharepointClient,
-    file_path: str,
-    base_path: str,
-    subpath: str,
-) -> None:
-    filename = os.path.basename(file_path)
-
-    full_path = f"{base_path}/{subpath}/1. Retrofit Assessment/A. Assessment"
-
-    client.upload_file(
-        file_path=file_path,
-        sharepoint_path=full_path,
-        file_name=filename,
-    )

backend/ecmk_fetcher/upload.py

@@ -0,0 +1,49 @@
+from datetime import datetime, timezone
+import os
+
+from backend.app.db.connection import db_session
+from backend.app.db.models.uploaded_file import (
+    FileSourceEnum,
+    FileTypeEnum,
+    UploadedFile,
+)
+from utils.s3 import upload_file_to_s3
+from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
+
+
+def upload_file_to_sharepoint(
+    client: DomnaSharepointClient,
+    file_path: str,
+    base_path: str,
+    subpath: str,
+) -> None:
+    filename = os.path.basename(file_path)
+
+    full_path = f"{base_path}/{subpath}/1. Retrofit Assessment/A. Assessment"
+
+    client.upload_file(
+        file_path=file_path,
+        sharepoint_path=full_path,
+        file_name=filename,
+    )
+
+
+def upload_file_to_s3_and_update_db(
+    bucket: str, file_path: str, hubspot_listing_id: str, file_type: FileTypeEnum
+) -> None:
+    key: str = f"documents/hubspot_listing_id/{hubspot_listing_id}"
+    upload_file_to_s3(file_path, bucket, key)
+
+    uploaded_file = UploadedFile(
+        s3_file_bucket=bucket,
+        s3_file_key=key,
+        s3_upload_timestamp=datetime.now(timezone.utc),
+        hubspot_listing_id=hubspot_listing_id,
+        file_source=FileSourceEnum.ECMK.value,
+        file_type=file_type,
+    )
+
+    with db_session() as session:
+        # TODO: we should do multiple files at once to reduce db trips
+        session.add(uploaded_file)
+        session.commit()