diff --git a/backend/ecmk_fetcher/processor.py b/backend/ecmk_fetcher/processor.py index dc52c342..0ca53c4c 100644 --- a/backend/ecmk_fetcher/processor.py +++ b/backend/ecmk_fetcher/processor.py @@ -29,7 +29,10 @@ from backend.ecmk_fetcher.reports import ( build_property_id, map_report_type_to_db_file_type, ) -from backend.ecmk_fetcher.sharepoint import upload_file_to_sharepoint +from backend.ecmk_fetcher.upload import ( + upload_file_to_s3_and_update_db, + upload_file_to_sharepoint, +) from utils.logger import setup_logger from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient from utils.sharepoint.domna_sites import DomnaSites @@ -56,7 +59,7 @@ def run_job() -> None: sharepoint_base_path: str = "/Projects/Southern Housing/SH-SURV-26-001/Assessments" - # s3_bucket: str = "retrofit-energy-assessments-dev" + s3_bucket: str = "retrofit-energy-assessments-dev" with sync_playwright() as p: browser: Browser = p.chromium.launch(headless=True) @@ -146,6 +149,15 @@ def run_job() -> None: logger.info( f"Successfully loaded {os.path.basename(file_path)} to sharepoint for {address}" ) + + # Upload to s3 and update db + upload_file_to_s3_and_update_db( + bucket=s3_bucket, + file_path=file_path, + hubspot_listing_id=hubspot_listing_id, + file_type=db_file_type, + ) + except Exception: raise finally: diff --git a/backend/ecmk_fetcher/sharepoint.py b/backend/ecmk_fetcher/sharepoint.py deleted file mode 100644 index 79db1294..00000000 --- a/backend/ecmk_fetcher/sharepoint.py +++ /dev/null @@ -1,20 +0,0 @@ -import os - -from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient - - -def upload_file_to_sharepoint( - client: DomnaSharepointClient, - file_path: str, - base_path: str, - subpath: str, -) -> None: - filename = os.path.basename(file_path) - - full_path = f"{base_path}/{subpath}/1. Retrofit Assessment/A. Assessment" - - client.upload_file( - file_path=file_path, - sharepoint_path=full_path, - file_name=filename, - ) diff --git a/backend/ecmk_fetcher/upload.py b/backend/ecmk_fetcher/upload.py new file mode 100644 index 00000000..00e2ec32 --- /dev/null +++ b/backend/ecmk_fetcher/upload.py @@ -0,0 +1,49 @@ +from datetime import datetime, timezone +import os + +from backend.app.db.connection import db_session +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) +from utils.s3 import upload_file_to_s3 +from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient + + +def upload_file_to_sharepoint( + client: DomnaSharepointClient, + file_path: str, + base_path: str, + subpath: str, +) -> None: + filename = os.path.basename(file_path) + + full_path = f"{base_path}/{subpath}/1. Retrofit Assessment/A. Assessment" + + client.upload_file( + file_path=file_path, + sharepoint_path=full_path, + file_name=filename, + ) + + +def upload_file_to_s3_and_update_db( + bucket: str, file_path: str, hubspot_listing_id: str, file_type: FileTypeEnum +) -> None: + key: str = f"documents/hubspot_listing_id/{hubspot_listing_id}" + upload_file_to_s3(file_path, bucket, key) + + uploaded_file = UploadedFile( + s3_file_bucket=bucket, + s3_file_key=key, + s3_upload_timestamp=datetime.now(timezone.utc), + hubspot_listing_id=hubspot_listing_id, + file_source=FileSourceEnum.ECMK.value, + file_type=file_type, + ) + + with db_session() as session: + # TODO: we should do multiple files at once to reduce db trips + session.add(uploaded_file) + session.commit()