From 97a0c9addd5436b9ecab944a2bc9ac06d3326d7a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 10:21:20 +0000 Subject: [PATCH] deem score can now be uploaded automatically --- etl/hubspot_to_invoice.py | 18 ++++++++++-- ...osmosis_monday_to_sharepoint_automation.py | 28 +++++++++---------- etl/scraper/scraper.py | 15 +++++++++- etl/surveyPrice/surveyPrice.py | 16 +++++++---- etl/utils/sharepoint/sharepoint.py | 23 +++++++++++++++ 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index f822105..156b2c7 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -14,10 +14,24 @@ sp = SurveyPrice() df = sp.calculate_all_price() + +verbose_file = "verbose_invoice_score.xlsx" +df.to_excel(verbose_file, index=False) +output_path = os.path.abspath(verbose_file) +sp.upload_to_sharepoint(output_path, verbose_file) + +lewis_view = "FOR_LEWIS.xlsx" +selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"] +minimal_df = df[selected_columns] +minimal_df.to_excel(lewis_view, index=False) +output_path = os.path.abspath(lewis_view) +sp.upload_to_sharepoint(output_path, lewis_view) + + + # TODO: -# Make W.C. Folder and upload to commercials in Sharepoint # Move deal in hub spot Surveyoed complete signed off, if normal -# Look into getting solar working. ACIS and Lewis # Write documentation for tech demos from Khalims demo +# Look into getting solar working. ACIS and Lewis # Figure out what to do if I see an address that isn't registered but surveyrod diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 30356b0..58c094d 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -11,7 +11,7 @@ import pandas as pd osmosis = SharePointScraper(SharePointInstaller.OSMOSIS) parent_folder = "Automated Example" -osmosis.create_file(parent_folder, "/JTK Test Folder") +osmosis.create_dir(parent_folder, "/JTK Test Folder") asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing") @@ -20,30 +20,30 @@ new_asset_list = [] parent_folder = "JTK Test Folder/Automated Example" # Create asset list and location for index, address in asset_list.iterrows(): - webUrl = osmosis.create_file(address['Name'], parent_folder) + webUrl = osmosis.create_dir(address['Name'], parent_folder) first_folder = "1. Retrofit Assessment" - osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") - osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir(first_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") second_folder = "2. RC Mid-Term Plan" - osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}") + osmosis.create_dir(second_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("SAP", parent_folder + f"/{address['Name']}/{second_folder}") third_folder = "3. Retrofit Design" - osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(third_folder, parent_folder + f"/{address['Name']}") fourth_folder = "4. Post EPC" - osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") + osmosis.create_dir(fourth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") fifth_folder = "5. Trustmark Lodgement" - osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir(fifth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") asset_data = { "Name": address['Name'], diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index f3ac352..3e65425 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -117,7 +117,7 @@ class SharePointScraper(): return True return False - def create_file(self, file_name, at_path="/"): + def create_dir(self, file_name, at_path="/"): sharepoint_client = SharePointClient( tenant_id=self.sharepoint_tenant_id, @@ -132,6 +132,19 @@ class SharePointScraper(): for folders in self.get_folders_in_path(at_path)['value']: if file_name.upper() in folders["name"].upper(): return folders["webUrl"] + + def upload_file(self, file_path, sharepoint_path, file_name): + sharepoint_client = SharePointClient( + tenant_id=self.sharepoint_tenant_id, + client_id=self.sharepoint_client_id, + client_secret=self.sharepoint_client_secret, + site_id=self.sharepoint_drive.value, + ) + def get_file_stream(file_path): + return open(file_path, 'rb') + + sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) + diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index b25ca15..596c006 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,11 +1,9 @@ -from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday from etl.hubSpotClient.hubspot import HubSpotClient, DealStage from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd -# This script will break if we change our current processses, obviously... - class SurveyPrice(): """ A class to work out all prices and uploads to sharepoint for review @@ -135,7 +133,8 @@ class SurveyPrice(): return self.all_hubspot_submissions def get_all_surveyed_data_from_sharepoint(self): - # TODO: A wrapper function + # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for + # DIfferent installers self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() return self.all_survey_info_from_sharepoint @@ -293,10 +292,15 @@ class SurveyPrice(): final_list.append(merged_row) return pd.concat(final_list, ignore_index=True) + + def upload_to_sharepoint(self, file_path_to_upload, file_name): + parent_folder = "Commercials/Rate Cards" + date = previous_monday() + self.sharepoint_client.create_dir(date, parent_folder) + sharepoint_path = parent_folder + "/" + date - - + self.sharepoint_client.upload_file(file_path_to_upload, sharepoint_path, file_name) # TODO diff --git a/etl/utils/sharepoint/sharepoint.py b/etl/utils/sharepoint/sharepoint.py index a366161..e175691 100644 --- a/etl/utils/sharepoint/sharepoint.py +++ b/etl/utils/sharepoint/sharepoint.py @@ -274,6 +274,29 @@ class SharePointClient: url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children" return 'POST', url, data + + def upload_file(self, file_name, file_stream, sharepoint_parent_id): + """ + Uploads a file to SharePoint using the Graph API. + PUT /drives/{drive-id}/root:/{path-to-file}:/content + + :param file_name: Name of the file to upload + :param sharepoint_path: Path within the SharePoint site (folder path) + :param file_stream: File content as a binary stream (e.g., BytesIO or open(file, 'rb')) + :return: Response JSON from the API + """ + url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content" + logger.debug(f"Uploading file to URL: {url}") + + response = requests.put(url, headers=self.headers, data=file_stream) + + if response.status_code in (200, 201): + logger.info(f"File '{file_name}' uploaded successfully.") + return response.json() + else: + retry = handle_error(response) + if retry == 'retry': + return self.upload_file(file_name, sharepoint_parent_id, file_stream) @staticmethod def download_sharepoint_file(download_url):