From 6bfab1d585098c64a80e45b9ab50fb1af646c5b5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 10:00:53 +0000 Subject: [PATCH 01/21] hubspot_to_deem --- .devcontainer/devcontainer.json | 4 +++- etl/hubspot_to_deem.py | 3 +++ etl/scraper/scraper.py | 1 + etl/surveyPrice/surveyPrice.py | 11 ++++++++--- 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 etl/hubspot_to_deem.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 823f974..1ccb286 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -20,7 +20,9 @@ "mechatroner.rainbow-csv", "ms-toolsai.datawrangler", "lindacong.vscode-book-reader", - "4ops.terraform" + "4ops.terraform", + "fabiospampinato.vscode-todo-plus", + "jgclark.vscode-todo-highlight" ] } } diff --git a/etl/hubspot_to_deem.py b/etl/hubspot_to_deem.py new file mode 100644 index 0000000..46a06fe --- /dev/null +++ b/etl/hubspot_to_deem.py @@ -0,0 +1,3 @@ +from etl.surveyPrice.surveyPrice import SurveyPrice + + diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index dadaf7b..21a76f5 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -27,6 +27,7 @@ class SharePointInstaller(Enum): BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4") OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") + WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4") class SharePointScraper(): """ diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 38a224f..ecd7d00 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,13 +1,16 @@ +from etl.scraper.scraper import SharePointScraper, SharePointInstaller -class surveyPrice(): +class SurveyPrice(): """ - A class to work out all prices and uploads to sharepoint + A class to work out all prices and uploads to sharepoint for review """ def __init__(self): + self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) pass def download_price_card(self): + self.sharepoint_client.get_folders_in_path("/Documents/Commercials/Rate Cards") pass def get_price_card(): @@ -17,9 +20,11 @@ class surveyPrice(): -# Step one +# TODO # Make a copy of the rate card and work out each price matrix # Make the price card downloadable as a single excelt sheet for viewing + + # For other things, do some TDD so it is a little robust # The script can run weekly, for development I can just get one data # Expected input, expect output etc From 5cc1e9e645cdf473db1d4792bbee9e5b52170fda Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 10:02:53 +0000 Subject: [PATCH 02/21] modification and set up for work --- etl/{hubspot_to_deem.py => hubspot_to_invoice.py} | 1 + 1 file changed, 1 insertion(+) rename etl/{hubspot_to_deem.py => hubspot_to_invoice.py} (98%) diff --git a/etl/hubspot_to_deem.py b/etl/hubspot_to_invoice.py similarity index 98% rename from etl/hubspot_to_deem.py rename to etl/hubspot_to_invoice.py index 46a06fe..ea4d762 100644 --- a/etl/hubspot_to_deem.py +++ b/etl/hubspot_to_invoice.py @@ -1,3 +1,4 @@ from etl.surveyPrice.surveyPrice import SurveyPrice + From c807917ad6e26e16e4ce645c0245aa23589cc787 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 11:07:54 +0000 Subject: [PATCH 03/21] save sheet names --- etl/hubspot_to_invoice.py | 12 ++++++++- etl/surveyPrice/surveyPrice.py | 47 ++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index ea4d762..cea0653 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -1,4 +1,14 @@ +import os + + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + from etl.surveyPrice.surveyPrice import SurveyPrice - +sp = SurveyPrice() +sp.get_price_card() \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index ecd7d00..a23088e 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,4 +1,5 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller +import pandas as pd class SurveyPrice(): @@ -7,24 +8,48 @@ class SurveyPrice(): """ def __init__(self): self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) - pass + self.master_rate_card_path = None + self.download_price_card() + self.sheet_names = [ + 'JJC - EMPTIES', + 'JJC - GENERAL EXTRACTIONS', + 'JJC - FORMALDEHYDE EXTRACTION', + 'JJC - SOLAR', + 'SCIS - GENERAL EXTRACTIONS', + 'SCIS - EMPTIES', + 'SCIS - SOLAR', + 'SGEC - EMPTIES', + 'SGEC - GENERAL EXTRACTIONS', + 'SGEC - FORMALDEHYDE EXTRACTION' + ] def download_price_card(self): - self.sharepoint_client.get_folders_in_path("/Documents/Commercials/Rate Cards") - pass + url = None + for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: + if files['name'] == "MASTER RATE CARD.xlsx": + url = files['@microsoft.graph.downloadUrl'] + break - def get_price_card(): - pass + if url: + content = self.sharepoint_client.get_file_content(url) + self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") + + return self.master_rate_card_path + + def get_price_card(self): + excel_file = pd.ExcelFile(self.master_rate_card_path) + return excel_file.sheet_names + + + def get_master_rate_card_path(self): + return self.master_rate_card_path() # TODO -# Make a copy of the rate card and work out each price matrix -# Make the price card downloadable as a single excelt sheet for viewing - - -# For other things, do some TDD so it is a little robust +# Get it working for JJC first, with an idea to make it more diverse +# Add some TDD to ensure JJC values are correct # The script can run weekly, for development I can just get one data -# Expected input, expect output etc +# Upload both the W.C. calculations and Rate Card to sharepoint \ No newline at end of file From 329e3425c420478abe5bed196b7265dfe5da39c4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 11:48:46 +0000 Subject: [PATCH 04/21] wrote some sanity tests --- etl/hubspot_to_invoice.py | 2 +- etl/surveyPrice/surveyPrice.py | 43 ++++++++++++++++++++++++++++++---- etl/tests/test_survey_price.py | 13 ++++++++++ 3 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 etl/tests/test_survey_price.py diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index cea0653..bc3d466 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -11,4 +11,4 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() -sp.get_price_card() \ No newline at end of file +sp.get_cavity_pricing_table("JJC_EMPTIES") \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index a23088e..81dbcdf 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -10,7 +10,7 @@ class SurveyPrice(): self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None self.download_price_card() - self.sheet_names = [ + self.required_sheets = [ 'JJC - EMPTIES', 'JJC - GENERAL EXTRACTIONS', 'JJC - FORMALDEHYDE EXTRACTION', @@ -36,14 +36,49 @@ class SurveyPrice(): return self.master_rate_card_path - def get_price_card(self): + def get_cavity_pricing_table(self, sheet_name): excel_file = pd.ExcelFile(self.master_rate_card_path) - return excel_file.sheet_names + available_sheets = excel_file.sheet_names + missing_sheets = [sheet for sheet in self.required_sheets if sheet not in available_sheets] + + if missing_sheets: + raise ValueError(f"Missing sheets: {missing_sheets}") + + pricing_table = self.get_price_matrix(sheet_name) + + return pricing_table def get_master_rate_card_path(self): - return self.master_rate_card_path() + return self.master_rate_card_path + def get_price_matrix(self, sheet_name): + df = pd.read_excel(self.master_rate_card_path, sheet_name) + + columns_to_check = { + "no extractors or ventilation required": {"Trickle Vent": 0, "wetrooms": 0}, + "Trickle Vents ONLY": {"Trickle Vent": 1, "wetrooms": 0}, + "1 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 1}, + "2 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 2}, + "3 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 3}, + 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "wetrooms": 1}, + 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "wetrooms": 2}, + 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "wetrooms": 3}, + } + + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "funding": row["Funding"], + "floor_area_group": row["Total Floor Area"][:-1], + **variables, + "price": row[key][1:] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table diff --git a/etl/tests/test_survey_price.py b/etl/tests/test_survey_price.py new file mode 100644 index 0000000..e6773d5 --- /dev/null +++ b/etl/tests/test_survey_price.py @@ -0,0 +1,13 @@ +import os +# WarmFront Sharepoint KEYS +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" + +from etl.surveyPrice.surveyPrice import SurveyPrice + +def test_get_price_matrix_jjc_empties(): + sp = SurveyPrice + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - EMPTIES") + + From 738c5f915e8b762eedbe2b49e5befcf529d7701a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 12:27:05 +0000 Subject: [PATCH 05/21] sanity check to ensure price matrix is correct format --- etl/surveyPrice/surveyPrice.py | 22 +++++++++++----------- etl/tests/test_survey_price.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 81dbcdf..5db58ed 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -56,14 +56,14 @@ class SurveyPrice(): df = pd.read_excel(self.master_rate_card_path, sheet_name) columns_to_check = { - "no extractors or ventilation required": {"Trickle Vent": 0, "wetrooms": 0}, - "Trickle Vents ONLY": {"Trickle Vent": 1, "wetrooms": 0}, - "1 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 1}, - "2 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 2}, - "3 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 3}, - 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "wetrooms": 1}, - 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "wetrooms": 2}, - 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "wetrooms": 3}, + "no extractors or ventilation required": {"Trickle Vent": 0, "Wetrooms": 0}, + "Trickle Vents ONLY": {"Trickle Vent": 1, "Wetrooms": 0}, + "1 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 1}, + "2 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 2}, + "3 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 3}, + 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 1}, + 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 2}, + 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 3}, } pricing_table = [] @@ -71,10 +71,10 @@ class SurveyPrice(): for key, variables in columns_to_check.items(): pricing_table.append( { - "funding": row["Funding"], - "floor_area_group": row["Total Floor Area"][:-1], + "WORK TYPE": row["WORK TYPE"], + "Floor Area Group": row["Total Floor Area"][:-1], **variables, - "price": row[key][1:] if row[key] != "Not viable" else None, + "PRICE": row[key] if row[key] != "Not viable" else None, } ) pricing_table = pd.DataFrame(pricing_table) diff --git a/etl/tests/test_survey_price.py b/etl/tests/test_survey_price.py index e6773d5..122ac48 100644 --- a/etl/tests/test_survey_price.py +++ b/etl/tests/test_survey_price.py @@ -5,9 +5,32 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" from etl.surveyPrice.surveyPrice import SurveyPrice +import pytest -def test_get_price_matrix_jjc_empties(): - sp = SurveyPrice + +@pytest.fixture(scope="module") +def sp(): + return SurveyPrice() + + +def cavity_price_dataframe_sanity_check(df): + assert df.shape == (160, 5) + assert df.columns.tolist() == ['WORK TYPE', 'Floor Area Group', 'Trickle Vent', 'Wetrooms', 'PRICE'] + +def test_get_price_matrix_jjc_empties(sp): jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - EMPTIES") - + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + +def test_get_price_matrix_jjc_general_extraction(sp): + sp = SurveyPrice() + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - GENERAL EXTRACTIONS") + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + +def test_get_price_matrix_jjc_foam(sp): + sp = SurveyPrice() + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - FORMALDEHYDE EXTRACTION") + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + + + From b4f7e29f753ef9a7c05ab753173524a229446b80 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 12:30:26 +0000 Subject: [PATCH 06/21] added test to ensure price matrix doesn't get effected --- etl/surveyPrice/surveyPrice.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 5db58ed..0b8bef3 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -80,9 +80,6 @@ class SurveyPrice(): pricing_table = pd.DataFrame(pricing_table) return pricing_table - - - # TODO # Get it working for JJC first, with an idea to make it more diverse # Add some TDD to ensure JJC values are correct From dbf663ef6884de5b8a495d67a08bd84213c028ab Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 12:36:02 +0000 Subject: [PATCH 07/21] added a todo list for hubspot --- etl/surveyPrice/surveyPrice.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 0b8bef3..3ceefe5 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -79,6 +79,11 @@ class SurveyPrice(): ) pricing_table = pd.DataFrame(pricing_table) return pricing_table + + def get_all_surveys_from_hubspot(): + # Make a script that gets all data from hubspot and puts in a dataframe + pass + # TODO # Get it working for JJC first, with an idea to make it more diverse From 2fccf6b8a338832e3d1d8bff334e9be7d40ca953 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 12:55:11 +0000 Subject: [PATCH 08/21] wrote what I'll do next --- etl/surveyPrice/surveyPrice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 3ceefe5..2c86fba 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -81,7 +81,7 @@ class SurveyPrice(): return pricing_table def get_all_surveys_from_hubspot(): - # Make a script that gets all data from hubspot and puts in a dataframe + # TODO Get all hubspot data and make a dataframe pass From aa232621b8a8db1fdce19f5ffdafcc98265f2db6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 14:01:43 +0000 Subject: [PATCH 09/21] check all hubspot submissions --- etl/surveyPrice/surveyPrice.py | 25 ++++++++++++++++++++++--- etl/tests/test_survey_price.py | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 2c86fba..e27680f 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,4 +1,5 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage import pandas as pd @@ -9,6 +10,7 @@ class SurveyPrice(): def __init__(self): self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None + self.all_hubspot_submissions = None self.download_price_card() self.required_sheets = [ 'JJC - EMPTIES', @@ -80,9 +82,26 @@ class SurveyPrice(): pricing_table = pd.DataFrame(pricing_table) return pricing_table - def get_all_surveys_from_hubspot(): - # TODO Get all hubspot data and make a dataframe - pass + def get_all_surveys_from_hubspot(self): + hubSpotClient = HubSpotClient() + deals = hubSpotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + all_deals = [] + + for deal in deals: + all_deals.append({ + "HUBSPOT_WORK_TYPE": deal.work_type, + "HUBSPOT_DEAL_ADDRESS": deal.deal_name, + "HUBSPOT_TRICKLE_VENT":1 if deal.needs_trickle_ventilation else 0, + "HUBSPOT_WALL_INSULATION": deal.existing_wall_insulation, + "HUBSPOT_POST_INSTALL_SAP_SCORE": deal.post_sap_score, + "HUBSPOT_INSTALLER": deal.installer + }) + + self.all_hubspot_submissions = pd.DataFrame(all_deals) + return self.all_hubspot_submissions + + # TODO diff --git a/etl/tests/test_survey_price.py b/etl/tests/test_survey_price.py index 122ac48..1dc7cad 100644 --- a/etl/tests/test_survey_price.py +++ b/etl/tests/test_survey_price.py @@ -34,3 +34,4 @@ def test_get_price_matrix_jjc_foam(sp): + From eca7779b447e53386fd4502ff161a5d7b10e4561 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 14:06:31 +0000 Subject: [PATCH 10/21] sharepoint warmfront to get csv but installer will be different --- etl/hubspot_to_invoice.py | 4 +++- etl/surveyPrice/surveyPrice.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index bc3d466..4b6fc4c 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -11,4 +11,6 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() -sp.get_cavity_pricing_table("JJC_EMPTIES") \ No newline at end of file +sp.get_cavity_pricing_table("JJC - EMPTIES") +df = sp.get_all_surveys_from_hubspot() +df \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index e27680f..10fb18f 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -8,7 +8,7 @@ class SurveyPrice(): A class to work out all prices and uploads to sharepoint for review """ def __init__(self): - self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) + self.warmfront_sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None self.all_hubspot_submissions = None self.download_price_card() @@ -27,14 +27,14 @@ class SurveyPrice(): def download_price_card(self): url = None - for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: + for files in self.warmfront_sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: if files['name'] == "MASTER RATE CARD.xlsx": url = files['@microsoft.graph.downloadUrl'] break if url: - content = self.sharepoint_client.get_file_content(url) - self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") + content = self.warmfront_sharepoint_client.get_file_content(url) + self.master_rate_card_path = self.warmfront_sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") return self.master_rate_card_path From ae4ddf49ebb3e7fcc5f30640158f8a3c6a2d8262 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 15:10:39 +0000 Subject: [PATCH 11/21] save current work --- etl/hubSpotClient/hubspot.py | 4 ++- etl/hubspot_to_invoice.py | 3 +- etl/scraper/scraper.py | 7 +++-- etl/surveyPrice/surveyPrice.py | 46 +++++++++++++++++++++++++++- etl/surveyedData/surveryedData.py | 51 +++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 6 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 353ecd2..a4a6de5 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -67,4 +67,6 @@ class HubSpotClient(): print(f"Pipeline: {pipeline.label}") for stage in pipeline.stages: print(f" - Label: {stage.label}") - print(f" ID: {stage.id}") # + print(f" ID: {stage.id}") + + diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 4b6fc4c..9bee806 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -12,5 +12,6 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() sp.get_cavity_pricing_table("JJC - EMPTIES") + df = sp.get_all_surveys_from_hubspot() -df \ No newline at end of file +df = sp.get_all_surveyed_data_from_sharepoint() \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 21a76f5..f3ac352 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,8 +14,8 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - # return f"W.C. 31.03.2025" - return f"W.C. {last_monday.strftime('%d.%m.%Y')}" + return f"W.C. 31.03.2025" + # return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) @@ -309,4 +309,5 @@ class SharePointScraper(): temp_file.write(content.getvalue()) self.logger.info(f"Temporary file created at: {path}") - return path \ No newline at end of file + return path + \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 10fb18f..7e437f0 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,5 +1,6 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd @@ -100,7 +101,50 @@ class SurveyPrice(): self.all_hubspot_submissions = pd.DataFrame(all_deals) return self.all_hubspot_submissions - + + def get_all_surveyed_data_from_sharepoint(self): + self.sharepoint_data_for_jjc() + + def sharepoint_data_for_jjc(self): + jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) + file_paths = jjc_sp.download_file_for_each_address() + jjc_surveys = [] + + for eachAddress in file_paths: + for address, files in eachAddress.items(): + jjc_surveys.append(surveyedDataProcessor(address, files)) + + all_survey_info = [] + for surveyInfo in jjc_surveys: + info = { + "SHAREPOINT INSTALLER": "J & J Crumps", + "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, + "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, + "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", + "SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND", + "SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND", + "SHAREPOINT INSULATION MATERIAL": None, + } + + if surveyInfo.pre_site_note: + floor_banding, total_floor_area = surveyInfo.work_out_total_floor_area() + info.update({ + "SHAREPOINT TOTAL_FLOOR_AREA": total_floor_area, + "SHAREPOINT FLOOR_AREA_BANDING": floor_banding, + "SHAREPOINT PRE_INSTALL_SAP_SCORE": surveyInfo.get_current_sap_score(), + }) + + if surveyInfo.csr: + info.update({ + "SHAREPOINT INSULATION MATERIAL": surveyInfo.get_insulation_info(), + }) + + + return jjc_surveys + + + + diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index bc3054a..d83f186 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,5 +1,6 @@ from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.pdfReader.reportType import ReportType +import math class surveyedDataProcessor(): def __init__(self, address, files): @@ -17,5 +18,55 @@ class surveyedDataProcessor(): if pdf: if pdf.type == ReportType.QUIDOS_PRESITE_NOTE: self.pre_site_note = pdf.get_reader() + self.address = self.pre_site_note.survey_information.address elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT: self.csr = pdf.get_reader() + + def get_insulation_info(self): + if self.csr: + if self.csr.insulation_info: + insultation = self.csr.insulation_info.type.upper() + return insultation + return None + + + def work_out_total_floor_area(self): + + total = 0 + def add_all_floors(floor_list): + total = 0 + for floor in floor_list: + total += floor.floor_area_m2 + + return total + + main = True if self.pre_site_note.property_description.no_of_main_property > 0 else False + ext1 = True if self.pre_site_note.property_description.no_of_extension_1 > 0 else False + ext2 = True if self.pre_site_note.property_description.no_of_extension_2 > 0 else False + ext3 = True if self.pre_site_note.property_description.no_of_extension_3 > 0 else False + ext4 = True if self.pre_site_note.property_description.no_of_extension_4 > 0 else False + + total += add_all_floors(self.pre_site_note.property_description.main_property.dimensions) if main is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex1_property.dimensions) if ext1 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex2_property.dimensions) if ext2 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0 + + + floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total) + if 0 <= floor_area <= 72: + return '0-72m', floor_area + elif 72 < floor_area <= 97: + return '73-97m', floor_area + elif 97 < floor_area <= 199: + return '98-199m', floor_area + elif 199 <= floor_area: + return 'over 200m', floor_area + else: + raise NotImplementedError(f"unknown floor area {floor_area} {self.pre_site_note.summary_information.address}") + + def get_current_sap_score(self): + return self.pre_site_note.survey_information.current_sap.split(" ")[1] + + + From cc07706fc124899d06d66bb220a829bfcc22597e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 15:47:35 +0000 Subject: [PATCH 12/21] merge pdf works now --- etl/hubspot_to_invoice.py | 6 ++++-- etl/surveyPrice/surveyPrice.py | 37 ++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 9bee806..3c3e2fe 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -13,5 +13,7 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() sp.get_cavity_pricing_table("JJC - EMPTIES") -df = sp.get_all_surveys_from_hubspot() -df = sp.get_all_surveyed_data_from_sharepoint() \ No newline at end of file +hubspot_df = sp.get_all_surveys_from_hubspot() +sharepoint_df = sp.get_all_surveyed_data_from_sharepoint() + +df = sp.merge_hub_spot_and_survey_information() \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 7e437f0..dd8c45e 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -12,6 +12,7 @@ class SurveyPrice(): self.warmfront_sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None self.all_hubspot_submissions = None + self.all_survey_info_from_sharepoint = None self.download_price_card() self.required_sheets = [ 'JJC - EMPTIES', @@ -103,7 +104,8 @@ class SurveyPrice(): return self.all_hubspot_submissions def get_all_surveyed_data_from_sharepoint(self): - self.sharepoint_data_for_jjc() + self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() + return self.all_survey_info_from_sharepoint def sharepoint_data_for_jjc(self): jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) @@ -117,13 +119,14 @@ class SurveyPrice(): all_survey_info = [] for surveyInfo in jjc_surveys: info = { - "SHAREPOINT INSTALLER": "J & J Crumps", + "SHAREPOINT INSTALLER": "J & J CRUMP", "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", "SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND", "SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND", "SHAREPOINT INSULATION MATERIAL": None, + "SHAREPOINT ADDRESS": address } if surveyInfo.pre_site_note: @@ -139,10 +142,36 @@ class SurveyPrice(): "SHAREPOINT INSULATION MATERIAL": surveyInfo.get_insulation_info(), }) + all_survey_info.append(info) - return jjc_surveys + return pd.DataFrame(all_survey_info) + + def merge_hub_spot_and_survey_information(self): + if self.all_survey_info_from_sharepoint is None: + raise RuntimeError("No survey information found from Sharepoint") + if self.all_hubspot_submissions is None: + raise RuntimeError("No information found from Hubspot") + + self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply( + lambda x: x.lower().replace(',', '').strip() + ) + + self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply( + lambda x: x.lower().replace(',', '').strip() + ) + + merged_df = pd.merge( + self.all_survey_info_from_sharepoint, + self.all_hubspot_submissions, + left_on=['clean_address', 'SHAREPOINT INSTALLER'], + right_on=['clean_address', 'HUBSPOT_INSTALLER'], + how='inner' + ) + + merged_df.drop(columns=['clean_address'], inplace=True) + + return merged_df - From 94f7ca3ab657b063a3f4f91789912e3a1b653013 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Apr 2025 15:56:42 +0000 Subject: [PATCH 13/21] some nice formatting to ensure there is only one installer --- etl/surveyPrice/surveyPrice.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index dd8c45e..6c46b07 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -152,6 +152,7 @@ class SurveyPrice(): if self.all_hubspot_submissions is None: raise RuntimeError("No information found from Hubspot") + # Standardise address self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply( lambda x: x.lower().replace(',', '').strip() ) @@ -160,11 +161,20 @@ class SurveyPrice(): lambda x: x.lower().replace(',', '').strip() ) + # re-name to installer + self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( + columns={'SHAREPOINT INSTALLER': 'INSTALLER'} + ) + + self.all_hubspot_submissions = self.all_hubspot_submissions.rename( + columns={'HUBSPOT_INSTALLER': 'INSTALLER'} + ) + merged_df = pd.merge( self.all_survey_info_from_sharepoint, self.all_hubspot_submissions, - left_on=['clean_address', 'SHAREPOINT INSTALLER'], - right_on=['clean_address', 'HUBSPOT_INSTALLER'], + left_on=['clean_address', 'INSTALLER'], + right_on=['clean_address', 'INSTALLER'], how='inner' ) From e1b61bca17d0330535245deead590df4799f43eb Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Apr 2025 14:36:34 +0000 Subject: [PATCH 14/21] deem score calculation works for all cavities --- etl/hubSpotClient/hubspot.py | 6 +- etl/hubspot_to_invoice.py | 9 +- etl/surveyPrice/surveyPrice.py | 189 ++++++++++++++++++++++++------ etl/surveyedData/surveryedData.py | 79 ++++++++++++- etl/transform/types.py | 6 + 5 files changed, 244 insertions(+), 45 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index a4a6de5..64aea46 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -53,9 +53,9 @@ class HubSpotClient(): deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], work_type=deal.properties["work_type"], - needs_trickle_ventilation=True if deal.properties["property_needs_trickle_vents"].upper() == "YES" else False, + needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False, post_sap_score=int(deal.properties["domna_survey_post_sap"]), - existing_wall_insulation=deal.properties["existing_wall_insulation"], + existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), installer=deal.properties["installer"], )) @@ -69,4 +69,4 @@ class HubSpotClient(): print(f" - Label: {stage.label}") print(f" ID: {stage.id}") - + diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 3c3e2fe..ae9d3b3 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -11,9 +11,10 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() -sp.get_cavity_pricing_table("JJC - EMPTIES") -hubspot_df = sp.get_all_surveys_from_hubspot() -sharepoint_df = sp.get_all_surveyed_data_from_sharepoint() +df = sp.calculate_all_price() -df = sp.merge_hub_spot_and_survey_information() \ No newline at end of file +# Surveyoed complete signed off, if normal + +# Make W.C. Folder and upload to commercials in Sharepoint +# Move deal in hub spot \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 6c46b07..b25ca15 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -4,16 +4,19 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd +# This script will break if we change our current processses, obviously... + class SurveyPrice(): """ A class to work out all prices and uploads to sharepoint for review """ def __init__(self): - self.warmfront_sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) + self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None self.all_hubspot_submissions = None self.all_survey_info_from_sharepoint = None self.download_price_card() + self.required_sheets = [ 'JJC - EMPTIES', 'JJC - GENERAL EXTRACTIONS', @@ -27,18 +30,41 @@ class SurveyPrice(): 'SGEC - FORMALDEHYDE EXTRACTION' ] + self.installer = { + "J & J CRUMP": "JJC", + "SCIS": "SCIS", + "SGEC": "SGEC", + "BAXTER KELLY": "BAXTER KELLY", + } + + self.domna_job_to_price_sheet_convertor = { + "JJC - SOLAR": "JJC - SOLAR", + "JJC - EMPTY CAVITY": "JJC - EMPTIES", + "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS", + "SCIS - SOLAR": "SCIS - SOLAR", + "SCIS - EMPTY CAVITY": "SCIS - EMPTIES", + "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION", + "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS", + "SGEC - EMPTY CAVITY": "SGEC - EMPTIES", + "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS", + } + def download_price_card(self): url = None - for files in self.warmfront_sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: + # TODO: Some sanity checks to ensure rate cards title stays consistent + for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: if files['name'] == "MASTER RATE CARD.xlsx": url = files['@microsoft.graph.downloadUrl'] break if url: - content = self.warmfront_sharepoint_client.get_file_content(url) - self.master_rate_card_path = self.warmfront_sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") + content = self.sharepoint_client.get_file_content(url) + self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") return self.master_rate_card_path + def get_cavity_pricing_table(self, sheet_name): excel_file = pd.ExcelFile(self.master_rate_card_path) @@ -58,31 +84,34 @@ class SurveyPrice(): def get_price_matrix(self, sheet_name): df = pd.read_excel(self.master_rate_card_path, sheet_name) - - columns_to_check = { - "no extractors or ventilation required": {"Trickle Vent": 0, "Wetrooms": 0}, - "Trickle Vents ONLY": {"Trickle Vent": 1, "Wetrooms": 0}, - "1 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 1}, - "2 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 2}, - "3 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 3}, - 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 1}, - 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 2}, - 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 3}, - } - pricing_table = [] - for _, row in df.iterrows(): - for key, variables in columns_to_check.items(): - pricing_table.append( - { - "WORK TYPE": row["WORK TYPE"], - "Floor Area Group": row["Total Floor Area"][:-1], - **variables, - "PRICE": row[key] if row[key] != "Not viable" else None, - } - ) - pricing_table = pd.DataFrame(pricing_table) - return pricing_table + if "SOLAR" in sheet_name.upper(): + return df + else: + columns_to_check = { + "no extractors or ventilation required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 0}, + "Trickle Vents ONLY": {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 0}, + "1 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 1}, + "2 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 2}, + "3 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 3}, + 'Trickle Vents + 1 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 1}, + 'Trickle Vents + 2 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 2}, + 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, + } + + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "WORK TYPE": row["WORK TYPE"], + "FLOOR_AREA_BANDING": row["Total Floor Area"][:-1], + **variables, + "PRICE": row[key] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table def get_all_surveys_from_hubspot(self): hubSpotClient = HubSpotClient() @@ -92,18 +121,21 @@ class SurveyPrice(): for deal in deals: all_deals.append({ + "HUBSPOT_DEAL_ID": deal.deal_id, "HUBSPOT_WORK_TYPE": deal.work_type, "HUBSPOT_DEAL_ADDRESS": deal.deal_name, "HUBSPOT_TRICKLE_VENT":1 if deal.needs_trickle_ventilation else 0, "HUBSPOT_WALL_INSULATION": deal.existing_wall_insulation, "HUBSPOT_POST_INSTALL_SAP_SCORE": deal.post_sap_score, - "HUBSPOT_INSTALLER": deal.installer + "HUBSPOT_INSTALLER": deal.installer, + "HUBSPOT_WETROOMS": deal.no_of_wet_rooms, }) self.all_hubspot_submissions = pd.DataFrame(all_deals) return self.all_hubspot_submissions def get_all_surveyed_data_from_sharepoint(self): + # TODO: A wrapper function self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() return self.all_survey_info_from_sharepoint @@ -118,6 +150,10 @@ class SurveyPrice(): all_survey_info = [] for surveyInfo in jjc_surveys: + cavity_wall_as_built = False + csr = False + foam_insulation = False + info = { "SHAREPOINT INSTALLER": "J & J CRUMP", "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, @@ -131,16 +167,47 @@ class SurveyPrice(): if surveyInfo.pre_site_note: floor_banding, total_floor_area = surveyInfo.work_out_total_floor_area() + pre_sap_score = surveyInfo.get_current_sap_score() + pre_sap_score_banding = surveyedDataProcessor.get_band(pre_sap_score) info.update({ "SHAREPOINT TOTAL_FLOOR_AREA": total_floor_area, "SHAREPOINT FLOOR_AREA_BANDING": floor_banding, - "SHAREPOINT PRE_INSTALL_SAP_SCORE": surveyInfo.get_current_sap_score(), + "SHAREPOINT PRE_INSTALL_SAP_SCORE": pre_sap_score, + "SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING": pre_sap_score_banding, }) + + if surveyInfo.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \ + and surveyInfo.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": + cavity_wall_as_built = True if surveyInfo.csr: + csr = True + insulation = surveyInfo.get_insulation_info() info.update({ - "SHAREPOINT INSULATION MATERIAL": surveyInfo.get_insulation_info(), + "SHAREPOINT INSULATION MATERIAL": insulation, }) + if "FOAM" in insulation.upper(): + foam_insulation = True + + if cavity_wall_as_built: + if csr: + if foam_insulation: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FOAM FILLED CAVITY", + }) + else: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FILLED CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "EMPTY CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "SOLAR" + }) + all_survey_info.append(info) @@ -163,11 +230,18 @@ class SurveyPrice(): # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( - columns={'SHAREPOINT INSTALLER': 'INSTALLER'} + columns={ + 'SHAREPOINT INSTALLER': 'INSTALLER', + 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', + } ) self.all_hubspot_submissions = self.all_hubspot_submissions.rename( - columns={'HUBSPOT_INSTALLER': 'INSTALLER'} + columns={ + 'HUBSPOT_INSTALLER': 'INSTALLER', + 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', + 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', + } ) merged_df = pd.merge( @@ -180,9 +254,47 @@ class SurveyPrice(): merged_df.drop(columns=['clean_address'], inplace=True) - return merged_df + def compute_energy_grant(row): + pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1] + post_band_letter = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"])[-1] + return surveyedDataProcessor.gbis_or_eco4_scheme(pre_band_letter, post_band_letter) + + def compute_banding_for_post_sap(row): + post_sap_banding = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"]) + return post_sap_banding + + def work_type(row): + if row["ENERGY_GRANT"] == "GBIS": + return row["ENERGY GRANT"] + else: + return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}" + + + # Add missing variables + merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1) + merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1) + merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1) + + return merged_df + + def calculate_all_price(self): + self.get_all_surveys_from_hubspot() + self.get_all_surveyed_data_from_sharepoint() + submission_data = self.merge_hub_spot_and_survey_information() + final_list = [] + for _, row in submission_data.iterrows(): + if "SOLAR" in row["DOMNA JOB TYPE"].upper(): + raise NotImplementedError("Please implement solar pricing") + else: + # Cavity wall + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') + final_list.append(merged_row) + + return pd.concat(final_list, ignore_index=True) + - @@ -191,4 +303,9 @@ class SurveyPrice(): # Get it working for JJC first, with an idea to make it more diverse # Add some TDD to ensure JJC values are correct # The script can run weekly, for development I can just get one data -# Upload both the W.C. calculations and Rate Card to sharepoint \ No newline at end of file +# Upload both the W.C. calculations and Rate Card to sharepoint +# Due considerations -> Piece of UI, as osmosis, dashboard upload and pushes site notes database and condition report to database. Link the two, make a due consideration report, due consideration report, propery_id () +# Property Tbale + +# Add hubspot deal id into pandas +# Deem score and move deal id to 'Surveyed Completed Signed off" diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index d83f186..1b5248e 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -28,6 +28,80 @@ class surveyedDataProcessor(): insultation = self.csr.insulation_info.type.upper() return insultation return None + + @staticmethod + def get_band(sap_score_number): + bands = [ + ("HIGH A", 96, float("inf")), + ("LOW A", 92, 96), + ("HIGH B", 86, 92), + ("LOW B", 81, 86), + ("HIGH C", 74.5, 81), + ("LOW C", 69, 74.5), + ("HIGH D", 61.5, 69), + ("LOW D", 55, 61.5), + ("HIGH E", 46.5, 55), + ("LOW E", 39, 46.5), + ("HIGH F", 29.5, 39), + ("LOW F", 21, 29.5), + ("HIGH G", 10.5, 21), + ("LOW G", 1, 10.5), + ] + + for band, lower, upper in bands: + if lower <= sap_score_number < upper: + return band + + return None + + @staticmethod + def gbis_or_eco4_scheme(presap_letter, postsap_letter): + """ + *ECO4 (minimum movement)* + D to C + E to C + F to D + G to D + + G -> ABCD + F -> ABCD + E -> ABC + D -> ABC + + *GBIS - Cavity Wall Insulation ONLY* + + D-D + E-D + E-E + F-E + F-F + G-E + G-F + G-G + """ + eco4 = { + "G": ['A', 'B', 'C', 'D'], + "F": ['A', 'B', 'C', 'D'], + "E": ['A', 'B', 'C'], + "D": ['A', 'B', 'C'], + } + + gbis ={ + 'D': ['D'], + 'E': ['E', 'D'], + 'F': ['E', 'F'], + 'G': ['E', 'F', 'G'], + } + + if presap_letter.upper() in eco4: + if postsap_letter.upper() in eco4[presap_letter.upper()]: + return "ECO4" + if presap_letter.upper() in gbis: + if postsap_letter.upper() in gbis[presap_letter.upper()]: + return "GBIS" + + return None + def work_out_total_floor_area(self): @@ -66,7 +140,8 @@ class surveyedDataProcessor(): raise NotImplementedError(f"unknown floor area {floor_area} {self.pre_site_note.summary_information.address}") def get_current_sap_score(self): - return self.pre_site_note.survey_information.current_sap.split(" ")[1] - + score_list = self.pre_site_note.survey_information.current_sap.split(" ") + score = int(score_list[1]) + return score diff --git a/etl/transform/types.py b/etl/transform/types.py index 3342bee..7e25682 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -203,5 +203,11 @@ class PropertyDescription(BaseModel): mainHeating2: Optional[Heating] secondaryHeatingType: Optional[HeatingType] +# class PropertyReport(): + # TODO: Property description + # TODO: Due consideration foregin key + # TODO: Which company did it (Osmosis, Warmfront etc) + # TODO: Links to more foreign keys per report etc + class Insulation(BaseModel): type: str \ No newline at end of file From a9cef610c35fc16641f6cc7616f6579e6a6aa906 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Apr 2025 14:52:55 +0000 Subject: [PATCH 15/21] deem score calculation works for all cavities --- etl/hubspot_to_invoice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index ae9d3b3..92a87b3 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -17,4 +17,5 @@ df = sp.calculate_all_price() # Surveyoed complete signed off, if normal # Make W.C. Folder and upload to commercials in Sharepoint -# Move deal in hub spot \ No newline at end of file +# Move deal in hub spot +# Look into getting solar working. ACIS and Lewis \ No newline at end of file From 6112a0e180fccefb2e2c83a74e366822c69877c0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Apr 2025 14:53:11 +0000 Subject: [PATCH 16/21] deem score calculation works for all cavities --- etl/hubspot_to_invoice.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 92a87b3..d5566b9 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -14,8 +14,7 @@ sp = SurveyPrice() df = sp.calculate_all_price() -# Surveyoed complete signed off, if normal - +# TODO: # Make W.C. Folder and upload to commercials in Sharepoint -# Move deal in hub spot +# Move deal in hub spot Surveyoed complete signed off, if normal # Look into getting solar working. ACIS and Lewis \ No newline at end of file From 913d8d6b73960471e8e2f325d8dc0297ad9665cd Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 09:11:58 +0000 Subject: [PATCH 17/21] save to do list --- etl/hubspot_to_invoice.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index d5566b9..f822105 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -17,4 +17,7 @@ df = sp.calculate_all_price() # TODO: # Make W.C. Folder and upload to commercials in Sharepoint # Move deal in hub spot Surveyoed complete signed off, if normal -# Look into getting solar working. ACIS and Lewis \ No newline at end of file +# Look into getting solar working. ACIS and Lewis +# Write documentation for tech demos from Khalims demo +# Figure out what to do if I see an address that isn't registered but surveyrod + From 97a0c9addd5436b9ecab944a2bc9ac06d3326d7a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 10:21:20 +0000 Subject: [PATCH 18/21] deem score can now be uploaded automatically --- etl/hubspot_to_invoice.py | 18 ++++++++++-- ...osmosis_monday_to_sharepoint_automation.py | 28 +++++++++---------- etl/scraper/scraper.py | 15 +++++++++- etl/surveyPrice/surveyPrice.py | 16 +++++++---- etl/utils/sharepoint/sharepoint.py | 23 +++++++++++++++ 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index f822105..156b2c7 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -14,10 +14,24 @@ sp = SurveyPrice() df = sp.calculate_all_price() + +verbose_file = "verbose_invoice_score.xlsx" +df.to_excel(verbose_file, index=False) +output_path = os.path.abspath(verbose_file) +sp.upload_to_sharepoint(output_path, verbose_file) + +lewis_view = "FOR_LEWIS.xlsx" +selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"] +minimal_df = df[selected_columns] +minimal_df.to_excel(lewis_view, index=False) +output_path = os.path.abspath(lewis_view) +sp.upload_to_sharepoint(output_path, lewis_view) + + + # TODO: -# Make W.C. Folder and upload to commercials in Sharepoint # Move deal in hub spot Surveyoed complete signed off, if normal -# Look into getting solar working. ACIS and Lewis # Write documentation for tech demos from Khalims demo +# Look into getting solar working. ACIS and Lewis # Figure out what to do if I see an address that isn't registered but surveyrod diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 30356b0..58c094d 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -11,7 +11,7 @@ import pandas as pd osmosis = SharePointScraper(SharePointInstaller.OSMOSIS) parent_folder = "Automated Example" -osmosis.create_file(parent_folder, "/JTK Test Folder") +osmosis.create_dir(parent_folder, "/JTK Test Folder") asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing") @@ -20,30 +20,30 @@ new_asset_list = [] parent_folder = "JTK Test Folder/Automated Example" # Create asset list and location for index, address in asset_list.iterrows(): - webUrl = osmosis.create_file(address['Name'], parent_folder) + webUrl = osmosis.create_dir(address['Name'], parent_folder) first_folder = "1. Retrofit Assessment" - osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") - osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir(first_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") second_folder = "2. RC Mid-Term Plan" - osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}") + osmosis.create_dir(second_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("SAP", parent_folder + f"/{address['Name']}/{second_folder}") third_folder = "3. Retrofit Design" - osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(third_folder, parent_folder + f"/{address['Name']}") fourth_folder = "4. Post EPC" - osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") + osmosis.create_dir(fourth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") fifth_folder = "5. Trustmark Lodgement" - osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir(fifth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") asset_data = { "Name": address['Name'], diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index f3ac352..3e65425 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -117,7 +117,7 @@ class SharePointScraper(): return True return False - def create_file(self, file_name, at_path="/"): + def create_dir(self, file_name, at_path="/"): sharepoint_client = SharePointClient( tenant_id=self.sharepoint_tenant_id, @@ -132,6 +132,19 @@ class SharePointScraper(): for folders in self.get_folders_in_path(at_path)['value']: if file_name.upper() in folders["name"].upper(): return folders["webUrl"] + + def upload_file(self, file_path, sharepoint_path, file_name): + sharepoint_client = SharePointClient( + tenant_id=self.sharepoint_tenant_id, + client_id=self.sharepoint_client_id, + client_secret=self.sharepoint_client_secret, + site_id=self.sharepoint_drive.value, + ) + def get_file_stream(file_path): + return open(file_path, 'rb') + + sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) + diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index b25ca15..596c006 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,11 +1,9 @@ -from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday from etl.hubSpotClient.hubspot import HubSpotClient, DealStage from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd -# This script will break if we change our current processses, obviously... - class SurveyPrice(): """ A class to work out all prices and uploads to sharepoint for review @@ -135,7 +133,8 @@ class SurveyPrice(): return self.all_hubspot_submissions def get_all_surveyed_data_from_sharepoint(self): - # TODO: A wrapper function + # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for + # DIfferent installers self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() return self.all_survey_info_from_sharepoint @@ -293,10 +292,15 @@ class SurveyPrice(): final_list.append(merged_row) return pd.concat(final_list, ignore_index=True) + + def upload_to_sharepoint(self, file_path_to_upload, file_name): + parent_folder = "Commercials/Rate Cards" + date = previous_monday() + self.sharepoint_client.create_dir(date, parent_folder) + sharepoint_path = parent_folder + "/" + date - - + self.sharepoint_client.upload_file(file_path_to_upload, sharepoint_path, file_name) # TODO diff --git a/etl/utils/sharepoint/sharepoint.py b/etl/utils/sharepoint/sharepoint.py index a366161..e175691 100644 --- a/etl/utils/sharepoint/sharepoint.py +++ b/etl/utils/sharepoint/sharepoint.py @@ -274,6 +274,29 @@ class SharePointClient: url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children" return 'POST', url, data + + def upload_file(self, file_name, file_stream, sharepoint_parent_id): + """ + Uploads a file to SharePoint using the Graph API. + PUT /drives/{drive-id}/root:/{path-to-file}:/content + + :param file_name: Name of the file to upload + :param sharepoint_path: Path within the SharePoint site (folder path) + :param file_stream: File content as a binary stream (e.g., BytesIO or open(file, 'rb')) + :return: Response JSON from the API + """ + url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content" + logger.debug(f"Uploading file to URL: {url}") + + response = requests.put(url, headers=self.headers, data=file_stream) + + if response.status_code in (200, 201): + logger.info(f"File '{file_name}' uploaded successfully.") + return response.json() + else: + retry = handle_error(response) + if retry == 'retry': + return self.upload_file(file_name, sharepoint_parent_id, file_stream) @staticmethod def download_sharepoint_file(download_url): From 2ccee63a8c8a3265fe0f65a78d7ba0f60445f010 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 11:10:20 +0000 Subject: [PATCH 19/21] deem score calcualted with hubspot and sharepoint integration --- etl/hubSpotClient/hubspot.py | 15 ++++++++++++++- etl/hubspot_to_invoice.py | 8 ++++++-- etl/surveyPrice/surveyPrice.py | 17 +++++------------ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 64aea46..cf022e6 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -1,12 +1,14 @@ import hubspot from enum import Enum from hubspot.crm.deals import PublicObjectSearchRequest +from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal class DealStage(Enum): SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914" SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" + SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" class HubSpotClient(): def __init__(self): @@ -69,4 +71,15 @@ class HubSpotClient(): print(f" - Label: {stage.label}") print(f" ID: {stage.id}") - + def move_deals_to_different_stage(self, list_of_deals_id, to_stage_id): + deal_properties = SimplePublicObjectInput( + properties={ + "dealstage": to_stage_id + } + ) + for deal_id in list_of_deals_id: + self.client.crm.deals.basic_api.update( + deal_id, + simple_public_object_input=deal_properties + ) + print(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 156b2c7..d8473f7 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -28,10 +28,14 @@ output_path = os.path.abspath(lewis_view) sp.upload_to_sharepoint(output_path, lewis_view) +deal_ids = df["HUBSPOT_DEAL_ID"].tolist() + +sp.move_deals_to_completed(deal_ids) + # TODO: -# Move deal in hub spot Surveyoed complete signed off, if normal # Write documentation for tech demos from Khalims demo -# Look into getting solar working. ACIS and Lewis +# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples # Figure out what to do if I see an address that isn't registered but surveyrod +# Review deem score with last weeks deem score values to ensure accuracy diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 596c006..34eda07 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -132,6 +132,11 @@ class SurveyPrice(): self.all_hubspot_submissions = pd.DataFrame(all_deals) return self.all_hubspot_submissions + def move_deals_to_completed(self, deals): + hubspot = HubSpotClient() + hubspot.move_deals_to_different_stage(deals, DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) + + def get_all_surveyed_data_from_sharepoint(self): # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for # DIfferent installers @@ -301,15 +306,3 @@ class SurveyPrice(): sharepoint_path = parent_folder + "/" + date self.sharepoint_client.upload_file(file_path_to_upload, sharepoint_path, file_name) - - -# TODO -# Get it working for JJC first, with an idea to make it more diverse -# Add some TDD to ensure JJC values are correct -# The script can run weekly, for development I can just get one data -# Upload both the W.C. calculations and Rate Card to sharepoint -# Due considerations -> Piece of UI, as osmosis, dashboard upload and pushes site notes database and condition report to database. Link the two, make a due consideration report, due consideration report, propery_id () -# Property Tbale - -# Add hubspot deal id into pandas -# Deem score and move deal id to 'Surveyed Completed Signed off" From 2fafc0d4fb4b218c0345b2fa6492e93ae1c72cc4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 17:48:01 +0000 Subject: [PATCH 20/21] added a script that allow domna notes to be extracted from deals --- etl/dimitra_hubspot_notes_gather.py | 109 ++++++++++++++++++++++++++++ etl/hubSpotClient/hubspot.py | 80 ++++++++++++++++++++ poetry.lock | 37 +++++++++- pyproject.toml | 1 + 4 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 etl/dimitra_hubspot_notes_gather.py diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py new file mode 100644 index 0000000..923c395 --- /dev/null +++ b/etl/dimitra_hubspot_notes_gather.py @@ -0,0 +1,109 @@ +import os + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +import pandas as pd +from bs4 import BeautifulSoup +from openpyxl import Workbook +from openpyxl.styles import Font + + +hubspot = HubSpotClient() +import time +pipelines_to_include =[ + "SALES - SOCIAL HOUSING", + "PVT PAY", + "NRLA GENERAL ENQUIRIES", + # "OSMOSIS - SALES", + +] +exclude_stage = { + "SALES - SOCIAL HOUSING" : [ + "HA TO REENGAGE", + "APPOINTMENT SCHEDULED", + "AWAITING ASSET LIST", + "ASSET LIST RECEIVED", + "ASSET LIST STANDARDISED", + "ROUTE MARCH CREATED", + "HA WEEKLY REPORTING", + ], + "PVT PAY": [ + "LIVE OPPORTUNITY", + "CLOSED LOST", + "INVOICED", + "COLD - KIT", + "CLOSED WON", + + ], + "NRLA GENERAL ENQUIRIES": [ + "CUSTOMER CONTACTED", + "LOST", + "COLD", + ] +} + +include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include] +exclude_stage_upper = [s.upper().strip() for s in exclude_stage] +notes_data = [] +pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals") +for pipeline in pipelines.results: + pipeline_name = pipeline.label.upper().strip() + if pipeline_name in pipelines_to_include: + for stage in pipeline.stages: + if stage.label.upper().strip() not in exclude_stage[pipeline_name]: + for deal_id in hubspot.get_all_deals_from_stage_id(stage.id): + notes = hubspot.get_notes_from_deals_id(deal_id) + for note in notes: + deal_name = hubspot.get_deal_name_by_id(deal_id) + html_body = note['note'] + soup = BeautifulSoup(html_body, "html.parser") + plain_text = soup.get_text(separator="\n") # Keeps line breaks + notes_data.append({ + "note_body": plain_text, + "deal_name": deal_name, # Include deal_id to relate the note to the deal + "pipeline_name": pipeline.label # Add the pipeline name + }) + + time.sleep(0.75) + print("delay to not bombard the server") + +notes_df = pd.DataFrame(notes_data) +notes_df.to_csv("output.csv") +df = notes_df + +wb = Workbook() +wb.remove(wb.active) # Remove default sheet + +for pipeline, group_df in df.groupby("pipeline_name"): + ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars + + # Sort by deal name + group_df = group_df.sort_values("deal_name") + + current_row = 1 + for deal_name, deal_notes in group_df.groupby("deal_name"): + # Bold header for each deal + ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}") + ws.cell(row=current_row, column=1).font = Font(bold=True) + current_row += 1 + + # Notes for the deal + for note in deal_notes["note_body"]: + ws.cell(row=current_row, column=2, value=note) + current_row += 1 + + # Add a blank row between groups + current_row += 1 + +# Save to Excel +file_name = "DEAL_NOTES_FROM_HUBSPOT.xlsx" +wb.save(file_name) +output_path = os.path.abspath(file_name) +sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) +sharepoint_client.upload_file(output_path, f"/02. Sales and Marketing/02. DEAL Notes from Hubspot/{previous_monday}",file_name) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index cf022e6..4924d27 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -4,6 +4,8 @@ from hubspot.crm.deals import PublicObjectSearchRequest from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal + + class DealStage(Enum): SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914" SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" @@ -17,7 +19,85 @@ class HubSpotClient(): def get_all_deals(self): return self.client.crm.deals.get_all() + + + def get_deal_name_by_id(self, deal_id): + try: + deal = self.client.crm.deals.basic_api.get_by_id(deal_id) + return deal.properties.get("dealname", "No deal name") + except Exception as e: + return "Unknown Deal" # Fallback if the deal name is not found + def get_notes_from_deals_id(self, deals_id): + from hubspot.crm.objects import PublicObjectSearchRequest + found_notes = [] + after = None + while True: + # Correct filter for notes associated with the given deal ID + search_request = PublicObjectSearchRequest( + filter_groups=[{ + "filters": [{ + "propertyName": "associations.deal", # Filter by association to the deal + "operator": "EQ", + "value": deals_id, + }] + }], + properties=["hs_note_body", "hubspot_owner_id"], # Properties of the note you need + limit=200, + after=after, + ) + # Call the search API + response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request) + + # Add the results to the found_notes list + found_notes.extend(response.results) + + # Handle pagination if more results are available + if not response.paging or not response.paging.next: + break + after = response.paging.next.after + + all_notes = [] + for note in found_notes: + # Extract note content and author information + note_body = note.properties.get("hs_note_body", "No content") + + # Collect note details in a dictionary + all_notes.append({ + "note_id": note.id, + "note": note_body, + }) + return all_notes + + + def get_all_deals_from_stage_id(self, stage_id): + found_deals = [] + after = None + while True: + search_request = PublicObjectSearchRequest( + filter_groups=[{ + "filters": [{ + "propertyName": "dealstage", + "operator": "EQ", + "value": stage_id, + }] + }], + properties=[ + "dealname", + ], + limit=200, + after=after, + ) + response = self.client.crm.deals.search_api.do_search(search_request) + found_deals.extend(response.results) + if not response.paging or not response.paging.next: + break + after = response.paging.next.after + + all_deals = [] + for deal in found_deals: + all_deals.append(deal.id) + return all_deals def get_deals_from_deal_stage(self, deal_stage: DealStage): found_deals = [] diff --git a/poetry.lock b/poetry.lock index b055ca9..ddd2ad3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -61,6 +61,29 @@ files = [ astroid = ["astroid (>=2,<4)"] test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "beautifulsoup4" +version = "4.13.4" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +files = [ + {file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"}, + {file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"}, +] + +[package.dependencies] +soupsieve = ">1.2" +typing-extensions = ">=4.0.0" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "certifi" version = "2025.1.31" @@ -1684,6 +1707,18 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + [[package]] name = "sqlalchemy" version = "2.0.40" @@ -1925,4 +1960,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "55a974b3a81d57c429f61ee6a12a84d38f5c703fdfdfdf2553bec6ba21c29bf5" +content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2" diff --git a/pyproject.toml b/pyproject.toml index c86b613..c26faba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "pytest (>=8.3.5,<9.0.0)", "hubspot-api-client (>=11.1.0,<12.0.0)", "monday (>=2.0.1,<3.0.0)", + "beautifulsoup4 (>=4.13.4,<5.0.0)", ] [tool.poetry] From 44b9e97600f8a6d647addfaa66c11282fbd85d63 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 18:02:32 +0000 Subject: [PATCH 21/21] added script to for marketing operations --- .github/workflows/hubspot_deal_notes.yml | 29 ++++++++++++++++++++++++ etl/dimitra_hubspot_notes_gather.py | 21 +++++++++++------ 2 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/hubspot_deal_notes.yml diff --git a/.github/workflows/hubspot_deal_notes.yml b/.github/workflows/hubspot_deal_notes.yml new file mode 100644 index 0000000..ce80afe --- /dev/null +++ b/.github/workflows/hubspot_deal_notes.yml @@ -0,0 +1,29 @@ +name: Deal Notes From HubSpot Scraper +on: + schedule: + - cron: '0 19 * * 0' + workflow_dispatch: + +jobs: + sharepoint-validator: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: run script + run: | + pwd + ls -la + poetry run python etl/dimitra_hubspot_notes_gather.py + env: + PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 923c395..fd3b385 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -1,10 +1,8 @@ import os -os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" -os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" -os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" -os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday from etl.hubSpotClient.hubspot import HubSpotClient, DealStage @@ -102,8 +100,17 @@ for pipeline, group_df in df.groupby("pipeline_name"): current_row += 1 # Save to Excel -file_name = "DEAL_NOTES_FROM_HUBSPOT.xlsx" +from datetime import datetime, timedelta +today = datetime.today() +days_ahead = (7 - today.weekday()) % 7 +days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday +next_monday = today + timedelta(days=days_ahead) + +formatted = next_monday.strftime("Monday %d-%m-%Y") + + +file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx" wb.save(file_name) output_path = os.path.abspath(file_name) sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) -sharepoint_client.upload_file(output_path, f"/02. Sales and Marketing/02. DEAL Notes from Hubspot/{previous_monday}",file_name) +sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name)