diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 823f974..1ccb286 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -20,7 +20,9 @@ "mechatroner.rainbow-csv", "ms-toolsai.datawrangler", "lindacong.vscode-book-reader", - "4ops.terraform" + "4ops.terraform", + "fabiospampinato.vscode-todo-plus", + "jgclark.vscode-todo-highlight" ] } } diff --git a/.github/workflows/hubspot_deal_notes.yml b/.github/workflows/hubspot_deal_notes.yml new file mode 100644 index 0000000..ce80afe --- /dev/null +++ b/.github/workflows/hubspot_deal_notes.yml @@ -0,0 +1,29 @@ +name: Deal Notes From HubSpot Scraper +on: + schedule: + - cron: '0 19 * * 0' + workflow_dispatch: + +jobs: + sharepoint-validator: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: run script + run: | + pwd + ls -la + poetry run python etl/dimitra_hubspot_notes_gather.py + env: + PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py new file mode 100644 index 0000000..fd3b385 --- /dev/null +++ b/etl/dimitra_hubspot_notes_gather.py @@ -0,0 +1,116 @@ +import os + +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" + +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +import pandas as pd +from bs4 import BeautifulSoup +from openpyxl import Workbook +from openpyxl.styles import Font + + +hubspot = HubSpotClient() +import time +pipelines_to_include =[ + "SALES - SOCIAL HOUSING", + "PVT PAY", + "NRLA GENERAL ENQUIRIES", + # "OSMOSIS - SALES", + +] +exclude_stage = { + "SALES - SOCIAL HOUSING" : [ + "HA TO REENGAGE", + "APPOINTMENT SCHEDULED", + "AWAITING ASSET LIST", + "ASSET LIST RECEIVED", + "ASSET LIST STANDARDISED", + "ROUTE MARCH CREATED", + "HA WEEKLY REPORTING", + ], + "PVT PAY": [ + "LIVE OPPORTUNITY", + "CLOSED LOST", + "INVOICED", + "COLD - KIT", + "CLOSED WON", + + ], + "NRLA GENERAL ENQUIRIES": [ + "CUSTOMER CONTACTED", + "LOST", + "COLD", + ] +} + +include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include] +exclude_stage_upper = [s.upper().strip() for s in exclude_stage] +notes_data = [] +pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals") +for pipeline in pipelines.results: + pipeline_name = pipeline.label.upper().strip() + if pipeline_name in pipelines_to_include: + for stage in pipeline.stages: + if stage.label.upper().strip() not in exclude_stage[pipeline_name]: + for deal_id in hubspot.get_all_deals_from_stage_id(stage.id): + notes = hubspot.get_notes_from_deals_id(deal_id) + for note in notes: + deal_name = hubspot.get_deal_name_by_id(deal_id) + html_body = note['note'] + soup = BeautifulSoup(html_body, "html.parser") + plain_text = soup.get_text(separator="\n") # Keeps line breaks + notes_data.append({ + "note_body": plain_text, + "deal_name": deal_name, # Include deal_id to relate the note to the deal + "pipeline_name": pipeline.label # Add the pipeline name + }) + + time.sleep(0.75) + print("delay to not bombard the server") + +notes_df = pd.DataFrame(notes_data) +notes_df.to_csv("output.csv") +df = notes_df + +wb = Workbook() +wb.remove(wb.active) # Remove default sheet + +for pipeline, group_df in df.groupby("pipeline_name"): + ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars + + # Sort by deal name + group_df = group_df.sort_values("deal_name") + + current_row = 1 + for deal_name, deal_notes in group_df.groupby("deal_name"): + # Bold header for each deal + ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}") + ws.cell(row=current_row, column=1).font = Font(bold=True) + current_row += 1 + + # Notes for the deal + for note in deal_notes["note_body"]: + ws.cell(row=current_row, column=2, value=note) + current_row += 1 + + # Add a blank row between groups + current_row += 1 + +# Save to Excel +from datetime import datetime, timedelta +today = datetime.today() +days_ahead = (7 - today.weekday()) % 7 +days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday +next_monday = today + timedelta(days=days_ahead) + +formatted = next_monday.strftime("Monday %d-%m-%Y") + + +file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx" +wb.save(file_name) +output_path = os.path.abspath(file_name) +sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) +sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 353ecd2..4924d27 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -1,12 +1,16 @@ import hubspot from enum import Enum from hubspot.crm.deals import PublicObjectSearchRequest +from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal + + class DealStage(Enum): SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914" SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" + SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" class HubSpotClient(): def __init__(self): @@ -15,7 +19,85 @@ class HubSpotClient(): def get_all_deals(self): return self.client.crm.deals.get_all() + + + def get_deal_name_by_id(self, deal_id): + try: + deal = self.client.crm.deals.basic_api.get_by_id(deal_id) + return deal.properties.get("dealname", "No deal name") + except Exception as e: + return "Unknown Deal" # Fallback if the deal name is not found + def get_notes_from_deals_id(self, deals_id): + from hubspot.crm.objects import PublicObjectSearchRequest + found_notes = [] + after = None + while True: + # Correct filter for notes associated with the given deal ID + search_request = PublicObjectSearchRequest( + filter_groups=[{ + "filters": [{ + "propertyName": "associations.deal", # Filter by association to the deal + "operator": "EQ", + "value": deals_id, + }] + }], + properties=["hs_note_body", "hubspot_owner_id"], # Properties of the note you need + limit=200, + after=after, + ) + # Call the search API + response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request) + + # Add the results to the found_notes list + found_notes.extend(response.results) + + # Handle pagination if more results are available + if not response.paging or not response.paging.next: + break + after = response.paging.next.after + + all_notes = [] + for note in found_notes: + # Extract note content and author information + note_body = note.properties.get("hs_note_body", "No content") + + # Collect note details in a dictionary + all_notes.append({ + "note_id": note.id, + "note": note_body, + }) + return all_notes + + + def get_all_deals_from_stage_id(self, stage_id): + found_deals = [] + after = None + while True: + search_request = PublicObjectSearchRequest( + filter_groups=[{ + "filters": [{ + "propertyName": "dealstage", + "operator": "EQ", + "value": stage_id, + }] + }], + properties=[ + "dealname", + ], + limit=200, + after=after, + ) + response = self.client.crm.deals.search_api.do_search(search_request) + found_deals.extend(response.results) + if not response.paging or not response.paging.next: + break + after = response.paging.next.after + + all_deals = [] + for deal in found_deals: + all_deals.append(deal.id) + return all_deals def get_deals_from_deal_stage(self, deal_stage: DealStage): found_deals = [] @@ -53,9 +135,9 @@ class HubSpotClient(): deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], work_type=deal.properties["work_type"], - needs_trickle_ventilation=True if deal.properties["property_needs_trickle_vents"].upper() == "YES" else False, + needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False, post_sap_score=int(deal.properties["domna_survey_post_sap"]), - existing_wall_insulation=deal.properties["existing_wall_insulation"], + existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), installer=deal.properties["installer"], )) @@ -67,4 +149,17 @@ class HubSpotClient(): print(f"Pipeline: {pipeline.label}") for stage in pipeline.stages: print(f" - Label: {stage.label}") - print(f" ID: {stage.id}") # + print(f" ID: {stage.id}") + + def move_deals_to_different_stage(self, list_of_deals_id, to_stage_id): + deal_properties = SimplePublicObjectInput( + properties={ + "dealstage": to_stage_id + } + ) + for deal_id in list_of_deals_id: + self.client.crm.deals.basic_api.update( + deal_id, + simple_public_object_input=deal_properties + ) + print(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py new file mode 100644 index 0000000..d8473f7 --- /dev/null +++ b/etl/hubspot_to_invoice.py @@ -0,0 +1,41 @@ +import os + + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.surveyPrice.surveyPrice import SurveyPrice + + +sp = SurveyPrice() + +df = sp.calculate_all_price() + + +verbose_file = "verbose_invoice_score.xlsx" +df.to_excel(verbose_file, index=False) +output_path = os.path.abspath(verbose_file) +sp.upload_to_sharepoint(output_path, verbose_file) + +lewis_view = "FOR_LEWIS.xlsx" +selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"] +minimal_df = df[selected_columns] +minimal_df.to_excel(lewis_view, index=False) +output_path = os.path.abspath(lewis_view) +sp.upload_to_sharepoint(output_path, lewis_view) + + +deal_ids = df["HUBSPOT_DEAL_ID"].tolist() + +sp.move_deals_to_completed(deal_ids) + + +# TODO: +# Write documentation for tech demos from Khalims demo +# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples +# Figure out what to do if I see an address that isn't registered but surveyrod +# Review deem score with last weeks deem score values to ensure accuracy + diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 30356b0..58c094d 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -11,7 +11,7 @@ import pandas as pd osmosis = SharePointScraper(SharePointInstaller.OSMOSIS) parent_folder = "Automated Example" -osmosis.create_file(parent_folder, "/JTK Test Folder") +osmosis.create_dir(parent_folder, "/JTK Test Folder") asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing") @@ -20,30 +20,30 @@ new_asset_list = [] parent_folder = "JTK Test Folder/Automated Example" # Create asset list and location for index, address in asset_list.iterrows(): - webUrl = osmosis.create_file(address['Name'], parent_folder) + webUrl = osmosis.create_dir(address['Name'], parent_folder) first_folder = "1. Retrofit Assessment" - osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") - osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir(first_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") second_folder = "2. RC Mid-Term Plan" - osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}") + osmosis.create_dir(second_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("SAP", parent_folder + f"/{address['Name']}/{second_folder}") third_folder = "3. Retrofit Design" - osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(third_folder, parent_folder + f"/{address['Name']}") fourth_folder = "4. Post EPC" - osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") + osmosis.create_dir(fourth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") fifth_folder = "5. Trustmark Lodgement" - osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}") - osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir(fifth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_dir("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") - osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") asset_data = { "Name": address['Name'], diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index dadaf7b..3e65425 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,8 +14,8 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - # return f"W.C. 31.03.2025" - return f"W.C. {last_monday.strftime('%d.%m.%Y')}" + return f"W.C. 31.03.2025" + # return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) @@ -27,6 +27,7 @@ class SharePointInstaller(Enum): BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4") OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") + WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4") class SharePointScraper(): """ @@ -116,7 +117,7 @@ class SharePointScraper(): return True return False - def create_file(self, file_name, at_path="/"): + def create_dir(self, file_name, at_path="/"): sharepoint_client = SharePointClient( tenant_id=self.sharepoint_tenant_id, @@ -131,6 +132,19 @@ class SharePointScraper(): for folders in self.get_folders_in_path(at_path)['value']: if file_name.upper() in folders["name"].upper(): return folders["webUrl"] + + def upload_file(self, file_path, sharepoint_path, file_name): + sharepoint_client = SharePointClient( + tenant_id=self.sharepoint_tenant_id, + client_id=self.sharepoint_client_id, + client_secret=self.sharepoint_client_secret, + site_id=self.sharepoint_drive.value, + ) + def get_file_stream(file_path): + return open(file_path, 'rb') + + sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) + @@ -308,4 +322,5 @@ class SharePointScraper(): temp_file.write(content.getvalue()) self.logger.info(f"Temporary file created at: {path}") - return path \ No newline at end of file + return path + \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 38a224f..34eda07 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -1,25 +1,308 @@ +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from etl.surveyedData.surveryedData import surveyedDataProcessor +import pandas as pd -class surveyPrice(): +class SurveyPrice(): """ - A class to work out all prices and uploads to sharepoint + A class to work out all prices and uploads to sharepoint for review """ def __init__(self): - pass + self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) + self.master_rate_card_path = None + self.all_hubspot_submissions = None + self.all_survey_info_from_sharepoint = None + self.download_price_card() + + self.required_sheets = [ + 'JJC - EMPTIES', + 'JJC - GENERAL EXTRACTIONS', + 'JJC - FORMALDEHYDE EXTRACTION', + 'JJC - SOLAR', + 'SCIS - GENERAL EXTRACTIONS', + 'SCIS - EMPTIES', + 'SCIS - SOLAR', + 'SGEC - EMPTIES', + 'SGEC - GENERAL EXTRACTIONS', + 'SGEC - FORMALDEHYDE EXTRACTION' + ] + + self.installer = { + "J & J CRUMP": "JJC", + "SCIS": "SCIS", + "SGEC": "SGEC", + "BAXTER KELLY": "BAXTER KELLY", + } + + self.domna_job_to_price_sheet_convertor = { + "JJC - SOLAR": "JJC - SOLAR", + "JJC - EMPTY CAVITY": "JJC - EMPTIES", + "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS", + "SCIS - SOLAR": "SCIS - SOLAR", + "SCIS - EMPTY CAVITY": "SCIS - EMPTIES", + "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION", + "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS", + "SGEC - EMPTY CAVITY": "SGEC - EMPTIES", + "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS", + } def download_price_card(self): - pass + url = None + # TODO: Some sanity checks to ensure rate cards title stays consistent + for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: + if files['name'] == "MASTER RATE CARD.xlsx": + url = files['@microsoft.graph.downloadUrl'] + break - def get_price_card(): - pass + if url: + content = self.sharepoint_client.get_file_content(url) + self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") + + return self.master_rate_card_path + + + def get_cavity_pricing_table(self, sheet_name): + excel_file = pd.ExcelFile(self.master_rate_card_path) + available_sheets = excel_file.sheet_names + missing_sheets = [sheet for sheet in self.required_sheets if sheet not in available_sheets] + + if missing_sheets: + raise ValueError(f"Missing sheets: {missing_sheets}") + + pricing_table = self.get_price_matrix(sheet_name) + + return pricing_table + def get_master_rate_card_path(self): + return self.master_rate_card_path + + def get_price_matrix(self, sheet_name): + df = pd.read_excel(self.master_rate_card_path, sheet_name) + + if "SOLAR" in sheet_name.upper(): + return df + else: + columns_to_check = { + "no extractors or ventilation required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 0}, + "Trickle Vents ONLY": {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 0}, + "1 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 1}, + "2 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 2}, + "3 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 3}, + 'Trickle Vents + 1 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 1}, + 'Trickle Vents + 2 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 2}, + 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, + } + + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "WORK TYPE": row["WORK TYPE"], + "FLOOR_AREA_BANDING": row["Total Floor Area"][:-1], + **variables, + "PRICE": row[key] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table + + def get_all_surveys_from_hubspot(self): + hubSpotClient = HubSpotClient() + deals = hubSpotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + all_deals = [] + + for deal in deals: + all_deals.append({ + "HUBSPOT_DEAL_ID": deal.deal_id, + "HUBSPOT_WORK_TYPE": deal.work_type, + "HUBSPOT_DEAL_ADDRESS": deal.deal_name, + "HUBSPOT_TRICKLE_VENT":1 if deal.needs_trickle_ventilation else 0, + "HUBSPOT_WALL_INSULATION": deal.existing_wall_insulation, + "HUBSPOT_POST_INSTALL_SAP_SCORE": deal.post_sap_score, + "HUBSPOT_INSTALLER": deal.installer, + "HUBSPOT_WETROOMS": deal.no_of_wet_rooms, + }) + + self.all_hubspot_submissions = pd.DataFrame(all_deals) + return self.all_hubspot_submissions + + def move_deals_to_completed(self, deals): + hubspot = HubSpotClient() + hubspot.move_deals_to_different_stage(deals, DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) + + + def get_all_surveyed_data_from_sharepoint(self): + # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for + # DIfferent installers + self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() + return self.all_survey_info_from_sharepoint + + def sharepoint_data_for_jjc(self): + jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) + file_paths = jjc_sp.download_file_for_each_address() + jjc_surveys = [] + + for eachAddress in file_paths: + for address, files in eachAddress.items(): + jjc_surveys.append(surveyedDataProcessor(address, files)) + + all_survey_info = [] + for surveyInfo in jjc_surveys: + cavity_wall_as_built = False + csr = False + foam_insulation = False + + info = { + "SHAREPOINT INSTALLER": "J & J CRUMP", + "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, + "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, + "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", + "SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND", + "SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND", + "SHAREPOINT INSULATION MATERIAL": None, + "SHAREPOINT ADDRESS": address + } + + if surveyInfo.pre_site_note: + floor_banding, total_floor_area = surveyInfo.work_out_total_floor_area() + pre_sap_score = surveyInfo.get_current_sap_score() + pre_sap_score_banding = surveyedDataProcessor.get_band(pre_sap_score) + info.update({ + "SHAREPOINT TOTAL_FLOOR_AREA": total_floor_area, + "SHAREPOINT FLOOR_AREA_BANDING": floor_banding, + "SHAREPOINT PRE_INSTALL_SAP_SCORE": pre_sap_score, + "SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING": pre_sap_score_banding, + }) + + if surveyInfo.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \ + and surveyInfo.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": + cavity_wall_as_built = True + + if surveyInfo.csr: + csr = True + insulation = surveyInfo.get_insulation_info() + info.update({ + "SHAREPOINT INSULATION MATERIAL": insulation, + }) + if "FOAM" in insulation.upper(): + foam_insulation = True + + if cavity_wall_as_built: + if csr: + if foam_insulation: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FOAM FILLED CAVITY", + }) + else: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FILLED CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "EMPTY CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "SOLAR" + }) + all_survey_info.append(info) -# Step one -# Make a copy of the rate card and work out each price matrix -# Make the price card downloadable as a single excelt sheet for viewing -# For other things, do some TDD so it is a little robust -# The script can run weekly, for development I can just get one data -# Expected input, expect output etc + return pd.DataFrame(all_survey_info) + + def merge_hub_spot_and_survey_information(self): + if self.all_survey_info_from_sharepoint is None: + raise RuntimeError("No survey information found from Sharepoint") + if self.all_hubspot_submissions is None: + raise RuntimeError("No information found from Hubspot") + + # Standardise address + self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply( + lambda x: x.lower().replace(',', '').strip() + ) + + self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply( + lambda x: x.lower().replace(',', '').strip() + ) + + # re-name to installer + self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( + columns={ + 'SHAREPOINT INSTALLER': 'INSTALLER', + 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', + } + ) + + self.all_hubspot_submissions = self.all_hubspot_submissions.rename( + columns={ + 'HUBSPOT_INSTALLER': 'INSTALLER', + 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', + 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', + } + ) + + merged_df = pd.merge( + self.all_survey_info_from_sharepoint, + self.all_hubspot_submissions, + left_on=['clean_address', 'INSTALLER'], + right_on=['clean_address', 'INSTALLER'], + how='inner' + ) + + merged_df.drop(columns=['clean_address'], inplace=True) + + def compute_energy_grant(row): + pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1] + post_band_letter = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"])[-1] + return surveyedDataProcessor.gbis_or_eco4_scheme(pre_band_letter, post_band_letter) + + def compute_banding_for_post_sap(row): + post_sap_banding = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"]) + return post_sap_banding + + def work_type(row): + if row["ENERGY_GRANT"] == "GBIS": + return row["ENERGY GRANT"] + else: + return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}" + + + # Add missing variables + merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1) + merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1) + merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1) + + return merged_df + + def calculate_all_price(self): + self.get_all_surveys_from_hubspot() + self.get_all_surveyed_data_from_sharepoint() + submission_data = self.merge_hub_spot_and_survey_information() + final_list = [] + for _, row in submission_data.iterrows(): + if "SOLAR" in row["DOMNA JOB TYPE"].upper(): + raise NotImplementedError("Please implement solar pricing") + else: + # Cavity wall + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') + final_list.append(merged_row) + + return pd.concat(final_list, ignore_index=True) + + def upload_to_sharepoint(self, file_path_to_upload, file_name): + parent_folder = "Commercials/Rate Cards" + date = previous_monday() + self.sharepoint_client.create_dir(date, parent_folder) + + sharepoint_path = parent_folder + "/" + date + + self.sharepoint_client.upload_file(file_path_to_upload, sharepoint_path, file_name) diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index bc3054a..1b5248e 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,5 +1,6 @@ from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.pdfReader.reportType import ReportType +import math class surveyedDataProcessor(): def __init__(self, address, files): @@ -17,5 +18,130 @@ class surveyedDataProcessor(): if pdf: if pdf.type == ReportType.QUIDOS_PRESITE_NOTE: self.pre_site_note = pdf.get_reader() + self.address = self.pre_site_note.survey_information.address elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT: self.csr = pdf.get_reader() + + def get_insulation_info(self): + if self.csr: + if self.csr.insulation_info: + insultation = self.csr.insulation_info.type.upper() + return insultation + return None + + @staticmethod + def get_band(sap_score_number): + bands = [ + ("HIGH A", 96, float("inf")), + ("LOW A", 92, 96), + ("HIGH B", 86, 92), + ("LOW B", 81, 86), + ("HIGH C", 74.5, 81), + ("LOW C", 69, 74.5), + ("HIGH D", 61.5, 69), + ("LOW D", 55, 61.5), + ("HIGH E", 46.5, 55), + ("LOW E", 39, 46.5), + ("HIGH F", 29.5, 39), + ("LOW F", 21, 29.5), + ("HIGH G", 10.5, 21), + ("LOW G", 1, 10.5), + ] + + for band, lower, upper in bands: + if lower <= sap_score_number < upper: + return band + + return None + + @staticmethod + def gbis_or_eco4_scheme(presap_letter, postsap_letter): + """ + *ECO4 (minimum movement)* + D to C + E to C + F to D + G to D + + G -> ABCD + F -> ABCD + E -> ABC + D -> ABC + + *GBIS - Cavity Wall Insulation ONLY* + + D-D + E-D + E-E + F-E + F-F + G-E + G-F + G-G + """ + eco4 = { + "G": ['A', 'B', 'C', 'D'], + "F": ['A', 'B', 'C', 'D'], + "E": ['A', 'B', 'C'], + "D": ['A', 'B', 'C'], + } + + gbis ={ + 'D': ['D'], + 'E': ['E', 'D'], + 'F': ['E', 'F'], + 'G': ['E', 'F', 'G'], + } + + if presap_letter.upper() in eco4: + if postsap_letter.upper() in eco4[presap_letter.upper()]: + return "ECO4" + if presap_letter.upper() in gbis: + if postsap_letter.upper() in gbis[presap_letter.upper()]: + return "GBIS" + + return None + + + + def work_out_total_floor_area(self): + + total = 0 + def add_all_floors(floor_list): + total = 0 + for floor in floor_list: + total += floor.floor_area_m2 + + return total + + main = True if self.pre_site_note.property_description.no_of_main_property > 0 else False + ext1 = True if self.pre_site_note.property_description.no_of_extension_1 > 0 else False + ext2 = True if self.pre_site_note.property_description.no_of_extension_2 > 0 else False + ext3 = True if self.pre_site_note.property_description.no_of_extension_3 > 0 else False + ext4 = True if self.pre_site_note.property_description.no_of_extension_4 > 0 else False + + total += add_all_floors(self.pre_site_note.property_description.main_property.dimensions) if main is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex1_property.dimensions) if ext1 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex2_property.dimensions) if ext2 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0 + total += add_all_floors(self.pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0 + + + floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total) + if 0 <= floor_area <= 72: + return '0-72m', floor_area + elif 72 < floor_area <= 97: + return '73-97m', floor_area + elif 97 < floor_area <= 199: + return '98-199m', floor_area + elif 199 <= floor_area: + return 'over 200m', floor_area + else: + raise NotImplementedError(f"unknown floor area {floor_area} {self.pre_site_note.summary_information.address}") + + def get_current_sap_score(self): + score_list = self.pre_site_note.survey_information.current_sap.split(" ") + score = int(score_list[1]) + return score + + diff --git a/etl/tests/test_survey_price.py b/etl/tests/test_survey_price.py new file mode 100644 index 0000000..1dc7cad --- /dev/null +++ b/etl/tests/test_survey_price.py @@ -0,0 +1,37 @@ +import os +# WarmFront Sharepoint KEYS +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" + +from etl.surveyPrice.surveyPrice import SurveyPrice +import pytest + + +@pytest.fixture(scope="module") +def sp(): + return SurveyPrice() + + +def cavity_price_dataframe_sanity_check(df): + assert df.shape == (160, 5) + assert df.columns.tolist() == ['WORK TYPE', 'Floor Area Group', 'Trickle Vent', 'Wetrooms', 'PRICE'] + +def test_get_price_matrix_jjc_empties(sp): + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - EMPTIES") + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + +def test_get_price_matrix_jjc_general_extraction(sp): + sp = SurveyPrice() + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - GENERAL EXTRACTIONS") + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + +def test_get_price_matrix_jjc_foam(sp): + sp = SurveyPrice() + jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - FORMALDEHYDE EXTRACTION") + cavity_price_dataframe_sanity_check(jjc_empties_price_table) + + + + + diff --git a/etl/transform/types.py b/etl/transform/types.py index 3342bee..7e25682 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -203,5 +203,11 @@ class PropertyDescription(BaseModel): mainHeating2: Optional[Heating] secondaryHeatingType: Optional[HeatingType] +# class PropertyReport(): + # TODO: Property description + # TODO: Due consideration foregin key + # TODO: Which company did it (Osmosis, Warmfront etc) + # TODO: Links to more foreign keys per report etc + class Insulation(BaseModel): type: str \ No newline at end of file diff --git a/etl/utils/sharepoint/sharepoint.py b/etl/utils/sharepoint/sharepoint.py index a366161..e175691 100644 --- a/etl/utils/sharepoint/sharepoint.py +++ b/etl/utils/sharepoint/sharepoint.py @@ -274,6 +274,29 @@ class SharePointClient: url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children" return 'POST', url, data + + def upload_file(self, file_name, file_stream, sharepoint_parent_id): + """ + Uploads a file to SharePoint using the Graph API. + PUT /drives/{drive-id}/root:/{path-to-file}:/content + + :param file_name: Name of the file to upload + :param sharepoint_path: Path within the SharePoint site (folder path) + :param file_stream: File content as a binary stream (e.g., BytesIO or open(file, 'rb')) + :return: Response JSON from the API + """ + url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content" + logger.debug(f"Uploading file to URL: {url}") + + response = requests.put(url, headers=self.headers, data=file_stream) + + if response.status_code in (200, 201): + logger.info(f"File '{file_name}' uploaded successfully.") + return response.json() + else: + retry = handle_error(response) + if retry == 'retry': + return self.upload_file(file_name, sharepoint_parent_id, file_stream) @staticmethod def download_sharepoint_file(download_url): diff --git a/poetry.lock b/poetry.lock index b055ca9..ddd2ad3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -61,6 +61,29 @@ files = [ astroid = ["astroid (>=2,<4)"] test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "beautifulsoup4" +version = "4.13.4" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +files = [ + {file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"}, + {file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"}, +] + +[package.dependencies] +soupsieve = ">1.2" +typing-extensions = ">=4.0.0" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "certifi" version = "2025.1.31" @@ -1684,6 +1707,18 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + [[package]] name = "sqlalchemy" version = "2.0.40" @@ -1925,4 +1960,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "55a974b3a81d57c429f61ee6a12a84d38f5c703fdfdfdf2553bec6ba21c29bf5" +content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2" diff --git a/pyproject.toml b/pyproject.toml index c86b613..c26faba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "pytest (>=8.3.5,<9.0.0)", "hubspot-api-client (>=11.1.0,<12.0.0)", "monday (>=2.0.1,<3.0.0)", + "beautifulsoup4 (>=4.13.4,<5.0.0)", ] [tool.poetry]