mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-07-12 13:29:08 +00:00
Merge pull request #45 from Hestia-Homes/feature/hubspot_to_compelte_deem
Feature/hubspot to compelte deem
This commit is contained in:
Jun-te Kim
GitHub
commit
4f4dc7aed2
14 changed files with 844 additions and 35 deletions
|
|
@ -20,7 +20,9 @@
|
|||
"mechatroner.rainbow-csv",
|
||||
"ms-toolsai.datawrangler",
|
||||
"lindacong.vscode-book-reader",
|
||||
"4ops.terraform"
|
||||
"4ops.terraform",
|
||||
"fabiospampinato.vscode-todo-plus",
|
||||
"jgclark.vscode-todo-highlight"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
29
.github/workflows/hubspot_deal_notes.yml
vendored
Normal file
29
.github/workflows/hubspot_deal_notes.yml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
name: Deal Notes From HubSpot Scraper
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 19 * * 0'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
sharepoint-validator:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install poetry
|
||||
poetry install --no-root
|
||||
|
||||
- name: run script
|
||||
run: |
|
||||
pwd
|
||||
ls -la
|
||||
poetry run python etl/dimitra_hubspot_notes_gather.py
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
116
etl/dimitra_hubspot_notes_gather.py
Normal file
116
etl/dimitra_hubspot_notes_gather.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import os
|
||||
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
|
||||
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
|
||||
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
|
||||
import pandas as pd
|
||||
from bs4 import BeautifulSoup
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font
|
||||
|
||||
|
||||
hubspot = HubSpotClient()
|
||||
import time
|
||||
pipelines_to_include =[
|
||||
"SALES - SOCIAL HOUSING",
|
||||
"PVT PAY",
|
||||
"NRLA GENERAL ENQUIRIES",
|
||||
# "OSMOSIS - SALES",
|
||||
|
||||
]
|
||||
exclude_stage = {
|
||||
"SALES - SOCIAL HOUSING" : [
|
||||
"HA TO REENGAGE",
|
||||
"APPOINTMENT SCHEDULED",
|
||||
"AWAITING ASSET LIST",
|
||||
"ASSET LIST RECEIVED",
|
||||
"ASSET LIST STANDARDISED",
|
||||
"ROUTE MARCH CREATED",
|
||||
"HA WEEKLY REPORTING",
|
||||
],
|
||||
"PVT PAY": [
|
||||
"LIVE OPPORTUNITY",
|
||||
"CLOSED LOST",
|
||||
"INVOICED",
|
||||
"COLD - KIT",
|
||||
"CLOSED WON",
|
||||
|
||||
],
|
||||
"NRLA GENERAL ENQUIRIES": [
|
||||
"CUSTOMER CONTACTED",
|
||||
"LOST",
|
||||
"COLD",
|
||||
]
|
||||
}
|
||||
|
||||
include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include]
|
||||
exclude_stage_upper = [s.upper().strip() for s in exclude_stage]
|
||||
notes_data = []
|
||||
pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals")
|
||||
for pipeline in pipelines.results:
|
||||
pipeline_name = pipeline.label.upper().strip()
|
||||
if pipeline_name in pipelines_to_include:
|
||||
for stage in pipeline.stages:
|
||||
if stage.label.upper().strip() not in exclude_stage[pipeline_name]:
|
||||
for deal_id in hubspot.get_all_deals_from_stage_id(stage.id):
|
||||
notes = hubspot.get_notes_from_deals_id(deal_id)
|
||||
for note in notes:
|
||||
deal_name = hubspot.get_deal_name_by_id(deal_id)
|
||||
html_body = note['note']
|
||||
soup = BeautifulSoup(html_body, "html.parser")
|
||||
plain_text = soup.get_text(separator="\n") # Keeps line breaks
|
||||
notes_data.append({
|
||||
"note_body": plain_text,
|
||||
"deal_name": deal_name, # Include deal_id to relate the note to the deal
|
||||
"pipeline_name": pipeline.label # Add the pipeline name
|
||||
})
|
||||
|
||||
time.sleep(0.75)
|
||||
print("delay to not bombard the server")
|
||||
|
||||
notes_df = pd.DataFrame(notes_data)
|
||||
notes_df.to_csv("output.csv")
|
||||
df = notes_df
|
||||
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active) # Remove default sheet
|
||||
|
||||
for pipeline, group_df in df.groupby("pipeline_name"):
|
||||
ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars
|
||||
|
||||
# Sort by deal name
|
||||
group_df = group_df.sort_values("deal_name")
|
||||
|
||||
current_row = 1
|
||||
for deal_name, deal_notes in group_df.groupby("deal_name"):
|
||||
# Bold header for each deal
|
||||
ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}")
|
||||
ws.cell(row=current_row, column=1).font = Font(bold=True)
|
||||
current_row += 1
|
||||
|
||||
# Notes for the deal
|
||||
for note in deal_notes["note_body"]:
|
||||
ws.cell(row=current_row, column=2, value=note)
|
||||
current_row += 1
|
||||
|
||||
# Add a blank row between groups
|
||||
current_row += 1
|
||||
|
||||
# Save to Excel
|
||||
from datetime import datetime, timedelta
|
||||
today = datetime.today()
|
||||
days_ahead = (7 - today.weekday()) % 7
|
||||
days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday
|
||||
next_monday = today + timedelta(days=days_ahead)
|
||||
|
||||
formatted = next_monday.strftime("Monday %d-%m-%Y")
|
||||
|
||||
|
||||
file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx"
|
||||
wb.save(file_name)
|
||||
output_path = os.path.abspath(file_name)
|
||||
sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA)
|
||||
sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name)
|
||||
|
|
@ -1,12 +1,16 @@
|
|||
import hubspot
|
||||
from enum import Enum
|
||||
from hubspot.crm.deals import PublicObjectSearchRequest
|
||||
from hubspot.crm.deals.models import SimplePublicObjectInput
|
||||
from etl.hubSpotClient.types import SubmissionInfoFromDeal
|
||||
|
||||
|
||||
|
||||
class DealStage(Enum):
|
||||
SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
|
||||
SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
|
||||
CUSTOMER_CONTACTED = "888730834"
|
||||
SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
|
||||
|
||||
class HubSpotClient():
|
||||
def __init__(self):
|
||||
|
|
@ -15,7 +19,85 @@ class HubSpotClient():
|
|||
|
||||
def get_all_deals(self):
|
||||
return self.client.crm.deals.get_all()
|
||||
|
||||
|
||||
def get_deal_name_by_id(self, deal_id):
|
||||
try:
|
||||
deal = self.client.crm.deals.basic_api.get_by_id(deal_id)
|
||||
return deal.properties.get("dealname", "No deal name")
|
||||
except Exception as e:
|
||||
return "Unknown Deal" # Fallback if the deal name is not found
|
||||
|
||||
def get_notes_from_deals_id(self, deals_id):
|
||||
from hubspot.crm.objects import PublicObjectSearchRequest
|
||||
found_notes = []
|
||||
after = None
|
||||
while True:
|
||||
# Correct filter for notes associated with the given deal ID
|
||||
search_request = PublicObjectSearchRequest(
|
||||
filter_groups=[{
|
||||
"filters": [{
|
||||
"propertyName": "associations.deal", # Filter by association to the deal
|
||||
"operator": "EQ",
|
||||
"value": deals_id,
|
||||
}]
|
||||
}],
|
||||
properties=["hs_note_body", "hubspot_owner_id"], # Properties of the note you need
|
||||
limit=200,
|
||||
after=after,
|
||||
)
|
||||
# Call the search API
|
||||
response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request)
|
||||
|
||||
# Add the results to the found_notes list
|
||||
found_notes.extend(response.results)
|
||||
|
||||
# Handle pagination if more results are available
|
||||
if not response.paging or not response.paging.next:
|
||||
break
|
||||
after = response.paging.next.after
|
||||
|
||||
all_notes = []
|
||||
for note in found_notes:
|
||||
# Extract note content and author information
|
||||
note_body = note.properties.get("hs_note_body", "No content")
|
||||
|
||||
# Collect note details in a dictionary
|
||||
all_notes.append({
|
||||
"note_id": note.id,
|
||||
"note": note_body,
|
||||
})
|
||||
return all_notes
|
||||
|
||||
|
||||
def get_all_deals_from_stage_id(self, stage_id):
|
||||
found_deals = []
|
||||
after = None
|
||||
while True:
|
||||
search_request = PublicObjectSearchRequest(
|
||||
filter_groups=[{
|
||||
"filters": [{
|
||||
"propertyName": "dealstage",
|
||||
"operator": "EQ",
|
||||
"value": stage_id,
|
||||
}]
|
||||
}],
|
||||
properties=[
|
||||
"dealname",
|
||||
],
|
||||
limit=200,
|
||||
after=after,
|
||||
)
|
||||
response = self.client.crm.deals.search_api.do_search(search_request)
|
||||
found_deals.extend(response.results)
|
||||
if not response.paging or not response.paging.next:
|
||||
break
|
||||
after = response.paging.next.after
|
||||
|
||||
all_deals = []
|
||||
for deal in found_deals:
|
||||
all_deals.append(deal.id)
|
||||
return all_deals
|
||||
|
||||
def get_deals_from_deal_stage(self, deal_stage: DealStage):
|
||||
found_deals = []
|
||||
|
|
@ -53,9 +135,9 @@ class HubSpotClient():
|
|||
deal_id= deal.properties["hs_object_id"],
|
||||
deal_name=deal.properties["dealname"],
|
||||
work_type=deal.properties["work_type"],
|
||||
needs_trickle_ventilation=True if deal.properties["property_needs_trickle_vents"].upper() == "YES" else False,
|
||||
needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False,
|
||||
post_sap_score=int(deal.properties["domna_survey_post_sap"]),
|
||||
existing_wall_insulation=deal.properties["existing_wall_insulation"],
|
||||
existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
|
||||
no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
|
||||
installer=deal.properties["installer"],
|
||||
))
|
||||
|
|
@ -67,4 +149,17 @@ class HubSpotClient():
|
|||
print(f"Pipeline: {pipeline.label}")
|
||||
for stage in pipeline.stages:
|
||||
print(f" - Label: {stage.label}")
|
||||
print(f" ID: {stage.id}") #
|
||||
print(f" ID: {stage.id}")
|
||||
|
||||
def move_deals_to_different_stage(self, list_of_deals_id, to_stage_id):
|
||||
deal_properties = SimplePublicObjectInput(
|
||||
properties={
|
||||
"dealstage": to_stage_id
|
||||
}
|
||||
)
|
||||
for deal_id in list_of_deals_id:
|
||||
self.client.crm.deals.basic_api.update(
|
||||
deal_id,
|
||||
simple_public_object_input=deal_properties
|
||||
)
|
||||
print(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")
|
||||
|
|
|
|||
41
etl/hubspot_to_invoice.py
Normal file
41
etl/hubspot_to_invoice.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import os
|
||||
|
||||
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
|
||||
from etl.surveyPrice.surveyPrice import SurveyPrice
|
||||
|
||||
|
||||
sp = SurveyPrice()
|
||||
|
||||
df = sp.calculate_all_price()
|
||||
|
||||
|
||||
verbose_file = "verbose_invoice_score.xlsx"
|
||||
df.to_excel(verbose_file, index=False)
|
||||
output_path = os.path.abspath(verbose_file)
|
||||
sp.upload_to_sharepoint(output_path, verbose_file)
|
||||
|
||||
lewis_view = "FOR_LEWIS.xlsx"
|
||||
selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"]
|
||||
minimal_df = df[selected_columns]
|
||||
minimal_df.to_excel(lewis_view, index=False)
|
||||
output_path = os.path.abspath(lewis_view)
|
||||
sp.upload_to_sharepoint(output_path, lewis_view)
|
||||
|
||||
|
||||
deal_ids = df["HUBSPOT_DEAL_ID"].tolist()
|
||||
|
||||
sp.move_deals_to_completed(deal_ids)
|
||||
|
||||
|
||||
# TODO:
|
||||
# Write documentation for tech demos from Khalims demo
|
||||
# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples
|
||||
# Figure out what to do if I see an address that isn't registered but surveyrod
|
||||
# Review deem score with last weeks deem score values to ensure accuracy
|
||||
|
||||
|
|
@ -11,7 +11,7 @@ import pandas as pd
|
|||
|
||||
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS)
|
||||
parent_folder = "Automated Example"
|
||||
osmosis.create_file(parent_folder, "/JTK Test Folder")
|
||||
osmosis.create_dir(parent_folder, "/JTK Test Folder")
|
||||
|
||||
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing")
|
||||
|
||||
|
|
@ -20,30 +20,30 @@ new_asset_list = []
|
|||
parent_folder = "JTK Test Folder/Automated Example"
|
||||
# Create asset list and location
|
||||
for index, address in asset_list.iterrows():
|
||||
webUrl = osmosis.create_file(address['Name'], parent_folder)
|
||||
webUrl = osmosis.create_dir(address['Name'], parent_folder)
|
||||
|
||||
first_folder = "1. Retrofit Assessment"
|
||||
osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}")
|
||||
osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}")
|
||||
osmosis.create_dir(first_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_dir("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}")
|
||||
osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}")
|
||||
|
||||
second_folder = "2. RC Mid-Term Plan"
|
||||
osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}")
|
||||
osmosis.create_dir(second_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_dir("SAP", parent_folder + f"/{address['Name']}/{second_folder}")
|
||||
|
||||
third_folder = "3. Retrofit Design"
|
||||
osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_dir(third_folder, parent_folder + f"/{address['Name']}")
|
||||
|
||||
fourth_folder = "4. Post EPC"
|
||||
osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_file(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}")
|
||||
osmosis.create_dir(fourth_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}")
|
||||
|
||||
fifth_folder = "5. Trustmark Lodgement"
|
||||
osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
osmosis.create_dir(fifth_folder, parent_folder + f"/{address['Name']}")
|
||||
osmosis.create_dir("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
|
||||
osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
osmosis.create_dir("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
osmosis.create_dir("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
|
||||
|
||||
asset_data = {
|
||||
"Name": address['Name'],
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ from datetime import datetime, timedelta
|
|||
def previous_monday():
|
||||
today = datetime.today()
|
||||
last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday
|
||||
# return f"W.C. 31.03.2025"
|
||||
return f"W.C. {last_monday.strftime('%d.%m.%Y')}"
|
||||
return f"W.C. 31.03.2025"
|
||||
# return f"W.C. {last_monday.strftime('%d.%m.%Y')}"
|
||||
|
||||
WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())
|
||||
|
||||
|
|
@ -27,6 +27,7 @@ class SharePointInstaller(Enum):
|
|||
BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2")
|
||||
DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4")
|
||||
OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f")
|
||||
WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4")
|
||||
|
||||
class SharePointScraper():
|
||||
"""
|
||||
|
|
@ -116,7 +117,7 @@ class SharePointScraper():
|
|||
return True
|
||||
return False
|
||||
|
||||
def create_file(self, file_name, at_path="/"):
|
||||
def create_dir(self, file_name, at_path="/"):
|
||||
|
||||
sharepoint_client = SharePointClient(
|
||||
tenant_id=self.sharepoint_tenant_id,
|
||||
|
|
@ -131,6 +132,19 @@ class SharePointScraper():
|
|||
for folders in self.get_folders_in_path(at_path)['value']:
|
||||
if file_name.upper() in folders["name"].upper():
|
||||
return folders["webUrl"]
|
||||
|
||||
def upload_file(self, file_path, sharepoint_path, file_name):
|
||||
sharepoint_client = SharePointClient(
|
||||
tenant_id=self.sharepoint_tenant_id,
|
||||
client_id=self.sharepoint_client_id,
|
||||
client_secret=self.sharepoint_client_secret,
|
||||
site_id=self.sharepoint_drive.value,
|
||||
)
|
||||
def get_file_stream(file_path):
|
||||
return open(file_path, 'rb')
|
||||
|
||||
sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path)
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -308,4 +322,5 @@ class SharePointScraper():
|
|||
temp_file.write(content.getvalue())
|
||||
|
||||
self.logger.info(f"Temporary file created at: {path}")
|
||||
return path
|
||||
return path
|
||||
|
||||
|
|
@ -1,25 +1,308 @@
|
|||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
|
||||
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class surveyPrice():
|
||||
class SurveyPrice():
|
||||
"""
|
||||
A class to work out all prices and uploads to sharepoint
|
||||
A class to work out all prices and uploads to sharepoint for review
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT)
|
||||
self.master_rate_card_path = None
|
||||
self.all_hubspot_submissions = None
|
||||
self.all_survey_info_from_sharepoint = None
|
||||
self.download_price_card()
|
||||
|
||||
self.required_sheets = [
|
||||
'JJC - EMPTIES',
|
||||
'JJC - GENERAL EXTRACTIONS',
|
||||
'JJC - FORMALDEHYDE EXTRACTION',
|
||||
'JJC - SOLAR',
|
||||
'SCIS - GENERAL EXTRACTIONS',
|
||||
'SCIS - EMPTIES',
|
||||
'SCIS - SOLAR',
|
||||
'SGEC - EMPTIES',
|
||||
'SGEC - GENERAL EXTRACTIONS',
|
||||
'SGEC - FORMALDEHYDE EXTRACTION'
|
||||
]
|
||||
|
||||
self.installer = {
|
||||
"J & J CRUMP": "JJC",
|
||||
"SCIS": "SCIS",
|
||||
"SGEC": "SGEC",
|
||||
"BAXTER KELLY": "BAXTER KELLY",
|
||||
}
|
||||
|
||||
self.domna_job_to_price_sheet_convertor = {
|
||||
"JJC - SOLAR": "JJC - SOLAR",
|
||||
"JJC - EMPTY CAVITY": "JJC - EMPTIES",
|
||||
"JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION",
|
||||
"JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS",
|
||||
"SCIS - SOLAR": "SCIS - SOLAR",
|
||||
"SCIS - EMPTY CAVITY": "SCIS - EMPTIES",
|
||||
"SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION",
|
||||
"SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS",
|
||||
"SGEC - EMPTY CAVITY": "SGEC - EMPTIES",
|
||||
"SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION",
|
||||
"SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS",
|
||||
}
|
||||
|
||||
def download_price_card(self):
|
||||
pass
|
||||
url = None
|
||||
# TODO: Some sanity checks to ensure rate cards title stays consistent
|
||||
for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']:
|
||||
if files['name'] == "MASTER RATE CARD.xlsx":
|
||||
url = files['@microsoft.graph.downloadUrl']
|
||||
break
|
||||
|
||||
def get_price_card():
|
||||
pass
|
||||
if url:
|
||||
content = self.sharepoint_client.get_file_content(url)
|
||||
self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx")
|
||||
|
||||
return self.master_rate_card_path
|
||||
|
||||
|
||||
def get_cavity_pricing_table(self, sheet_name):
|
||||
excel_file = pd.ExcelFile(self.master_rate_card_path)
|
||||
available_sheets = excel_file.sheet_names
|
||||
missing_sheets = [sheet for sheet in self.required_sheets if sheet not in available_sheets]
|
||||
|
||||
if missing_sheets:
|
||||
raise ValueError(f"Missing sheets: {missing_sheets}")
|
||||
|
||||
pricing_table = self.get_price_matrix(sheet_name)
|
||||
|
||||
return pricing_table
|
||||
|
||||
|
||||
def get_master_rate_card_path(self):
|
||||
return self.master_rate_card_path
|
||||
|
||||
def get_price_matrix(self, sheet_name):
|
||||
df = pd.read_excel(self.master_rate_card_path, sheet_name)
|
||||
|
||||
if "SOLAR" in sheet_name.upper():
|
||||
return df
|
||||
else:
|
||||
columns_to_check = {
|
||||
"no extractors or ventilation required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 0},
|
||||
"Trickle Vents ONLY": {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 0},
|
||||
"1 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 1},
|
||||
"2 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 2},
|
||||
"3 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 3},
|
||||
'Trickle Vents + 1 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 1},
|
||||
'Trickle Vents + 2 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 2},
|
||||
'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3},
|
||||
}
|
||||
|
||||
pricing_table = []
|
||||
for _, row in df.iterrows():
|
||||
for key, variables in columns_to_check.items():
|
||||
pricing_table.append(
|
||||
{
|
||||
"WORK TYPE": row["WORK TYPE"],
|
||||
"FLOOR_AREA_BANDING": row["Total Floor Area"][:-1],
|
||||
**variables,
|
||||
"PRICE": row[key] if row[key] != "Not viable" else None,
|
||||
}
|
||||
)
|
||||
pricing_table = pd.DataFrame(pricing_table)
|
||||
return pricing_table
|
||||
|
||||
def get_all_surveys_from_hubspot(self):
|
||||
hubSpotClient = HubSpotClient()
|
||||
deals = hubSpotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF)
|
||||
|
||||
all_deals = []
|
||||
|
||||
for deal in deals:
|
||||
all_deals.append({
|
||||
"HUBSPOT_DEAL_ID": deal.deal_id,
|
||||
"HUBSPOT_WORK_TYPE": deal.work_type,
|
||||
"HUBSPOT_DEAL_ADDRESS": deal.deal_name,
|
||||
"HUBSPOT_TRICKLE_VENT":1 if deal.needs_trickle_ventilation else 0,
|
||||
"HUBSPOT_WALL_INSULATION": deal.existing_wall_insulation,
|
||||
"HUBSPOT_POST_INSTALL_SAP_SCORE": deal.post_sap_score,
|
||||
"HUBSPOT_INSTALLER": deal.installer,
|
||||
"HUBSPOT_WETROOMS": deal.no_of_wet_rooms,
|
||||
})
|
||||
|
||||
self.all_hubspot_submissions = pd.DataFrame(all_deals)
|
||||
return self.all_hubspot_submissions
|
||||
|
||||
def move_deals_to_completed(self, deals):
|
||||
hubspot = HubSpotClient()
|
||||
hubspot.move_deals_to_different_stage(deals, DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value)
|
||||
|
||||
|
||||
def get_all_surveyed_data_from_sharepoint(self):
|
||||
# TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for
|
||||
# DIfferent installers
|
||||
self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc()
|
||||
return self.all_survey_info_from_sharepoint
|
||||
|
||||
def sharepoint_data_for_jjc(self):
|
||||
jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True)
|
||||
file_paths = jjc_sp.download_file_for_each_address()
|
||||
jjc_surveys = []
|
||||
|
||||
for eachAddress in file_paths:
|
||||
for address, files in eachAddress.items():
|
||||
jjc_surveys.append(surveyedDataProcessor(address, files))
|
||||
|
||||
all_survey_info = []
|
||||
for surveyInfo in jjc_surveys:
|
||||
cavity_wall_as_built = False
|
||||
csr = False
|
||||
foam_insulation = False
|
||||
|
||||
info = {
|
||||
"SHAREPOINT INSTALLER": "J & J CRUMP",
|
||||
"SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False,
|
||||
"SHAREPOINT CSR FOUND": True if surveyInfo.csr else False,
|
||||
"SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND",
|
||||
"SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND",
|
||||
"SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND",
|
||||
"SHAREPOINT INSULATION MATERIAL": None,
|
||||
"SHAREPOINT ADDRESS": address
|
||||
}
|
||||
|
||||
if surveyInfo.pre_site_note:
|
||||
floor_banding, total_floor_area = surveyInfo.work_out_total_floor_area()
|
||||
pre_sap_score = surveyInfo.get_current_sap_score()
|
||||
pre_sap_score_banding = surveyedDataProcessor.get_band(pre_sap_score)
|
||||
info.update({
|
||||
"SHAREPOINT TOTAL_FLOOR_AREA": total_floor_area,
|
||||
"SHAREPOINT FLOOR_AREA_BANDING": floor_banding,
|
||||
"SHAREPOINT PRE_INSTALL_SAP_SCORE": pre_sap_score,
|
||||
"SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING": pre_sap_score_banding,
|
||||
})
|
||||
|
||||
if surveyInfo.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \
|
||||
and surveyInfo.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall":
|
||||
cavity_wall_as_built = True
|
||||
|
||||
if surveyInfo.csr:
|
||||
csr = True
|
||||
insulation = surveyInfo.get_insulation_info()
|
||||
info.update({
|
||||
"SHAREPOINT INSULATION MATERIAL": insulation,
|
||||
})
|
||||
if "FOAM" in insulation.upper():
|
||||
foam_insulation = True
|
||||
|
||||
if cavity_wall_as_built:
|
||||
if csr:
|
||||
if foam_insulation:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "REMIDIAL FOAM FILLED CAVITY",
|
||||
})
|
||||
else:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "REMIDIAL FILLED CAVITY"
|
||||
})
|
||||
else:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "EMPTY CAVITY"
|
||||
})
|
||||
else:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "SOLAR"
|
||||
})
|
||||
|
||||
|
||||
all_survey_info.append(info)
|
||||
|
||||
# Step one
|
||||
# Make a copy of the rate card and work out each price matrix
|
||||
# Make the price card downloadable as a single excelt sheet for viewing
|
||||
# For other things, do some TDD so it is a little robust
|
||||
# The script can run weekly, for development I can just get one data
|
||||
# Expected input, expect output etc
|
||||
return pd.DataFrame(all_survey_info)
|
||||
|
||||
def merge_hub_spot_and_survey_information(self):
|
||||
if self.all_survey_info_from_sharepoint is None:
|
||||
raise RuntimeError("No survey information found from Sharepoint")
|
||||
if self.all_hubspot_submissions is None:
|
||||
raise RuntimeError("No information found from Hubspot")
|
||||
|
||||
# Standardise address
|
||||
self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply(
|
||||
lambda x: x.lower().replace(',', '').strip()
|
||||
)
|
||||
|
||||
self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply(
|
||||
lambda x: x.lower().replace(',', '').strip()
|
||||
)
|
||||
|
||||
# re-name to installer
|
||||
self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename(
|
||||
columns={
|
||||
'SHAREPOINT INSTALLER': 'INSTALLER',
|
||||
'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING',
|
||||
}
|
||||
)
|
||||
|
||||
self.all_hubspot_submissions = self.all_hubspot_submissions.rename(
|
||||
columns={
|
||||
'HUBSPOT_INSTALLER': 'INSTALLER',
|
||||
'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS',
|
||||
'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT',
|
||||
}
|
||||
)
|
||||
|
||||
merged_df = pd.merge(
|
||||
self.all_survey_info_from_sharepoint,
|
||||
self.all_hubspot_submissions,
|
||||
left_on=['clean_address', 'INSTALLER'],
|
||||
right_on=['clean_address', 'INSTALLER'],
|
||||
how='inner'
|
||||
)
|
||||
|
||||
merged_df.drop(columns=['clean_address'], inplace=True)
|
||||
|
||||
def compute_energy_grant(row):
|
||||
pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1]
|
||||
post_band_letter = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"])[-1]
|
||||
return surveyedDataProcessor.gbis_or_eco4_scheme(pre_band_letter, post_band_letter)
|
||||
|
||||
def compute_banding_for_post_sap(row):
|
||||
post_sap_banding = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"])
|
||||
return post_sap_banding
|
||||
|
||||
def work_type(row):
|
||||
if row["ENERGY_GRANT"] == "GBIS":
|
||||
return row["ENERGY GRANT"]
|
||||
else:
|
||||
return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}"
|
||||
|
||||
|
||||
# Add missing variables
|
||||
merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1)
|
||||
merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1)
|
||||
merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1)
|
||||
|
||||
return merged_df
|
||||
|
||||
def calculate_all_price(self):
|
||||
self.get_all_surveys_from_hubspot()
|
||||
self.get_all_surveyed_data_from_sharepoint()
|
||||
submission_data = self.merge_hub_spot_and_survey_information()
|
||||
final_list = []
|
||||
for _, row in submission_data.iterrows():
|
||||
if "SOLAR" in row["DOMNA JOB TYPE"].upper():
|
||||
raise NotImplementedError("Please implement solar pricing")
|
||||
else:
|
||||
# Cavity wall
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
price_matrix = self.get_price_matrix(sheet_name)
|
||||
merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left')
|
||||
final_list.append(merged_row)
|
||||
|
||||
return pd.concat(final_list, ignore_index=True)
|
||||
|
||||
def upload_to_sharepoint(self, file_path_to_upload, file_name):
|
||||
parent_folder = "Commercials/Rate Cards"
|
||||
date = previous_monday()
|
||||
self.sharepoint_client.create_dir(date, parent_folder)
|
||||
|
||||
sharepoint_path = parent_folder + "/" + date
|
||||
|
||||
self.sharepoint_client.upload_file(file_path_to_upload, sharepoint_path, file_name)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from etl.pdfReader.pdfReaderToText import pdfReaderToText
|
||||
from etl.pdfReader.reportType import ReportType
|
||||
import math
|
||||
|
||||
class surveyedDataProcessor():
|
||||
def __init__(self, address, files):
|
||||
|
|
@ -17,5 +18,130 @@ class surveyedDataProcessor():
|
|||
if pdf:
|
||||
if pdf.type == ReportType.QUIDOS_PRESITE_NOTE:
|
||||
self.pre_site_note = pdf.get_reader()
|
||||
self.address = self.pre_site_note.survey_information.address
|
||||
elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT:
|
||||
self.csr = pdf.get_reader()
|
||||
|
||||
def get_insulation_info(self):
|
||||
if self.csr:
|
||||
if self.csr.insulation_info:
|
||||
insultation = self.csr.insulation_info.type.upper()
|
||||
return insultation
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_band(sap_score_number):
|
||||
bands = [
|
||||
("HIGH A", 96, float("inf")),
|
||||
("LOW A", 92, 96),
|
||||
("HIGH B", 86, 92),
|
||||
("LOW B", 81, 86),
|
||||
("HIGH C", 74.5, 81),
|
||||
("LOW C", 69, 74.5),
|
||||
("HIGH D", 61.5, 69),
|
||||
("LOW D", 55, 61.5),
|
||||
("HIGH E", 46.5, 55),
|
||||
("LOW E", 39, 46.5),
|
||||
("HIGH F", 29.5, 39),
|
||||
("LOW F", 21, 29.5),
|
||||
("HIGH G", 10.5, 21),
|
||||
("LOW G", 1, 10.5),
|
||||
]
|
||||
|
||||
for band, lower, upper in bands:
|
||||
if lower <= sap_score_number < upper:
|
||||
return band
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def gbis_or_eco4_scheme(presap_letter, postsap_letter):
|
||||
"""
|
||||
*ECO4 (minimum movement)*
|
||||
D to C
|
||||
E to C
|
||||
F to D
|
||||
G to D
|
||||
|
||||
G -> ABCD
|
||||
F -> ABCD
|
||||
E -> ABC
|
||||
D -> ABC
|
||||
|
||||
*GBIS - Cavity Wall Insulation ONLY*
|
||||
|
||||
D-D
|
||||
E-D
|
||||
E-E
|
||||
F-E
|
||||
F-F
|
||||
G-E
|
||||
G-F
|
||||
G-G
|
||||
"""
|
||||
eco4 = {
|
||||
"G": ['A', 'B', 'C', 'D'],
|
||||
"F": ['A', 'B', 'C', 'D'],
|
||||
"E": ['A', 'B', 'C'],
|
||||
"D": ['A', 'B', 'C'],
|
||||
}
|
||||
|
||||
gbis ={
|
||||
'D': ['D'],
|
||||
'E': ['E', 'D'],
|
||||
'F': ['E', 'F'],
|
||||
'G': ['E', 'F', 'G'],
|
||||
}
|
||||
|
||||
if presap_letter.upper() in eco4:
|
||||
if postsap_letter.upper() in eco4[presap_letter.upper()]:
|
||||
return "ECO4"
|
||||
if presap_letter.upper() in gbis:
|
||||
if postsap_letter.upper() in gbis[presap_letter.upper()]:
|
||||
return "GBIS"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
|
||||
def work_out_total_floor_area(self):
|
||||
|
||||
total = 0
|
||||
def add_all_floors(floor_list):
|
||||
total = 0
|
||||
for floor in floor_list:
|
||||
total += floor.floor_area_m2
|
||||
|
||||
return total
|
||||
|
||||
main = True if self.pre_site_note.property_description.no_of_main_property > 0 else False
|
||||
ext1 = True if self.pre_site_note.property_description.no_of_extension_1 > 0 else False
|
||||
ext2 = True if self.pre_site_note.property_description.no_of_extension_2 > 0 else False
|
||||
ext3 = True if self.pre_site_note.property_description.no_of_extension_3 > 0 else False
|
||||
ext4 = True if self.pre_site_note.property_description.no_of_extension_4 > 0 else False
|
||||
|
||||
total += add_all_floors(self.pre_site_note.property_description.main_property.dimensions) if main is True else 0
|
||||
total += add_all_floors(self.pre_site_note.property_description.ex1_property.dimensions) if ext1 is True else 0
|
||||
total += add_all_floors(self.pre_site_note.property_description.ex2_property.dimensions) if ext2 is True else 0
|
||||
total += add_all_floors(self.pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0
|
||||
total += add_all_floors(self.pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0
|
||||
|
||||
|
||||
floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total)
|
||||
if 0 <= floor_area <= 72:
|
||||
return '0-72m', floor_area
|
||||
elif 72 < floor_area <= 97:
|
||||
return '73-97m', floor_area
|
||||
elif 97 < floor_area <= 199:
|
||||
return '98-199m', floor_area
|
||||
elif 199 <= floor_area:
|
||||
return 'over 200m', floor_area
|
||||
else:
|
||||
raise NotImplementedError(f"unknown floor area {floor_area} {self.pre_site_note.summary_information.address}")
|
||||
|
||||
def get_current_sap_score(self):
|
||||
score_list = self.pre_site_note.survey_information.current_sap.split(" ")
|
||||
score = int(score_list[1])
|
||||
return score
|
||||
|
||||
|
||||
|
|
|
|||
37
etl/tests/test_survey_price.py
Normal file
37
etl/tests/test_survey_price.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import os
|
||||
# WarmFront Sharepoint KEYS
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
|
||||
from etl.surveyPrice.surveyPrice import SurveyPrice
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def sp():
|
||||
return SurveyPrice()
|
||||
|
||||
|
||||
def cavity_price_dataframe_sanity_check(df):
|
||||
assert df.shape == (160, 5)
|
||||
assert df.columns.tolist() == ['WORK TYPE', 'Floor Area Group', 'Trickle Vent', 'Wetrooms', 'PRICE']
|
||||
|
||||
def test_get_price_matrix_jjc_empties(sp):
|
||||
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - EMPTIES")
|
||||
cavity_price_dataframe_sanity_check(jjc_empties_price_table)
|
||||
|
||||
def test_get_price_matrix_jjc_general_extraction(sp):
|
||||
sp = SurveyPrice()
|
||||
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - GENERAL EXTRACTIONS")
|
||||
cavity_price_dataframe_sanity_check(jjc_empties_price_table)
|
||||
|
||||
def test_get_price_matrix_jjc_foam(sp):
|
||||
sp = SurveyPrice()
|
||||
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - FORMALDEHYDE EXTRACTION")
|
||||
cavity_price_dataframe_sanity_check(jjc_empties_price_table)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -203,5 +203,11 @@ class PropertyDescription(BaseModel):
|
|||
mainHeating2: Optional[Heating]
|
||||
secondaryHeatingType: Optional[HeatingType]
|
||||
|
||||
# class PropertyReport():
|
||||
# TODO: Property description
|
||||
# TODO: Due consideration foregin key
|
||||
# TODO: Which company did it (Osmosis, Warmfront etc)
|
||||
# TODO: Links to more foreign keys per report etc
|
||||
|
||||
class Insulation(BaseModel):
|
||||
type: str
|
||||
|
|
@ -274,6 +274,29 @@ class SharePointClient:
|
|||
url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children"
|
||||
|
||||
return 'POST', url, data
|
||||
|
||||
def upload_file(self, file_name, file_stream, sharepoint_parent_id):
|
||||
"""
|
||||
Uploads a file to SharePoint using the Graph API.
|
||||
PUT /drives/{drive-id}/root:/{path-to-file}:/content
|
||||
|
||||
:param file_name: Name of the file to upload
|
||||
:param sharepoint_path: Path within the SharePoint site (folder path)
|
||||
:param file_stream: File content as a binary stream (e.g., BytesIO or open(file, 'rb'))
|
||||
:return: Response JSON from the API
|
||||
"""
|
||||
url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content"
|
||||
logger.debug(f"Uploading file to URL: {url}")
|
||||
|
||||
response = requests.put(url, headers=self.headers, data=file_stream)
|
||||
|
||||
if response.status_code in (200, 201):
|
||||
logger.info(f"File '{file_name}' uploaded successfully.")
|
||||
return response.json()
|
||||
else:
|
||||
retry = handle_error(response)
|
||||
if retry == 'retry':
|
||||
return self.upload_file(file_name, sharepoint_parent_id, file_stream)
|
||||
|
||||
@staticmethod
|
||||
def download_sharepoint_file(download_url):
|
||||
|
|
|
|||
37
poetry.lock
generated
37
poetry.lock
generated
|
|
@ -61,6 +61,29 @@ files = [
|
|||
astroid = ["astroid (>=2,<4)"]
|
||||
test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"]
|
||||
|
||||
[[package]]
|
||||
name = "beautifulsoup4"
|
||||
version = "4.13.4"
|
||||
description = "Screen-scraping library"
|
||||
optional = false
|
||||
python-versions = ">=3.7.0"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"},
|
||||
{file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
soupsieve = ">1.2"
|
||||
typing-extensions = ">=4.0.0"
|
||||
|
||||
[package.extras]
|
||||
cchardet = ["cchardet"]
|
||||
chardet = ["chardet"]
|
||||
charset-normalizer = ["charset-normalizer"]
|
||||
html5lib = ["html5lib"]
|
||||
lxml = ["lxml"]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2025.1.31"
|
||||
|
|
@ -1684,6 +1707,18 @@ files = [
|
|||
{file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.6"
|
||||
description = "A modern CSS selector implementation for Beautiful Soup."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"},
|
||||
{file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlalchemy"
|
||||
version = "2.0.40"
|
||||
|
|
@ -1925,4 +1960,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.12"
|
||||
content-hash = "55a974b3a81d57c429f61ee6a12a84d38f5c703fdfdfdf2553bec6ba21c29bf5"
|
||||
content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ dependencies = [
|
|||
"pytest (>=8.3.5,<9.0.0)",
|
||||
"hubspot-api-client (>=11.1.0,<12.0.0)",
|
||||
"monday (>=2.0.1,<3.0.0)",
|
||||
"beautifulsoup4 (>=4.13.4,<5.0.0)",
|
||||
]
|
||||
|
||||
[tool.poetry]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue