From f771fee1e00413723c3a11e178add6d61053235d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Apr 2025 16:52:17 +0000 Subject: [PATCH] survey completed --- etl/hubspot_to_invoice.py | 10 ++++--- etl/scraper/scraper.py | 7 ++--- etl/surveyPrice/surveyPrice.py | 49 +++++++++++++++++++++++++--------- poetry.lock | 26 ++++++++++++++++-- pyproject.toml | 1 + 5 files changed, 73 insertions(+), 20 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 7413e49..eca4271 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -52,9 +52,9 @@ sp.move_deals_to_completed(deal_ids) # 3 examples Cavity Wall, FOAM, Empty and General ideally # (in hubspot )111 Duddell Road General ( fibre) - 500, 2 wet rooms # Empty -# ( in hubspot ) 29 Lower King ( empty ) - 500 +# ( in hubspot ) 29 Lower King ( empty ) - 500 - 400 # Foam -# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 +# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200 # SCIS # 3 examples of Solar @@ -71,4 +71,8 @@ sp.move_deals_to_completed(deal_ids) # Change w.c. date to a weird one to speed up automation -value = df[df["SHAREPOINT ADDRESS"] == "29 Lower King, BRAINTREE, CM7 3XZ"] \ No newline at end of file + +# Observation: +""" +2 queensway is wrong due the fact that csr and empty cavity but deem score says cavity +""" \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index a379894..64d5750 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -8,6 +8,7 @@ from etl.utils.sharepoint.sharepoint import SharePointClient from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator +from tqdm import tqdm from datetime import datetime, timedelta @@ -153,7 +154,7 @@ class SharePointScraper(): @ensure_surveyor_names_loaded def get_date_folder_names(self): - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): dates_folders = self.get_folders_in_path(f"/{name}") if 'value' not in dates_folders: raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}") @@ -233,7 +234,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def get_number_of_surverys_completed(self): - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): if name in self.surveyor_to_housing_assosications: for house_ass in self.surveyor_to_housing_assosications[name]: address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") @@ -272,7 +273,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def download_file_for_each_address(self): paths = [] - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]: for house_ass in self.surveyor_to_housing_assosications[name]: address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index c58dd8b..a5190a5 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -2,6 +2,7 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous from etl.hubSpotClient.hubspot import HubSpotClient, DealStage from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd +from tqdm import tqdm class SurveyPrice(): @@ -35,18 +36,33 @@ class SurveyPrice(): "BAXTER KELLY": "BAXTER KELLY", } - self.domna_job_to_price_sheet_convertor = { + self.hubspot_job_to_price_sheet_convertor = { + # JJC "JJC - ECO4 PV": "JJC - SOLAR", - "JJC - EMPTY CAVITY": "JJC - EMPTIES", - "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION", - "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS", + "JJC - ECO4 CWI EMPTY": "JJC - EMPTIES", + "JJC - GBIS CWI EMPTY": "JJC - EMPTIES", + "JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + "JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + + # SCIS "SCIS - ECO4 PV": "SCIS - SOLAR", - "SCIS - EMPTY CAVITY": "SCIS - EMPTIES", - "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION", - "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS", - "SGEC - EMPTY CAVITY": "SGEC - EMPTIES", - "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION", - "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS", + "SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES", + "SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES", + "SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", + "SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + "SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", + "SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + + # SGEC + "SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES", + "SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES", + + "SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", + "SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", } def download_price_card(self): @@ -271,6 +287,8 @@ class SurveyPrice(): how='inner' ) + # if hubspot detects + merged_df.drop(columns=['address_start', 'postcode'], inplace=True) @@ -306,7 +324,7 @@ class SurveyPrice(): final_list = [] for _, row in submission_data.iterrows(): if "PV" in row["HUBSPOT_WORK_TYPE"].upper(): - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}" + sheet_name = f"{self.hubspot_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}" price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge( row.to_frame().T, @@ -317,7 +335,14 @@ class SurveyPrice(): ) else: # Cavity wall - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}' + if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER": + sheet_name += " - GENERAL" + elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY": + pass + else: + sheet_name += " - FOAM" + sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name] price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') final_list.append(merged_row) diff --git a/poetry.lock b/poetry.lock index ddd2ad3..0f1cdd6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -286,11 +286,11 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main", "dev"] -markers = "sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""} [[package]] name = "comm" @@ -1872,6 +1872,28 @@ files = [ {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "traitlets" version = "5.14.3" @@ -1960,4 +1982,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2" +content-hash = "b5221708d5a15633f7272103bf12970d3da3b05f5861b3e6f3fdfd2b42d8ddad" diff --git a/pyproject.toml b/pyproject.toml index c26faba..096377a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "hubspot-api-client (>=11.1.0,<12.0.0)", "monday (>=2.0.1,<3.0.0)", "beautifulsoup4 (>=4.13.4,<5.0.0)", + "tqdm (>=4.67.1,<5.0.0)", ] [tool.poetry]