From 6de1177e64b64f16be8ced2ef273c9cc6a4d59d1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 25 Apr 2025 11:40:50 +0000 Subject: [PATCH] deem scroe values updates --- .vscode/settings.json | 4 +--- etl/hubspot_to_invoice.py | 2 +- etl/scraper/scraper.py | 7 +++---- etl/surveyPrice/surveyPrice.py | 20 ++++++++------------ 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9868a02..49f0804 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,7 @@ { "jupyter.interactiveWindow.textEditor.executeSelection": true, "python.REPL.sendToNativeREPL": true, - "notebook.output.scrolling": true, - "notebook.output.textLineLimit": 0 - + "notebook.output.scrolling": true // Hot reload setting that needs to be in user settings // "jupyter.runStartupCommands": [ diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index eca4271..d969c93 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -54,7 +54,7 @@ sp.move_deals_to_completed(deal_ids) # Empty # ( in hubspot ) 29 Lower King ( empty ) - 500 - 400 # Foam -# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200 +# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 # SCIS # 3 examples of Solar diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 64d5750..a379894 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -8,7 +8,6 @@ from etl.utils.sharepoint.sharepoint import SharePointClient from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator -from tqdm import tqdm from datetime import datetime, timedelta @@ -154,7 +153,7 @@ class SharePointScraper(): @ensure_surveyor_names_loaded def get_date_folder_names(self): - for name in tqdm(self.surveyor_names): + for name in self.surveyor_names: dates_folders = self.get_folders_in_path(f"/{name}") if 'value' not in dates_folders: raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}") @@ -234,7 +233,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def get_number_of_surverys_completed(self): - for name in tqdm(self.surveyor_names): + for name in self.surveyor_names: if name in self.surveyor_to_housing_assosications: for house_ass in self.surveyor_to_housing_assosications[name]: address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") @@ -273,7 +272,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def download_file_for_each_address(self): paths = [] - for name in tqdm(self.surveyor_names): + for name in self.surveyor_names: if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]: for house_ass in self.surveyor_to_housing_assosications[name]: address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index a5190a5..f54da9b 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -42,27 +42,27 @@ class SurveyPrice(): "JJC - ECO4 CWI EMPTY": "JJC - EMPTIES", "JJC - GBIS CWI EMPTY": "JJC - EMPTIES", "JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", - "JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + "JJC - ECO4 CWI REMEDIAL": "JJC - GENERAL EXTRACTIONS", "JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", - "JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + "JJC - GBIS CWI REMEDIAL": "JJC - GENERAL EXTRACTIONS", # SCIS "SCIS - ECO4 PV": "SCIS - SOLAR", "SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES", "SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES", "SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", - "SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + "SCIS - ECO4 CWI REMEDIAL": "SCIS - GENERAL EXTRACTIONS", "SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", - "SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + "SCIS - GBIS CWI REMEDIAL": "SCIS - GENERAL EXTRACTIONS", # SGEC "SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES", "SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES", "SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", - "SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", + "SGEC - ECO4 CWI REMEDIAL": "SGEC - GENERAL EXTRACTIONS", "SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", - "SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", + "SGEC - GBIS CWI REMEDIAL": "SGEC - GENERAL EXTRACTIONS", } def download_price_card(self): @@ -172,7 +172,7 @@ class SurveyPrice(): file_paths = sp.download_file_for_each_address() surveys = [] - for eachAddress in file_paths: + for eachAddress in tqdm(file_paths): for address, files in eachAddress.items(): surveys.append(surveyedDataProcessor(address, files)) @@ -336,11 +336,7 @@ class SurveyPrice(): else: # Cavity wall sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}' - if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER": - sheet_name += " - GENERAL" - elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY": - pass - else: + if row['HUBSPOT_WALL_INSULATION'].upper() == "FORMALDEHYDE UFFI FOAM": sheet_name += " - FOAM" sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name] price_matrix = self.get_price_matrix(sheet_name)