junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

deem scroe values updates

Browse source
This commit is contained in:
Jun-te Kim 2025-04-25 11:40:50 +00:00
parent f05222ae5c
commit 6de1177e64
4 changed files with 13 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.vscode/settings.json vendored
View file

@ -1,9 +1,7 @@
{
"jupyter.interactiveWindow.textEditor.executeSelection": true,
"python.REPL.sendToNativeREPL": true,
"notebook.output.scrolling": true,
"notebook.output.textLineLimit": 0
"notebook.output.scrolling": true
// Hot reload setting that needs to be in user settings
// "jupyter.runStartupCommands": [

2
etl/hubspot_to_invoice.py
View file

@ -54,7 +54,7 @@ sp.move_deals_to_completed(deal_ids)
# Empty
# ( in hubspot ) 29 Lower King ( empty ) - 500 - 400
# Foam
# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200
# ( in hubspot ) 6 STOKESAY STREET (foam) - 400
# SCIS
# 3 examples of Solar

7
etl/scraper/scraper.py
View file

@ -8,7 +8,6 @@ from etl.utils.sharepoint.sharepoint import SharePointClient
from functools import wraps
import re
from etl.validator.validator import DomnaSharePointValidator
from tqdm import tqdm
from datetime import datetime, timedelta
@ -154,7 +153,7 @@ class SharePointScraper():
@ensure_surveyor_names_loaded
def get_date_folder_names(self):
for name in tqdm(self.surveyor_names):
for name in self.surveyor_names:
dates_folders = self.get_folders_in_path(f"/{name}")
if 'value' not in dates_folders:
raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}")
@ -234,7 +233,7 @@ class SharePointScraper():
@ensure_housing_assosiation_is_loaded
def get_number_of_surverys_completed(self):
for name in tqdm(self.surveyor_names):
for name in self.surveyor_names:
if name in self.surveyor_to_housing_assosications:
for house_ass in self.surveyor_to_housing_assosications[name]:
address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")
@ -273,7 +272,7 @@ class SharePointScraper():
@ensure_housing_assosiation_is_loaded
def download_file_for_each_address(self):
paths = []
for name in tqdm(self.surveyor_names):
for name in self.surveyor_names:
if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]:
for house_ass in self.surveyor_to_housing_assosications[name]:
address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")

20
etl/surveyPrice/surveyPrice.py
View file

@ -42,27 +42,27 @@ class SurveyPrice():
"JJC - ECO4 CWI EMPTY": "JJC - EMPTIES",
"JJC - GBIS CWI EMPTY": "JJC - EMPTIES",
"JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
"JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
"JJC - ECO4 CWI REMEDIAL": "JJC - GENERAL EXTRACTIONS",
"JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
"JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
"JJC - GBIS CWI REMEDIAL": "JJC - GENERAL EXTRACTIONS",
# SCIS
"SCIS - ECO4 PV": "SCIS - SOLAR",
"SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES",
"SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES",
"SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
"SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
"SCIS - ECO4 CWI REMEDIAL": "SCIS - GENERAL EXTRACTIONS",
"SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
"SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
"SCIS - GBIS CWI REMEDIAL": "SCIS - GENERAL EXTRACTIONS",
# SGEC
"SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES",
"SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES",
"SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
"SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
"SGEC - ECO4 CWI REMEDIAL": "SGEC - GENERAL EXTRACTIONS",
"SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
"SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
"SGEC - GBIS CWI REMEDIAL": "SGEC - GENERAL EXTRACTIONS",
}
def download_price_card(self):
@ -172,7 +172,7 @@ class SurveyPrice():
file_paths = sp.download_file_for_each_address()
surveys = []
for eachAddress in file_paths:
for eachAddress in tqdm(file_paths):
for address, files in eachAddress.items():
surveys.append(surveyedDataProcessor(address, files))
@ -336,11 +336,7 @@ class SurveyPrice():
else:
# Cavity wall
sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}'
if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER":
sheet_name += " - GENERAL"
elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY":
pass
else:
if row['HUBSPOT_WALL_INSULATION'].upper() == "FORMALDEHYDE UFFI FOAM":
sheet_name += " - FOAM"
sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name]
price_matrix = self.get_price_matrix(sheet_name)