survey completed

2026-06-30 13:10:56 +00:00 · 2025-04-23 16:52:17 +00:00 · 2025-04-23 16:52:17 +00:00 · f771fee1e0
commit f771fee1e0
parent ba521fb004
5 changed files with 73 additions and 20 deletions
--- a/etl/hubspot_to_invoice.py
+++ b/etl/hubspot_to_invoice.py
@ -52,9 +52,9 @@ sp.move_deals_to_completed(deal_ids)
 # 3 examples Cavity Wall, FOAM, Empty and General ideally
 # (in hubspot )111 Duddell Road General ( fibre) - 500, 2 wet rooms
 # Empty
-# ( in hubspot ) 29  Lower King ( empty ) - 500
+# ( in hubspot ) 29  Lower King ( empty ) - 500 - 400
 # Foam
-# ( in hubspot ) 6 STOKESAY STREET (foam) - 400
+# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200

 # SCIS
 # 3 examples of Solar
@ -71,4 +71,8 @@ sp.move_deals_to_completed(deal_ids)

 # Change w.c. date to a weird one to speed up automation

-value = df[df["SHAREPOINT ADDRESS"] == "29 Lower King, BRAINTREE, CM7 3XZ"]
+
+# Observation:
+"""
+2 queensway is wrong due the fact that csr and empty cavity but deem score says cavity
+"""
--- a/etl/scraper/scraper.py
+++ b/etl/scraper/scraper.py
@ -8,6 +8,7 @@ from etl.utils.sharepoint.sharepoint import SharePointClient
 from functools import wraps
 import re
 from etl.validator.validator import DomnaSharePointValidator
+from tqdm import tqdm

 from datetime import datetime, timedelta

@ -153,7 +154,7 @@ class SharePointScraper():

    @ensure_surveyor_names_loaded
    def get_date_folder_names(self):
-        for name in self.surveyor_names:
+        for name in tqdm(self.surveyor_names):
            dates_folders = self.get_folders_in_path(f"/{name}")
            if 'value' not in dates_folders:
                raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}")
@ -233,7 +234,7 @@ class SharePointScraper():
    
    @ensure_housing_assosiation_is_loaded
    def get_number_of_surverys_completed(self):
-        for name in self.surveyor_names:
+        for name in tqdm(self.surveyor_names):
            if name in self.surveyor_to_housing_assosications:
                for house_ass in self.surveyor_to_housing_assosications[name]:
                    address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")
@ -272,7 +273,7 @@ class SharePointScraper():
    @ensure_housing_assosiation_is_loaded
    def download_file_for_each_address(self):
        paths = []
-        for name in self.surveyor_names:
+        for name in tqdm(self.surveyor_names):
            if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]:
                for house_ass in self.surveyor_to_housing_assosications[name]:
                    address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")
--- a/etl/surveyPrice/surveyPrice.py
+++ b/etl/surveyPrice/surveyPrice.py
@ -2,6 +2,7 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous
 from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
 from etl.surveyedData.surveryedData import surveyedDataProcessor
 import pandas as pd
+from tqdm import tqdm


 class SurveyPrice():
@ -35,18 +36,33 @@ class SurveyPrice():
            "BAXTER KELLY": "BAXTER KELLY",
        }

-        self.domna_job_to_price_sheet_convertor = {
+        self.hubspot_job_to_price_sheet_convertor = {
+            # JJC
            "JJC - ECO4 PV": "JJC - SOLAR",
-            "JJC - EMPTY CAVITY": "JJC - EMPTIES",
-            "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION",
-            "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS",
+            "JJC - ECO4 CWI EMPTY": "JJC - EMPTIES",
+            "JJC - GBIS CWI EMPTY": "JJC - EMPTIES",
+            "JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
+            "JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
+            "JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
+            "JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
+
+            # SCIS
            "SCIS - ECO4 PV": "SCIS - SOLAR",
-            "SCIS - EMPTY CAVITY": "SCIS - EMPTIES",
-            "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION",
-            "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS",
-            "SGEC - EMPTY CAVITY": "SGEC - EMPTIES",
-            "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION",
-            "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS",
+            "SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES",
+            "SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES",
+            "SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
+            "SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
+            "SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
+            "SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
+
+            # SGEC
+            "SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES",
+            "SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES",
+
+            "SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
+            "SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
+            "SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
+            "SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
        }

    def download_price_card(self):
@ -271,6 +287,8 @@ class SurveyPrice():
            how='inner'
        )

+        # if hubspot detects 
+
        merged_df.drop(columns=['address_start', 'postcode'], inplace=True)


@ -306,7 +324,7 @@ class SurveyPrice():
        final_list = []
        for _, row in submission_data.iterrows():
            if "PV" in row["HUBSPOT_WORK_TYPE"].upper():
-                sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}"
+                sheet_name = f"{self.hubspot_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}"
                price_matrix = self.get_price_matrix(sheet_name)
                merged_row = pd.merge(
                    row.to_frame().T,
@ -317,7 +335,14 @@ class SurveyPrice():
                )               
            else:
                # Cavity wall
-                sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
+                sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}'
+                if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER":
+                    sheet_name += " - GENERAL"
+                elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY":
+                    pass
+                else:
+                    sheet_name += " - FOAM"
+                sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name]
                price_matrix = self.get_price_matrix(sheet_name)
                merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left')               
            final_list.append(merged_row)
--- a/poetry.lock
+++ b/poetry.lock
@ -286,11 +286,11 @@ description = "Cross-platform colored terminal text."
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 groups = ["main", "dev"]
-markers = "sys_platform == \"win32\""
 files = [
    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
+markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""}

 [[package]]
 name = "comm"
@ -1872,6 +1872,28 @@ files = [
    {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"},
 ]

+[[package]]
+name = "tqdm"
+version = "4.67.1"
+description = "Fast, Extensible Progress Meter"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"},
+    {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"]
+discord = ["requests"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
 [[package]]
 name = "traitlets"
 version = "5.14.3"
@ -1960,4 +1982,4 @@ files = [
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.12"
-content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2"
+content-hash = "b5221708d5a15633f7272103bf12970d3da3b05f5861b3e6f3fdfd2b42d8ddad"
--- a/pyproject.toml
+++ b/pyproject.toml
@ -22,6 +22,7 @@ dependencies = [
    "hubspot-api-client (>=11.1.0,<12.0.0)",
    "monday (>=2.0.1,<3.0.0)",
    "beautifulsoup4 (>=4.13.4,<5.0.0)",
+    "tqdm (>=4.67.1,<5.0.0)",
 ]

 [tool.poetry]