diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index a3be877..fbf490f 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -32,12 +32,19 @@ deal_ids = df["HUBSPOT_DEAL_ID"].tolist() sp.move_deals_to_completed(deal_ids) +""" + TODO: + P3) Improve dimirtra script by adding dates to the mixA + # Add dates + # Add owner's name if possible ( might need to do something to add info in hubspot mnaually) + # Add value information + # All notes in a particular order + # Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes + P3 Check to see if emails has arrived + P3) Write documentation for tech demos from Khalims demo -# TODO: -# Call with nybble -# Figure out what to do if I see an address that isn't registered but surveyrod -# Review deem score with last weeks deem score values to ensure accuracy -# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples -# Write documentation for tech demos from Khalims demo -# Improve dimirtra script by adding dates to the mix - + Tuesday + P1) - Get ready for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total + P2) Review deem score with last weeks deem score values to ensure accuracy + P3) Figure out what to do if I see an address that isn't registered but surveyrod +""" \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 3e65425..9474eff 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -58,8 +58,10 @@ class SharePointScraper(): # Delete me for production if development: - self.surveyor_names = ['Jun-te Kim (JJ Crump)'] - self.surveyor_to_dates_folder = {'Jun-te Kim (JJ Crump)': 'W.C. 31.03.2025'} + self.surveyor_names = ["Jun-te Kim (SCIS)"] + self.surveyor_to_dates_folder = { + 'Jun-te Kim (SCIS)': 'W.C. 31.03.2025', + } @@ -145,9 +147,6 @@ class SharePointScraper(): sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) - - - @ensure_surveyor_names_loaded def get_surveryor_names(self): return self.surveyor_names diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 34eda07..ded0f61 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -82,8 +82,14 @@ class SurveyPrice(): def get_price_matrix(self, sheet_name): df = pd.read_excel(self.master_rate_card_path, sheet_name) + pricing_table = [] if "SOLAR" in sheet_name.upper(): + for _, row in df.iterrows(): + pricing_table.append({ + "WORK TYPE": row["WORK TYPE"], + "PRICE": row["PRICE"], + }) return df else: columns_to_check = { @@ -97,7 +103,6 @@ class SurveyPrice(): 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, } - pricing_table = [] for _, row in df.iterrows(): for key, variables in columns_to_check.items(): pricing_table.append( @@ -140,26 +145,30 @@ class SurveyPrice(): def get_all_surveyed_data_from_sharepoint(self): # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for # DIfferent installers - self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() + # jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC) + scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION) + # self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True) + self.all_survey_info_from_sharepoint = scis_pd return self.all_survey_info_from_sharepoint + - def sharepoint_data_for_jjc(self): - jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) - file_paths = jjc_sp.download_file_for_each_address() - jjc_surveys = [] + def sharepoint_data_for_installer(self, installer): + sp = SharePointScraper(installer, development=True) + file_paths = sp.download_file_for_each_address() + surveys = [] for eachAddress in file_paths: for address, files in eachAddress.items(): - jjc_surveys.append(surveyedDataProcessor(address, files)) + surveys.append(surveyedDataProcessor(address, files)) all_survey_info = [] - for surveyInfo in jjc_surveys: + for surveyInfo in surveys: cavity_wall_as_built = False csr = False foam_insulation = False info = { - "SHAREPOINT INSTALLER": "J & J CRUMP", + "SHAREPOINT INSTALLER": installer.name, "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", @@ -235,14 +244,12 @@ class SurveyPrice(): # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( columns={ - 'SHAREPOINT INSTALLER': 'INSTALLER', 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', } ) self.all_hubspot_submissions = self.all_hubspot_submissions.rename( columns={ - 'HUBSPOT_INSTALLER': 'INSTALLER', 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', } @@ -251,8 +258,8 @@ class SurveyPrice(): merged_df = pd.merge( self.all_survey_info_from_sharepoint, self.all_hubspot_submissions, - left_on=['clean_address', 'INSTALLER'], - right_on=['clean_address', 'INSTALLER'], + left_on=['clean_address'], + right_on=['clean_address'], how='inner' ) @@ -288,13 +295,21 @@ class SurveyPrice(): final_list = [] for _, row in submission_data.iterrows(): if "SOLAR" in row["DOMNA JOB TYPE"].upper(): - raise NotImplementedError("Please implement solar pricing") + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge( + row.to_frame().T, + price_matrix, + left_on='DOMNA JOB TYPE', + right_on='WORK TYPE', + how='outer' + ) else: # Cavity wall - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') - final_list.append(merged_row) + final_list.append(merged_row) return pd.concat(final_list, ignore_index=True)