junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge pdf works now

Browse source
This commit is contained in:
Jun-te Kim 2025-04-14 15:47:35 +00:00
parent ae4ddf49eb
commit cc07706fc1
2 changed files with 37 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
etl/hubspot_to_invoice.py
View file

@ -13,5 +13,7 @@ from etl.surveyPrice.surveyPrice import SurveyPrice
sp = SurveyPrice()
sp.get_cavity_pricing_table("JJC - EMPTIES")
df = sp.get_all_surveys_from_hubspot()
df = sp.get_all_surveyed_data_from_sharepoint()
hubspot_df = sp.get_all_surveys_from_hubspot()
sharepoint_df = sp.get_all_surveyed_data_from_sharepoint()
df = sp.merge_hub_spot_and_survey_information()

37
etl/surveyPrice/surveyPrice.py
View file

@ -12,6 +12,7 @@ class SurveyPrice():
self.warmfront_sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT)
self.master_rate_card_path = None
self.all_hubspot_submissions = None
self.all_survey_info_from_sharepoint = None
self.download_price_card()
self.required_sheets = [
'JJC - EMPTIES',
@ -103,7 +104,8 @@ class SurveyPrice():
return self.all_hubspot_submissions
def get_all_surveyed_data_from_sharepoint(self):
self.sharepoint_data_for_jjc()
self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc()
return self.all_survey_info_from_sharepoint
def sharepoint_data_for_jjc(self):
jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True)
@ -117,13 +119,14 @@ class SurveyPrice():
all_survey_info = []
for surveyInfo in jjc_surveys:
info = {
"SHAREPOINT INSTALLER": "J & J Crumps",
"SHAREPOINT INSTALLER": "J & J CRUMP",
"SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False,
"SHAREPOINT CSR FOUND": True if surveyInfo.csr else False,
"SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND",
"SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND",
"SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND",
"SHAREPOINT INSULATION MATERIAL": None,
"SHAREPOINT ADDRESS": address
}
if surveyInfo.pre_site_note:
@ -139,10 +142,36 @@ class SurveyPrice():
"SHAREPOINT INSULATION MATERIAL": surveyInfo.get_insulation_info(),
})
all_survey_info.append(info)
return jjc_surveys
return pd.DataFrame(all_survey_info)
def merge_hub_spot_and_survey_information(self):
if self.all_survey_info_from_sharepoint is None:
raise RuntimeError("No survey information found from Sharepoint")
if self.all_hubspot_submissions is None:
raise RuntimeError("No information found from Hubspot")
self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply(
lambda x: x.lower().replace(',', '').strip()
)
self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply(
lambda x: x.lower().replace(',', '').strip()
)
merged_df = pd.merge(
self.all_survey_info_from_sharepoint,
self.all_hubspot_submissions,
left_on=['clean_address', 'SHAREPOINT INSTALLER'],
right_on=['clean_address', 'HUBSPOT_INSTALLER'],
how='inner'
)
merged_df.drop(columns=['clean_address'], inplace=True)
return merged_df