junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

save dimitra working copy

Browse source
This commit is contained in:
Jun-te Kim 2025-06-17 07:55:11 +00:00
parent 5f12cea953
commit 413538ac3c
2 changed files with 45 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
etl/dimitra_hubspot_notes_gather.py
View file

@ -63,6 +63,7 @@ for pipeline in pipelines.results:
for stage in pipeline.stages:
if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]:
for deals in hubspot.get_all_deals_from_stage_id(stage.id):
print(f"Scraping deal {deals['deal_id']}")
time.sleep(1)
deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []}
notes = hubspot.get_notes_from_deals_id(deals["deal_id"])
@ -72,6 +73,7 @@ for pipeline in pipelines.results:
if not week_label:
continue
html_body = note['note']
print(f"Debugging purposes html_body looks like {html_body}")
soup = BeautifulSoup(html_body, "html.parser")
plain_text = soup.get_text(separator="\n")
deal_notes_by_week[week_label].append(plain_text)
@ -85,6 +87,7 @@ for pipeline in pipelines.results:
except:
owner_name = "Couldn't find owner information"
# Unique identifier to Domna Homes' hubspot
portal_id = 145275138
notes_data[pipeline_name].append({

65
etl/pdfReader/sitenotes.py
View file

@ -95,15 +95,34 @@ class ConditionReport(SiteNotesExtractor):
self.setup()
def setup(self):
self.get_section_1()
self.get_section_2()
self.get_section_3()
self.get_section_4()
self.get_section_5()
self.get_section_0()
assesor_details, inspection_and_project, the_property, main_elevation, elevations = self.get_section_1()
access_and_elevations = self.get_section_2()
rooms = self.get_section_3()
general_condition_of_heating_system, main_heating_one, main_heating_two, secondary_heating, heating_by_room, renewables = self.get_section_4()
occupants, energy_use, heating, shower_and_bath, appliances, fridge_and_freezers, cooker, tumble_dryer = self.get_section_5()
site_name, reference_code, address, postcode = self.get_section_0()
# TODO: Complete this function
self.make_condition_report_object(
assesor_details,
inspection_and_project,
the_property,
main_elevation,
elevations,
access_and_elevations,
rooms,
)
def get_section_0(self):
print("Please complete")
data = self.get_data_between("Project Site Name", "1. General Information")
site_name = self.get_next_value(data, "Project Site Name")
reference_code = self.get_next_value(data, "Property Reference Code")
address = self.get_next_value(data, "Property Address")
postcode = self.get_data_between("Postcode", "Main Image")[1:]
postcode = " ".join(postcode)
return site_name, reference_code, address, postcode
def get_section_1(self):
assessor_details = self.get_assessor_details()
@ -479,13 +498,13 @@ class ConditionReport(SiteNotesExtractor):
)
def get_section_4(self):
_ = self.get_general_condition_of_heating_system()
_ = self.get_main_heating_one()
_ = self.get_main_heating_two()
_ = self.get_secondary_heating()
_ = self.get_heating_by_room()
_ = self.get_renewables()
pprint(_)
general_condition_of_heating_system = self.get_general_condition_of_heating_system()
main_heating_one = self.get_main_heating_one()
main_heating_two = self.get_main_heating_two()
secondary_heating = self.get_secondary_heating()
heating_by_room = self.get_heating_by_room()
renewables = self.get_renewables()
return general_condition_of_heating_system, main_heating_one, main_heating_two, secondary_heating, heating_by_room, renewables
def get_main_heating_one(self):
data = self.get_data_between("Main Heating 1", "Main Heating 2")
@ -565,15 +584,15 @@ class ConditionReport(SiteNotesExtractor):
)
def get_section_5(self):
_ = self.get_occupants()
_ = self.get_energy_use()
_ = self.get_heating()
_ = self.get_shower_and_bath()
_ = self.get_appliances()
_ = self.get_fridge_and_freezers()
_ = self.get_cooker()
_ = self.get_tumble_dryer()
pprint(_)
occupants = self.get_occupants()
energy_use = self.get_energy_use()
heating = self.get_heating()
shower_and_bath = self.get_shower_and_bath()
appliances = self.get_appliances()
fridge_and_freezers = self.get_fridge_and_freezers()
cooker = self.get_cooker()
tumble_dryer = self.get_tumble_dryer()
return occupants, energy_use, heating, shower_and_bath, appliances, fridge_and_freezers, cooker, tumble_dryer
def get_occupants(self):
data = self.get_data_between("Occupants", "Energy use")