From e5abbd633a97a37ca295818136ba9cfaff3f42cb Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 08:50:46 +0000 Subject: [PATCH 1/6] dimitra hubspot notes --- etl/dimitra_hubspot_notes_gather.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 2d92e4a..f173826 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -169,3 +169,6 @@ sharepoint_client.upload_file( f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}", file_name ) + + +print("hello world") \ No newline at end of file From af77c0ffbce655369e3fd3499db86ebbc4a66ed9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 10:29:11 +0000 Subject: [PATCH 2/6] occupant information --- .github/workflows/lambda_main.yml | 2 +- .../lambda/extractor_and_loader/docker/app.py | 16 +++++++-- etl/fileReader/sitenotes.py | 33 ++++++++++++------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 94e053a..6330833 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -2,7 +2,7 @@ name: Lambda Main Workflow on: push: - branches: [main, feature/document_upload] + branches: [main, feautre/additional_features_in_condition_report_extraction] env: AWS_REGION: eu-west-2 diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 929c16a..2da0228 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -57,15 +57,17 @@ def serialize_model(model: Any): else: return model -def make_final_json(rooms_obj, heating_system_obj): +def make_final_json(rooms_obj, heating_system_obj, occupant): # Convert to dict recursively rooms_data = serialize_model(rooms_obj) heating_data = serialize_model(heating_system_obj) + occupant_data = serialize_model(occupant) # Combine into one big JSON-ready dict final_data = { "rooms": rooms_data, - "heating_system": heating_data + "heating_system": heating_data, + "occupant_info": occupant_data, } # Convert to pretty JSON string @@ -198,10 +200,18 @@ def handler(event, context): print("Downloading file locally for extraction...") local_path = download_private_s3_file(file_uri) + local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) + + + print("Extracting file...") reader = pdfReaderToText(local_path) obj = WarmHomesConditionReport(reader.text_list) - json_ = make_final_json(obj.master_obj[0], obj.master_obj[1]) + json_ = make_final_json( + obj.master_obj[0], + obj.master_obj[1], + obj.master_obj[2] + ) print("Extracted completed, made json") print("uploading json to s3 bucket...") diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 073a30a..0901ed1 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -105,18 +105,19 @@ class WarmHomesConditionReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.WARM_HOMES_CONDITION_REPORT - room, heating_system = self.setup_condition_report() - self.master_obj = room, heating_system + room, heating_system, occupant = self.setup_condition_report() + + self.master_obj = room, heating_system, occupant def setup_condition_report(self): # general_information = self.get_section_1() # access_and_elevations = self.get_section_2() rooms = self.get_section_3() heating_system = self.get_section_4() - # occupant_assessment = self.get_section_5() + occupant_assessment = self.get_section_5() # site_name, reference_code, address, postcode = self.get_section_0() - return rooms, heating_system + return rooms, heating_system, occupant_assessment def get_section_0(self): data = self.get_data_between("Project Site Name", "1. General Information") @@ -606,13 +607,14 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_section_5(self): occupants = self.get_occupants() - energy_use = self.get_energy_use() - heating = self.get_heating() - shower_and_bath = self.get_shower_and_bath() - appliances = self.get_appliances() - fridge_and_freezers = self.get_fridge_and_freezers() - cooker = self.get_cooker() - tumble_dryer = self.get_tumble_dryer() + # energy_use = self.get_energy_use() + # heating = self.get_heating() + # shower_and_bath = self.get_shower_and_bath() + # appliances = self.get_appliances() + # fridge_and_freezers = self.get_fridge_and_freezers() + # cooker = self.get_cooker() + # tumble_dryer = self.get_tumble_dryer() + return occupants return OccupantAssessment( occupant=occupants, energy_use=energy_use, @@ -627,12 +629,19 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_occupants(self): data = self.get_data_between("Occupants", "Energy use") second_data = self.get_data_between("Tumble dryer", "Media summary") + no_of_child_occupants = self.get_next_value(data, "No. of Child Occupants (Under 18)") + if no_of_child_occupants == "\xa0": + no_of_child_occupants = 0 + else: + no_of_child_occupants = int(no_of_child_occupants) + + print(self.get_next_value(data, "No. of Child Occupants (Under 18)")) return Occupant( name=self.get_next_value(second_data, "Name of the occupant:"), have_evidence_of_12_months_of_fuel_bill_data= True if self.get_next_value(second_data, "Have you evidenced 12 months of fuel bill data?").lower() == "yes" else False, total_number_of_occupants=int(self.get_next_value(data, "Total number of occupants:")), no_of_adult_occupants=int(self.get_next_value(data, "No. of Adult Occupants (18+)")), - no_of_child_occupants=int(self.get_next_value(data, "No. of Child Occupants (Under 18)")), + no_of_child_occupants=no_of_child_occupants, no_of_occupant_of_a_pensionable_age=int(self.get_next_value(data, "No. of occupant of a pensionable age")), are_there_any_vulnerable_people=True if self.get_next_value(data, "Are there any vulnerable people?").lower() == "yes" else False, is_there_anyone_with_a_disability=True if self.get_next_value(data, "Is there anyone with a disability?").lower() == "yes" else False, From 5ff1d781e9bee4587e87b3eeebd2d0b3da29cb20 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 10:30:20 +0000 Subject: [PATCH 3/6] occupant information --- deployment/lambda/extractor_and_loader/docker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 2da0228..9da6715 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -200,7 +200,7 @@ def handler(event, context): print("Downloading file locally for extraction...") local_path = download_private_s3_file(file_uri) - local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) From a21d0f2a9e194ce7a9da6482881c529320e770d6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 10:32:13 +0000 Subject: [PATCH 4/6] added occupant information --- deployment/lambda/extractor_and_loader/docker/app.py | 2 +- etl/fileReader/sitenotes.py | 2 +- etl/transform/conditionReportTypes.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 9da6715..2da0228 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -200,7 +200,7 @@ def handler(event, context): print("Downloading file locally for extraction...") local_path = download_private_s3_file(file_uri) - # local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) + local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 0901ed1..736dec4 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -637,7 +637,7 @@ class WarmHomesConditionReport(SiteNotesExtractor): print(self.get_next_value(data, "No. of Child Occupants (Under 18)")) return Occupant( - name=self.get_next_value(second_data, "Name of the occupant:"), + # name=self.get_next_value(second_data, "Name of the occupant:"), have_evidence_of_12_months_of_fuel_bill_data= True if self.get_next_value(second_data, "Have you evidenced 12 months of fuel bill data?").lower() == "yes" else False, total_number_of_occupants=int(self.get_next_value(data, "Total number of occupants:")), no_of_adult_occupants=int(self.get_next_value(data, "No. of Adult Occupants (18+)")), diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index abd53f8..fe0d816 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -218,7 +218,7 @@ class HeatingSystem(BaseModel): class Occupant(BaseModel): - name: str + # name: str have_evidence_of_12_months_of_fuel_bill_data: bool total_number_of_occupants: int no_of_adult_occupants: int From bb878cd6c8c02cee4feefe0a3fd1b07e4a6422a1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 10:55:02 +0000 Subject: [PATCH 5/6] loft space --- deployment/lambda/extractor_and_loader/docker/app.py | 3 ++- etl/fileReader/sitenotes.py | 10 ++++++++++ etl/transform/conditionReportTypes.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 2da0228..d345df2 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -200,7 +200,8 @@ def handler(event, context): print("Downloading file locally for extraction...") local_path = download_private_s3_file(file_uri) - local_path = os.path.join(os.path.join(os.getcwd(), "../..", "home/Downloads/67-Aylestone-Road-1.pdf")) + # Local development of file, please comment out for prod + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf")) diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 736dec4..425c5a3 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -497,6 +497,16 @@ class WarmHomesConditionReport(SiteNotesExtractor): return LoftSpace( is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), + does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), + is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), + is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), + condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), + is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), + is_it_accessible=self.get_next_value(data, "Is it accessible?"), ) def get_room_in_roof(self): diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index fe0d816..ee3c091 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -147,6 +147,18 @@ class Bathroom(BaseModel): class LoftSpace(BaseModel): is_the_main_loft_space_accessible: str is_there_more_than_one_loft_space: bool + overall_condition_of_the_loft_space: Optional[str] + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] + does_the_loft_space_have_any_defects: Optional[str] + existing_depth_of_loft_insulation: Optional[str] + is_the_insulation_layer_even_across_the_loft: Optional[str] + is_the_loft_boarded_in_any_area: Optional[str] + condition_of_existing_roof_lining: Optional[str] + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] + is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] + is_it_accessible: Optional[str] + + class RoomInRoof(BaseModel): is_there_a_room_in_roof: bool From c3f1a15993887cdf541a97cc7793e8edbc40aba6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 11:36:35 +0000 Subject: [PATCH 6/6] access and elevation --- deployment/lambda/extractor_and_loader/docker/app.py | 7 +++++-- etl/fileReader/sitenotes.py | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index d345df2..8a08c48 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -57,17 +57,19 @@ def serialize_model(model: Any): else: return model -def make_final_json(rooms_obj, heating_system_obj, occupant): +def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations): # Convert to dict recursively rooms_data = serialize_model(rooms_obj) heating_data = serialize_model(heating_system_obj) occupant_data = serialize_model(occupant) + access_and_elevations_data = serialize_model(access_and_elevations) # Combine into one big JSON-ready dict final_data = { "rooms": rooms_data, "heating_system": heating_data, "occupant_info": occupant_data, + "access_and_elevations": access_and_elevations_data, } # Convert to pretty JSON string @@ -211,7 +213,8 @@ def handler(event, context): json_ = make_final_json( obj.master_obj[0], obj.master_obj[1], - obj.master_obj[2] + obj.master_obj[2], + obj.master_obj[3], ) print("Extracted completed, made json") diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 425c5a3..f2de52d 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -105,19 +105,19 @@ class WarmHomesConditionReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.WARM_HOMES_CONDITION_REPORT - room, heating_system, occupant = self.setup_condition_report() + room, heating_system, occupant, access_and_elevation = self.setup_condition_report() - self.master_obj = room, heating_system, occupant + self.master_obj = room, heating_system, occupant, access_and_elevation def setup_condition_report(self): # general_information = self.get_section_1() - # access_and_elevations = self.get_section_2() + access_and_elevations = self.get_section_2() rooms = self.get_section_3() heating_system = self.get_section_4() occupant_assessment = self.get_section_5() # site_name, reference_code, address, postcode = self.get_section_0() - return rooms, heating_system, occupant_assessment + return rooms, heating_system, occupant_assessment, access_and_elevations def get_section_0(self): data = self.get_data_between("Project Site Name", "1. General Information")