diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml index 94e053a..6330833 100644 --- a/.github/workflows/lambda_main.yml +++ b/.github/workflows/lambda_main.yml @@ -2,7 +2,7 @@ name: Lambda Main Workflow on: push: - branches: [main, feature/document_upload] + branches: [main, feautre/additional_features_in_condition_report_extraction] env: AWS_REGION: eu-west-2 diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 929c16a..8a08c48 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -57,15 +57,19 @@ def serialize_model(model: Any): else: return model -def make_final_json(rooms_obj, heating_system_obj): +def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations): # Convert to dict recursively rooms_data = serialize_model(rooms_obj) heating_data = serialize_model(heating_system_obj) + occupant_data = serialize_model(occupant) + access_and_elevations_data = serialize_model(access_and_elevations) # Combine into one big JSON-ready dict final_data = { "rooms": rooms_data, - "heating_system": heating_data + "heating_system": heating_data, + "occupant_info": occupant_data, + "access_and_elevations": access_and_elevations_data, } # Convert to pretty JSON string @@ -198,10 +202,20 @@ def handler(event, context): print("Downloading file locally for extraction...") local_path = download_private_s3_file(file_uri) + # Local development of file, please comment out for prod + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf")) + + + print("Extracting file...") reader = pdfReaderToText(local_path) obj = WarmHomesConditionReport(reader.text_list) - json_ = make_final_json(obj.master_obj[0], obj.master_obj[1]) + json_ = make_final_json( + obj.master_obj[0], + obj.master_obj[1], + obj.master_obj[2], + obj.master_obj[3], + ) print("Extracted completed, made json") print("uploading json to s3 bucket...") diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 2d92e4a..f173826 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -169,3 +169,6 @@ sharepoint_client.upload_file( f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}", file_name ) + + +print("hello world") \ No newline at end of file diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 073a30a..f2de52d 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -105,18 +105,19 @@ class WarmHomesConditionReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.WARM_HOMES_CONDITION_REPORT - room, heating_system = self.setup_condition_report() - self.master_obj = room, heating_system + room, heating_system, occupant, access_and_elevation = self.setup_condition_report() + + self.master_obj = room, heating_system, occupant, access_and_elevation def setup_condition_report(self): # general_information = self.get_section_1() - # access_and_elevations = self.get_section_2() + access_and_elevations = self.get_section_2() rooms = self.get_section_3() heating_system = self.get_section_4() - # occupant_assessment = self.get_section_5() + occupant_assessment = self.get_section_5() # site_name, reference_code, address, postcode = self.get_section_0() - return rooms, heating_system + return rooms, heating_system, occupant_assessment, access_and_elevations def get_section_0(self): data = self.get_data_between("Project Site Name", "1. General Information") @@ -496,6 +497,16 @@ class WarmHomesConditionReport(SiteNotesExtractor): return LoftSpace( is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), + does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), + is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), + is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), + condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), + is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), + is_it_accessible=self.get_next_value(data, "Is it accessible?"), ) def get_room_in_roof(self): @@ -606,13 +617,14 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_section_5(self): occupants = self.get_occupants() - energy_use = self.get_energy_use() - heating = self.get_heating() - shower_and_bath = self.get_shower_and_bath() - appliances = self.get_appliances() - fridge_and_freezers = self.get_fridge_and_freezers() - cooker = self.get_cooker() - tumble_dryer = self.get_tumble_dryer() + # energy_use = self.get_energy_use() + # heating = self.get_heating() + # shower_and_bath = self.get_shower_and_bath() + # appliances = self.get_appliances() + # fridge_and_freezers = self.get_fridge_and_freezers() + # cooker = self.get_cooker() + # tumble_dryer = self.get_tumble_dryer() + return occupants return OccupantAssessment( occupant=occupants, energy_use=energy_use, @@ -627,12 +639,19 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_occupants(self): data = self.get_data_between("Occupants", "Energy use") second_data = self.get_data_between("Tumble dryer", "Media summary") + no_of_child_occupants = self.get_next_value(data, "No. of Child Occupants (Under 18)") + if no_of_child_occupants == "\xa0": + no_of_child_occupants = 0 + else: + no_of_child_occupants = int(no_of_child_occupants) + + print(self.get_next_value(data, "No. of Child Occupants (Under 18)")) return Occupant( - name=self.get_next_value(second_data, "Name of the occupant:"), + # name=self.get_next_value(second_data, "Name of the occupant:"), have_evidence_of_12_months_of_fuel_bill_data= True if self.get_next_value(second_data, "Have you evidenced 12 months of fuel bill data?").lower() == "yes" else False, total_number_of_occupants=int(self.get_next_value(data, "Total number of occupants:")), no_of_adult_occupants=int(self.get_next_value(data, "No. of Adult Occupants (18+)")), - no_of_child_occupants=int(self.get_next_value(data, "No. of Child Occupants (Under 18)")), + no_of_child_occupants=no_of_child_occupants, no_of_occupant_of_a_pensionable_age=int(self.get_next_value(data, "No. of occupant of a pensionable age")), are_there_any_vulnerable_people=True if self.get_next_value(data, "Are there any vulnerable people?").lower() == "yes" else False, is_there_anyone_with_a_disability=True if self.get_next_value(data, "Is there anyone with a disability?").lower() == "yes" else False, diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index abd53f8..ee3c091 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -147,6 +147,18 @@ class Bathroom(BaseModel): class LoftSpace(BaseModel): is_the_main_loft_space_accessible: str is_there_more_than_one_loft_space: bool + overall_condition_of_the_loft_space: Optional[str] + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] + does_the_loft_space_have_any_defects: Optional[str] + existing_depth_of_loft_insulation: Optional[str] + is_the_insulation_layer_even_across_the_loft: Optional[str] + is_the_loft_boarded_in_any_area: Optional[str] + condition_of_existing_roof_lining: Optional[str] + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] + is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] + is_it_accessible: Optional[str] + + class RoomInRoof(BaseModel): is_there_a_room_in_roof: bool @@ -218,7 +230,7 @@ class HeatingSystem(BaseModel): class Occupant(BaseModel): - name: str + # name: str have_evidence_of_12_months_of_fuel_bill_data: bool total_number_of_occupants: int no_of_adult_occupants: int