Merge pull request #79 from Hestia-Homes/feautre/additional_features_in_condition_report_extraction

Feautre/additional features in condition report extraction

.github/workflows/lambda_main.yml

@@ -2,7 +2,7 @@ name: Lambda Main Workflow
 
 on:
   push:
-    branches: [main, feature/document_upload]
+    branches: [main, feautre/additional_features_in_condition_report_extraction]
 
 env:
   AWS_REGION: eu-west-2

deployment/lambda/extractor_and_loader/docker/app.py

@@ -57,15 +57,19 @@ def serialize_model(model: Any):
     else:
         return model
 
-def make_final_json(rooms_obj, heating_system_obj):
+def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations):
     # Convert to dict recursively
     rooms_data = serialize_model(rooms_obj)
     heating_data = serialize_model(heating_system_obj)
+    occupant_data = serialize_model(occupant)
+    access_and_elevations_data = serialize_model(access_and_elevations)
 
     # Combine into one big JSON-ready dict
     final_data = {
         "rooms": rooms_data,
-        "heating_system": heating_data
+        "heating_system": heating_data,
+        "occupant_info": occupant_data,
+        "access_and_elevations": access_and_elevations_data,
     }
 
     # Convert to pretty JSON string
@@ -198,10 +202,20 @@ def handler(event, context):
             print("Downloading file locally for extraction...")
             local_path = download_private_s3_file(file_uri)
 
+            # Local development of file, please comment out for prod
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf"))
+
+
+
             print("Extracting file...")
             reader = pdfReaderToText(local_path)
             obj = WarmHomesConditionReport(reader.text_list)
-            json_ = make_final_json(obj.master_obj[0], obj.master_obj[1])
+            json_ = make_final_json(
+                obj.master_obj[0],
+                obj.master_obj[1],
+                obj.master_obj[2],
+                obj.master_obj[3],
+            )
             print("Extracted completed, made json")
             
             print("uploading json to s3 bucket...")

etl/dimitra_hubspot_notes_gather.py

@@ -169,3 +169,6 @@ sharepoint_client.upload_file(
     f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",
     file_name
 )
+
+
+print("hello world")

etl/fileReader/sitenotes.py

@@ -105,18 +105,19 @@ class WarmHomesConditionReport(SiteNotesExtractor):
     def __init__(self, data_list):
         super().__init__(data_list)
         self.type = ReportType.WARM_HOMES_CONDITION_REPORT
-        room, heating_system = self.setup_condition_report()
-        self.master_obj = room, heating_system
+        room, heating_system, occupant, access_and_elevation = self.setup_condition_report()
+
+        self.master_obj = room, heating_system, occupant, access_and_elevation
 
     def setup_condition_report(self):
         # general_information = self.get_section_1()
-        # access_and_elevations = self.get_section_2()
+        access_and_elevations = self.get_section_2()
         rooms = self.get_section_3()
         heating_system = self.get_section_4()
-        # occupant_assessment = self.get_section_5()
+        occupant_assessment = self.get_section_5()
         # site_name, reference_code, address, postcode = self.get_section_0()
 
-        return rooms, heating_system
+        return rooms, heating_system, occupant_assessment, access_and_elevations
         
     def get_section_0(self):
         data = self.get_data_between("Project Site Name", "1. General Information")
@@ -496,6 +497,16 @@ class WarmHomesConditionReport(SiteNotesExtractor):
         return LoftSpace(
             is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"),
             is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False,
+            overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"),
+            are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"),
+            does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
+            existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"),
+            is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"),
+            is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"),
+            condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"),
+            is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"),
+            is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"),
+            is_it_accessible=self.get_next_value(data, "Is it accessible?"),
         )
     
     def get_room_in_roof(self):
@@ -606,13 +617,14 @@ class WarmHomesConditionReport(SiteNotesExtractor):
     
     def get_section_5(self):
         occupants = self.get_occupants()
-        energy_use = self.get_energy_use()
-        heating = self.get_heating()
-        shower_and_bath = self.get_shower_and_bath()
-        appliances = self.get_appliances()
-        fridge_and_freezers = self.get_fridge_and_freezers()
-        cooker = self.get_cooker()
-        tumble_dryer = self.get_tumble_dryer()
+        # energy_use = self.get_energy_use()
+        # heating = self.get_heating()
+        # shower_and_bath = self.get_shower_and_bath()
+        # appliances = self.get_appliances()
+        # fridge_and_freezers = self.get_fridge_and_freezers()
+        # cooker = self.get_cooker()
+        # tumble_dryer = self.get_tumble_dryer()
+        return occupants
         return OccupantAssessment(
             occupant=occupants,
             energy_use=energy_use,
@@ -627,12 +639,19 @@ class WarmHomesConditionReport(SiteNotesExtractor):
     def get_occupants(self):
         data = self.get_data_between("Occupants", "Energy use")
         second_data = self.get_data_between("Tumble dryer", "Media summary")
+        no_of_child_occupants = self.get_next_value(data, "No. of Child Occupants (Under 18)")
+        if no_of_child_occupants == "\xa0":
+            no_of_child_occupants = 0
+        else:
+            no_of_child_occupants = int(no_of_child_occupants)
+        
+        print(self.get_next_value(data, "No. of Child Occupants (Under 18)"))
         return Occupant(
-            name=self.get_next_value(second_data, "Name of the occupant:"),
+            # name=self.get_next_value(second_data, "Name of the occupant:"),
             have_evidence_of_12_months_of_fuel_bill_data= True if self.get_next_value(second_data, "Have you evidenced 12 months of fuel bill data?").lower() == "yes" else False,
             total_number_of_occupants=int(self.get_next_value(data, "Total number of occupants:")),
             no_of_adult_occupants=int(self.get_next_value(data, "No. of Adult Occupants (18+)")),
-            no_of_child_occupants=int(self.get_next_value(data, "No. of Child Occupants (Under 18)")),
+            no_of_child_occupants=no_of_child_occupants,
             no_of_occupant_of_a_pensionable_age=int(self.get_next_value(data, "No. of occupant of a pensionable age")),
             are_there_any_vulnerable_people=True if self.get_next_value(data, "Are there any vulnerable people?").lower() == "yes" else False,
             is_there_anyone_with_a_disability=True if self.get_next_value(data, "Is there anyone with a disability?").lower() == "yes" else False,

etl/transform/conditionReportTypes.py

@@ -147,6 +147,18 @@ class Bathroom(BaseModel):
 class LoftSpace(BaseModel):
     is_the_main_loft_space_accessible: str
     is_there_more_than_one_loft_space: bool
+    overall_condition_of_the_loft_space: Optional[str]
+    are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str]
+    does_the_loft_space_have_any_defects: Optional[str]
+    existing_depth_of_loft_insulation: Optional[str]
+    is_the_insulation_layer_even_across_the_loft: Optional[str]
+    is_the_loft_boarded_in_any_area: Optional[str]
+    condition_of_existing_roof_lining: Optional[str]
+    is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str]
+    is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str]
+    is_it_accessible: Optional[str]
+
+
 
 class RoomInRoof(BaseModel):
     is_there_a_room_in_roof: bool
@@ -218,7 +230,7 @@ class HeatingSystem(BaseModel):
 
 
 class Occupant(BaseModel):
-    name: str
+    # name: str
     have_evidence_of_12_months_of_fuel_bill_data: bool
     total_number_of_occupants: int
     no_of_adult_occupants: int