diff --git a/etl/development.py b/etl/development.py index 50d4749..e0f093a 100644 --- a/etl/development.py +++ b/etl/development.py @@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() -print(doc1.property_description.main_property) +print(doc1.property_description.door) # Transform diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 3ace9bc..068324f 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -2,7 +2,7 @@ from etl.pdfReader.reportType import ReportType from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, - Walls, Roofs, + Walls, Roofs, Floors, Door ) from datetime import datetime @@ -36,8 +36,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): """ self.transform_summary_information() self.transform_sections() - # self.get_section_9() - # self.get_section_10() # self.get_section_11() # self.get_section_12() # self.get_section_13() @@ -177,7 +175,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 3 age_bands = self.get_age_band() - + print() # Section 4 no_of_main_property = int(get_value("Main Property")) no_of_extension_1 = int(get_value('Extension 1') or 0) @@ -202,6 +200,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 8 roofs = self.get_roof() + # Section 9 + floors = self.get_floors() + + # Section 10 + door = self.get_door() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -221,26 +225,31 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): dimensions=dimensions["main"] if "main" in dimensions else [], wall=walls[0], roof=roofs[0], + floor=floors[0], )if no_of_main_property > 0 else None, ex1_property=PropertyDetail( age_band= age_bands[1], dimensions= dimensions["ex1"] if "ex1" in dimensions else [], wall=walls[1], roof=roofs[1], + floor=floors[1], )if no_of_extension_1 > 0 else None, ex2_property=PropertyDetail( age_band= age_bands[2], dimensions= dimensions["ex2"] if "ex2" in dimensions else [], wall=walls[2], roof=roofs[2], + floor=floors[2], )if no_of_extension_2 > 0 else None, ex3_property=PropertyDetail( age_band= age_bands[3], dimensions=dimensions["ex3"] if "ex3" in dimensions else [], wall=walls[3], roof=roofs[3], + floor=floors[3], )if no_of_extension_4 > 0 else None, conservatory=conservatory, + door=door, ) @@ -446,7 +455,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return group_data - def get_section_9(self): + def get_floors(self): data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')] sub_titles = [ "Floor Type", @@ -465,9 +474,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): ] - self.two_column_with_extension_processor(data, sub_titles, titles, 9) + lst = self.two_column_with_extension_processor(data, sub_titles, titles) + floors = [] + for floor in lst: + floors.append(Floors( + floor_type=floor.get("floor_type", ""), + ground_floor_construction=floor.get("get_floor_construction", ""), + ground_floor_insulation_type=floor.get("ground_floor_insulation_type", ""), + floor_insulation_thickness_mm=floor.get("floor_insulation_thickness_(mm)", -1) if floor.get("floor_insulation_thickness_(mm)", -1) is not None else -1, + u_value_known=True if floor.get("u_value_known").upper() == "YES" else False, + )) + return floors - def get_section_10(self): + def get_door(self): data = self.raw_data[self.raw_data.index("10.0 Doors"): self.raw_data.index("11.0 Windows")] avoid = [ "10.0 Doors", @@ -478,9 +497,15 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Number of Insulated Doors", "U-value (W/m²K)", ] - self.two_columns_processor(data, sub_titles, avoid, 10) + dict_ = self.two_columns_processor(data, sub_titles, avoid) - def two_columns_processor(self, data, sub_titles_to_gather, avoid, section, indexAdd = 1): + return Door( + no_of_doors=int(dict_.get("no_of_doors", -1)), + no_of_insulated_doors=int(dict_.get("number_of_insulated_doors", -1)), + u_value_w_m2_k=dict_.get("u_value_(w/m²k)", "") if dict_.get("u_value_(w/m²k), '')") is not None else "", + ) + + def two_columns_processor(self, data, sub_titles_to_gather, avoid, indexAdd = 1): def get_value(key): try: index = data.index(key) @@ -488,12 +513,13 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return None if value in avoid else value except (ValueError, IndexError): return None - + dict_ = {} for items in data: if items in avoid: continue elif items in sub_titles_to_gather: - setattr(self, f"section_{section}_{items.lower().replace('-', '_').replace(' ','_')}", get_value(items)) + dict_.update({f"{items.lower().replace('-', '_').replace(' ','_')}":get_value(items)}) + return dict_ def get_section_11(self): data = self.get_data_between("Window Location", "12.0 Ventilation & Cooling") diff --git a/etl/transform/types.py b/etl/transform/types.py index 7b2081e..880de6b 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -62,13 +62,13 @@ class Floors(BaseModel): floor_type: str ground_floor_construction: str ground_floor_insulation_type: str - floor_insulation_thickness_mm: Optional[float] + floor_insulation_thickness_mm: Optional[float] = -1 u_value_known: bool -class Doors(BaseModel): - no_of_doofs: int - nu_of_insulated_doors: int - u_value_w_m2_k: Optional[str] = None +class Door(BaseModel): + no_of_doors: int + no_of_insulated_doors: int + u_value_w_m2_k: Optional[str] class AssessorInfo(BaseModel): accreditation_number: str @@ -115,6 +115,7 @@ class PropertyDetail(BaseModel): dimensions: List[Dimension] = [] wall: Optional[Walls] = None roof: Optional[Roofs] = None + floor: Optional[Floors] = None class PropertyDescription(BaseModel): built_form: str @@ -135,4 +136,5 @@ class PropertyDescription(BaseModel): ex2_property: Optional[PropertyDetail] = None ex3_property: Optional[PropertyDetail] = None ex4_property: Optional[PropertyDetail] = None - conservatory: bool \ No newline at end of file + conservatory: bool + door: Optional[Door] \ No newline at end of file