From ba6cf1402d59430ab16f834f2892adcf79cb83c8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 14 Mar 2025 12:32:53 +0000 Subject: [PATCH] save progress - finished roofs and added more sectios --- etl/development.py | 2 +- etl/pdfReader/sitenotes.py | 81 ++++++++++++++++++++++++++++++++------ etl/transform/types.py | 64 ++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 14 deletions(-) diff --git a/etl/development.py b/etl/development.py index 06a9bb1..50d4749 100644 --- a/etl/development.py +++ b/etl/development.py @@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() -print(doc1.survey_information) +print(doc1.property_description.main_property) # Transform diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index edd30fc..3ace9bc 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -1,5 +1,9 @@ from etl.pdfReader.reportType import ReportType -from transform.types import CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension +from transform.types import ( + CompanyInfo, SurverySummaryInfo, AssessorInfo, + PropertyDescription, PropertyDetail, Dimension, + Walls, Roofs, +) from datetime import datetime class SiteNotesExtractor(): @@ -32,8 +36,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): """ self.transform_summary_information() self.transform_sections() - # self.get_section_7() - # self.get_section_8() # self.get_section_9() # self.get_section_10() # self.get_section_11() @@ -155,7 +157,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 2 - data = self.raw_data[self.raw_data.index("2.0 Number Of"):self.raw_data.index("3.0 Date Built")] + data = self.get_data_between("2.0 Number Of","3.0 Date Built") avoid = [ '2.0 Number Of', @@ -193,6 +195,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 5 conservatory = self.is_there_a_conservatory() + + # Section 7 + walls = self.get_walls() + + # Section 8 + roofs = self.get_roof() self.property_description = PropertyDescription( built_form = get_value("Built Form"), @@ -209,20 +217,28 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): terrain_type = get_value('Terrain Type'), percentage_of_draught_proofed= int(get_value('Percentage of Draught Proofed(%)')), main_property=PropertyDetail( - age_band= age_bands[0], - dimensions= dimensions["main"] if "main" in dimensions else [], + age_band=age_bands[0], + dimensions=dimensions["main"] if "main" in dimensions else [], + wall=walls[0], + roof=roofs[0], )if no_of_main_property > 0 else None, ex1_property=PropertyDetail( age_band= age_bands[1], dimensions= dimensions["ex1"] if "ex1" in dimensions else [], + wall=walls[1], + roof=roofs[1], )if no_of_extension_1 > 0 else None, ex2_property=PropertyDetail( age_band= age_bands[2], dimensions= dimensions["ex2"] if "ex2" in dimensions else [], + wall=walls[2], + roof=roofs[2], )if no_of_extension_2 > 0 else None, ex3_property=PropertyDetail( age_band= age_bands[3], dimensions=dimensions["ex3"] if "ex3" in dimensions else [], + wall=walls[3], + roof=roofs[3], )if no_of_extension_4 > 0 else None, conservatory=conservatory, ) @@ -320,7 +336,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): def get_section_6(self): pass - def get_section_7(self): + def get_walls(self): data = self.get_data_between('7.0 Walls','8.0 Roofs') sub_titles = [ @@ -345,9 +361,27 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Extension 3", "Extension 4", ] - self.two_column_with_extension_processor(data, sub_titles, main_titles, 7) + lst = self.two_column_with_extension_processor(data, sub_titles, main_titles) + + walls = [] + for ext in lst: + walls.append(Walls( + construction=ext["construction"] if ext["construction"] is not None else "", + insulation=str(ext["insulation"]), + insulation_thickness_mm=str(ext["insulation_thickness(mm)"]), + wall_thickness_measured=True if ext["wall_thickness_measured"].upper() == "YES" else False, + wall_thickness_mm=int(ext.get("wall_thickness(mm)",-1)), + u_value_known=True if ext["u_value_known"].upper() == "YES" else False, + u_value_w_m2_k=float(ext.get("u_value_(w/m²k)", -1.0)), + dry_lining=True if ext.get("dry_lining", "NO").upper() == "YES" else False, + alternative_wall_present=True if ext.get("alternative_wall_present", "NO").upper() == "YES" else False, + )) + return walls - def get_section_8(self): + + + + def get_roof(self): data = self.raw_data[self.raw_data.index('8.0 Roofs'): self.raw_data.index('9.0 Floors')] sub_titles = [ "Construction", @@ -364,9 +398,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Extension 4", ] - self.two_column_with_extension_processor(data, sub_titles, titles, 8) + lst = self.two_column_with_extension_processor(data, sub_titles, titles) + roofs = [] + for roof in lst: + roofs.append(Roofs( + construction=roof["construction"] if roof["construction"] is not None else "", + insulation_type=str(roof["insulation_type"]), + insulation_thickness=str(roof["insulation_thickness"]), + u_value_known=True if roof["u_value_known"].upper() == "YES" else False, + )) + return roofs - def two_column_with_extension_processor(self, data, sub_titles, main_titles, section): + + def two_column_with_extension_processor(self, data, sub_titles, main_titles): def get_value(key): try: index = proc_data.index(key) @@ -374,22 +418,33 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return None if value in sub_titles else value except (ValueError, IndexError): return None - title = None proc_data = data + return_dict = {} + group_data = [] for items in data: if items in main_titles: title = items.lower().replace(" ", "_").replace("-", "_") index = main_titles.index(items) if main_titles[index] in data: + if return_dict: + group_data.append(return_dict) + return_dict = {} proc_data = data[data.index(main_titles[index]):] continue else: + if return_dict: + group_data.append(return_dict) break if title is None: continue if items in sub_titles: - setattr(self, f"section_{section}_{title}_{items.lower().replace(' ', '_').replace('-','_')}", get_value(items)) + return_dict.update({f"{items.lower().replace("?", "").replace(' ', '_').replace('-','_')}": get_value(items)}) + + if return_dict: + group_data.append(return_dict) + + return group_data def get_section_9(self): data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')] diff --git a/etl/transform/types.py b/etl/transform/types.py index 1058637..7b2081e 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -41,6 +41,34 @@ class SurverySummaryInfo(BaseModel): current_annual_emission_including_0925_multiplayer: str current_annual_energy_costs: str +class Walls(BaseModel): + construction: str + insulation: str + insulation_thickness_mm: str + wall_thickness_measured: bool + wall_thickness_mm: Optional[int] + u_value_known: bool + u_value_w_m2_k: Optional[float] + dry_lining: bool + alternative_wall_present: bool + +class Roofs(BaseModel): + construction: str + insulation_type: str + insulation_thickness: str + u_value_known: bool + +class Floors(BaseModel): + floor_type: str + ground_floor_construction: str + ground_floor_insulation_type: str + floor_insulation_thickness_mm: Optional[float] + u_value_known: bool + +class Doors(BaseModel): + no_of_doofs: int + nu_of_insulated_doors: int + u_value_w_m2_k: Optional[str] = None class AssessorInfo(BaseModel): accreditation_number: str @@ -48,9 +76,45 @@ class AssessorInfo(BaseModel): phone_number: Optional[str] = None email_address: Optional[EmailStr] = None +class Windows(BaseModel): + glazing_type: str + area_m2: float + roof_window: bool + orientation: str + u_value_w_m2_k: int + g_value: int + +class FirePlaces(BaseModel): + no_of_open_fireplaces: int + ventilation_type: str + space_cooling_system_present: bool + +class Lighting(BaseModel): + total_no_of_light_fittings: int + total_no_of_lel_fittings: int + +class MainHeatingSystemControls(BaseModel): + control_type: str + flue_type: str + fan_assisted_flue: str + heat_emitter_type: str + electricity_meter_type: Optional[str] + mains_gas_available: Optional[bool] + +class MainHeating(BaseModel): + heating_source: str + efficiency_source: str + heating_fuel: str + brand_name: str + model_name: str + model_qualifer: str + controls: MainHeatingSystemControls + class PropertyDetail(BaseModel): age_band: str dimensions: List[Dimension] = [] + wall: Optional[Walls] = None + roof: Optional[Roofs] = None class PropertyDescription(BaseModel): built_form: str