From ba6cf1402d59430ab16f834f2892adcf79cb83c8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 14 Mar 2025 12:32:53 +0000 Subject: [PATCH 1/8] save progress - finished roofs and added more sectios --- etl/development.py | 2 +- etl/pdfReader/sitenotes.py | 81 ++++++++++++++++++++++++++++++++------ etl/transform/types.py | 64 ++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 14 deletions(-) diff --git a/etl/development.py b/etl/development.py index 06a9bb1..50d4749 100644 --- a/etl/development.py +++ b/etl/development.py @@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() -print(doc1.survey_information) +print(doc1.property_description.main_property) # Transform diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index edd30fc..3ace9bc 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -1,5 +1,9 @@ from etl.pdfReader.reportType import ReportType -from transform.types import CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension +from transform.types import ( + CompanyInfo, SurverySummaryInfo, AssessorInfo, + PropertyDescription, PropertyDetail, Dimension, + Walls, Roofs, +) from datetime import datetime class SiteNotesExtractor(): @@ -32,8 +36,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): """ self.transform_summary_information() self.transform_sections() - # self.get_section_7() - # self.get_section_8() # self.get_section_9() # self.get_section_10() # self.get_section_11() @@ -155,7 +157,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 2 - data = self.raw_data[self.raw_data.index("2.0 Number Of"):self.raw_data.index("3.0 Date Built")] + data = self.get_data_between("2.0 Number Of","3.0 Date Built") avoid = [ '2.0 Number Of', @@ -193,6 +195,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 5 conservatory = self.is_there_a_conservatory() + + # Section 7 + walls = self.get_walls() + + # Section 8 + roofs = self.get_roof() self.property_description = PropertyDescription( built_form = get_value("Built Form"), @@ -209,20 +217,28 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): terrain_type = get_value('Terrain Type'), percentage_of_draught_proofed= int(get_value('Percentage of Draught Proofed(%)')), main_property=PropertyDetail( - age_band= age_bands[0], - dimensions= dimensions["main"] if "main" in dimensions else [], + age_band=age_bands[0], + dimensions=dimensions["main"] if "main" in dimensions else [], + wall=walls[0], + roof=roofs[0], )if no_of_main_property > 0 else None, ex1_property=PropertyDetail( age_band= age_bands[1], dimensions= dimensions["ex1"] if "ex1" in dimensions else [], + wall=walls[1], + roof=roofs[1], )if no_of_extension_1 > 0 else None, ex2_property=PropertyDetail( age_band= age_bands[2], dimensions= dimensions["ex2"] if "ex2" in dimensions else [], + wall=walls[2], + roof=roofs[2], )if no_of_extension_2 > 0 else None, ex3_property=PropertyDetail( age_band= age_bands[3], dimensions=dimensions["ex3"] if "ex3" in dimensions else [], + wall=walls[3], + roof=roofs[3], )if no_of_extension_4 > 0 else None, conservatory=conservatory, ) @@ -320,7 +336,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): def get_section_6(self): pass - def get_section_7(self): + def get_walls(self): data = self.get_data_between('7.0 Walls','8.0 Roofs') sub_titles = [ @@ -345,9 +361,27 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Extension 3", "Extension 4", ] - self.two_column_with_extension_processor(data, sub_titles, main_titles, 7) + lst = self.two_column_with_extension_processor(data, sub_titles, main_titles) + + walls = [] + for ext in lst: + walls.append(Walls( + construction=ext["construction"] if ext["construction"] is not None else "", + insulation=str(ext["insulation"]), + insulation_thickness_mm=str(ext["insulation_thickness(mm)"]), + wall_thickness_measured=True if ext["wall_thickness_measured"].upper() == "YES" else False, + wall_thickness_mm=int(ext.get("wall_thickness(mm)",-1)), + u_value_known=True if ext["u_value_known"].upper() == "YES" else False, + u_value_w_m2_k=float(ext.get("u_value_(w/m²k)", -1.0)), + dry_lining=True if ext.get("dry_lining", "NO").upper() == "YES" else False, + alternative_wall_present=True if ext.get("alternative_wall_present", "NO").upper() == "YES" else False, + )) + return walls - def get_section_8(self): + + + + def get_roof(self): data = self.raw_data[self.raw_data.index('8.0 Roofs'): self.raw_data.index('9.0 Floors')] sub_titles = [ "Construction", @@ -364,9 +398,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Extension 4", ] - self.two_column_with_extension_processor(data, sub_titles, titles, 8) + lst = self.two_column_with_extension_processor(data, sub_titles, titles) + roofs = [] + for roof in lst: + roofs.append(Roofs( + construction=roof["construction"] if roof["construction"] is not None else "", + insulation_type=str(roof["insulation_type"]), + insulation_thickness=str(roof["insulation_thickness"]), + u_value_known=True if roof["u_value_known"].upper() == "YES" else False, + )) + return roofs - def two_column_with_extension_processor(self, data, sub_titles, main_titles, section): + + def two_column_with_extension_processor(self, data, sub_titles, main_titles): def get_value(key): try: index = proc_data.index(key) @@ -374,22 +418,33 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return None if value in sub_titles else value except (ValueError, IndexError): return None - title = None proc_data = data + return_dict = {} + group_data = [] for items in data: if items in main_titles: title = items.lower().replace(" ", "_").replace("-", "_") index = main_titles.index(items) if main_titles[index] in data: + if return_dict: + group_data.append(return_dict) + return_dict = {} proc_data = data[data.index(main_titles[index]):] continue else: + if return_dict: + group_data.append(return_dict) break if title is None: continue if items in sub_titles: - setattr(self, f"section_{section}_{title}_{items.lower().replace(' ', '_').replace('-','_')}", get_value(items)) + return_dict.update({f"{items.lower().replace("?", "").replace(' ', '_').replace('-','_')}": get_value(items)}) + + if return_dict: + group_data.append(return_dict) + + return group_data def get_section_9(self): data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')] diff --git a/etl/transform/types.py b/etl/transform/types.py index 1058637..7b2081e 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -41,6 +41,34 @@ class SurverySummaryInfo(BaseModel): current_annual_emission_including_0925_multiplayer: str current_annual_energy_costs: str +class Walls(BaseModel): + construction: str + insulation: str + insulation_thickness_mm: str + wall_thickness_measured: bool + wall_thickness_mm: Optional[int] + u_value_known: bool + u_value_w_m2_k: Optional[float] + dry_lining: bool + alternative_wall_present: bool + +class Roofs(BaseModel): + construction: str + insulation_type: str + insulation_thickness: str + u_value_known: bool + +class Floors(BaseModel): + floor_type: str + ground_floor_construction: str + ground_floor_insulation_type: str + floor_insulation_thickness_mm: Optional[float] + u_value_known: bool + +class Doors(BaseModel): + no_of_doofs: int + nu_of_insulated_doors: int + u_value_w_m2_k: Optional[str] = None class AssessorInfo(BaseModel): accreditation_number: str @@ -48,9 +76,45 @@ class AssessorInfo(BaseModel): phone_number: Optional[str] = None email_address: Optional[EmailStr] = None +class Windows(BaseModel): + glazing_type: str + area_m2: float + roof_window: bool + orientation: str + u_value_w_m2_k: int + g_value: int + +class FirePlaces(BaseModel): + no_of_open_fireplaces: int + ventilation_type: str + space_cooling_system_present: bool + +class Lighting(BaseModel): + total_no_of_light_fittings: int + total_no_of_lel_fittings: int + +class MainHeatingSystemControls(BaseModel): + control_type: str + flue_type: str + fan_assisted_flue: str + heat_emitter_type: str + electricity_meter_type: Optional[str] + mains_gas_available: Optional[bool] + +class MainHeating(BaseModel): + heating_source: str + efficiency_source: str + heating_fuel: str + brand_name: str + model_name: str + model_qualifer: str + controls: MainHeatingSystemControls + class PropertyDetail(BaseModel): age_band: str dimensions: List[Dimension] = [] + wall: Optional[Walls] = None + roof: Optional[Roofs] = None class PropertyDescription(BaseModel): built_form: str From a724c37c3af4ad782e0a2fe9dd2dc4ebb31593e9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 14 Mar 2025 16:31:20 +0000 Subject: [PATCH 2/8] doors finished --- etl/development.py | 2 +- etl/pdfReader/sitenotes.py | 48 +++++++++++++++++++++++++++++--------- etl/transform/types.py | 14 ++++++----- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/etl/development.py b/etl/development.py index 50d4749..e0f093a 100644 --- a/etl/development.py +++ b/etl/development.py @@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() -print(doc1.property_description.main_property) +print(doc1.property_description.door) # Transform diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 3ace9bc..068324f 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -2,7 +2,7 @@ from etl.pdfReader.reportType import ReportType from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, - Walls, Roofs, + Walls, Roofs, Floors, Door ) from datetime import datetime @@ -36,8 +36,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): """ self.transform_summary_information() self.transform_sections() - # self.get_section_9() - # self.get_section_10() # self.get_section_11() # self.get_section_12() # self.get_section_13() @@ -177,7 +175,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 3 age_bands = self.get_age_band() - + print() # Section 4 no_of_main_property = int(get_value("Main Property")) no_of_extension_1 = int(get_value('Extension 1') or 0) @@ -202,6 +200,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 8 roofs = self.get_roof() + # Section 9 + floors = self.get_floors() + + # Section 10 + door = self.get_door() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -221,26 +225,31 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): dimensions=dimensions["main"] if "main" in dimensions else [], wall=walls[0], roof=roofs[0], + floor=floors[0], )if no_of_main_property > 0 else None, ex1_property=PropertyDetail( age_band= age_bands[1], dimensions= dimensions["ex1"] if "ex1" in dimensions else [], wall=walls[1], roof=roofs[1], + floor=floors[1], )if no_of_extension_1 > 0 else None, ex2_property=PropertyDetail( age_band= age_bands[2], dimensions= dimensions["ex2"] if "ex2" in dimensions else [], wall=walls[2], roof=roofs[2], + floor=floors[2], )if no_of_extension_2 > 0 else None, ex3_property=PropertyDetail( age_band= age_bands[3], dimensions=dimensions["ex3"] if "ex3" in dimensions else [], wall=walls[3], roof=roofs[3], + floor=floors[3], )if no_of_extension_4 > 0 else None, conservatory=conservatory, + door=door, ) @@ -446,7 +455,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return group_data - def get_section_9(self): + def get_floors(self): data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')] sub_titles = [ "Floor Type", @@ -465,9 +474,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): ] - self.two_column_with_extension_processor(data, sub_titles, titles, 9) + lst = self.two_column_with_extension_processor(data, sub_titles, titles) + floors = [] + for floor in lst: + floors.append(Floors( + floor_type=floor.get("floor_type", ""), + ground_floor_construction=floor.get("get_floor_construction", ""), + ground_floor_insulation_type=floor.get("ground_floor_insulation_type", ""), + floor_insulation_thickness_mm=floor.get("floor_insulation_thickness_(mm)", -1) if floor.get("floor_insulation_thickness_(mm)", -1) is not None else -1, + u_value_known=True if floor.get("u_value_known").upper() == "YES" else False, + )) + return floors - def get_section_10(self): + def get_door(self): data = self.raw_data[self.raw_data.index("10.0 Doors"): self.raw_data.index("11.0 Windows")] avoid = [ "10.0 Doors", @@ -478,9 +497,15 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Number of Insulated Doors", "U-value (W/m²K)", ] - self.two_columns_processor(data, sub_titles, avoid, 10) + dict_ = self.two_columns_processor(data, sub_titles, avoid) - def two_columns_processor(self, data, sub_titles_to_gather, avoid, section, indexAdd = 1): + return Door( + no_of_doors=int(dict_.get("no_of_doors", -1)), + no_of_insulated_doors=int(dict_.get("number_of_insulated_doors", -1)), + u_value_w_m2_k=dict_.get("u_value_(w/m²k)", "") if dict_.get("u_value_(w/m²k), '')") is not None else "", + ) + + def two_columns_processor(self, data, sub_titles_to_gather, avoid, indexAdd = 1): def get_value(key): try: index = data.index(key) @@ -488,12 +513,13 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return None if value in avoid else value except (ValueError, IndexError): return None - + dict_ = {} for items in data: if items in avoid: continue elif items in sub_titles_to_gather: - setattr(self, f"section_{section}_{items.lower().replace('-', '_').replace(' ','_')}", get_value(items)) + dict_.update({f"{items.lower().replace('-', '_').replace(' ','_')}":get_value(items)}) + return dict_ def get_section_11(self): data = self.get_data_between("Window Location", "12.0 Ventilation & Cooling") diff --git a/etl/transform/types.py b/etl/transform/types.py index 7b2081e..880de6b 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -62,13 +62,13 @@ class Floors(BaseModel): floor_type: str ground_floor_construction: str ground_floor_insulation_type: str - floor_insulation_thickness_mm: Optional[float] + floor_insulation_thickness_mm: Optional[float] = -1 u_value_known: bool -class Doors(BaseModel): - no_of_doofs: int - nu_of_insulated_doors: int - u_value_w_m2_k: Optional[str] = None +class Door(BaseModel): + no_of_doors: int + no_of_insulated_doors: int + u_value_w_m2_k: Optional[str] class AssessorInfo(BaseModel): accreditation_number: str @@ -115,6 +115,7 @@ class PropertyDetail(BaseModel): dimensions: List[Dimension] = [] wall: Optional[Walls] = None roof: Optional[Roofs] = None + floor: Optional[Floors] = None class PropertyDescription(BaseModel): built_form: str @@ -135,4 +136,5 @@ class PropertyDescription(BaseModel): ex2_property: Optional[PropertyDetail] = None ex3_property: Optional[PropertyDetail] = None ex4_property: Optional[PropertyDetail] = None - conservatory: bool \ No newline at end of file + conservatory: bool + door: Optional[Door] \ No newline at end of file From 677ba42f2cc11aa463b59ca6f8caec122e7fce51 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 14:57:53 +0000 Subject: [PATCH 3/8] section 12 --- etl/pdfReader/sitenotes.py | 24 ++++++++++++++++++------ etl/transform/types.py | 5 +++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 068324f..4bb4547 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -2,7 +2,7 @@ from etl.pdfReader.reportType import ReportType from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, - Walls, Roofs, Floors, Door + Walls, Roofs, Floors, Door, VentilationAndCooling ) from datetime import datetime @@ -36,8 +36,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): """ self.transform_summary_information() self.transform_sections() + + # Saves windows till the end as that requires thought # self.get_section_11() - # self.get_section_12() + + + # start section 12, should just be dict_ # self.get_section_13() # self.get_section_14() # self.get_section_14_1() @@ -175,7 +179,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 3 age_bands = self.get_age_band() - print() + # Section 4 no_of_main_property = int(get_value("Main Property")) no_of_extension_1 = int(get_value('Extension 1') or 0) @@ -193,7 +197,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 5 conservatory = self.is_there_a_conservatory() - # Section 7 walls = self.get_walls() @@ -206,6 +209,9 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 10 door = self.get_door() + # Section 12 + ventilationAndCooling = self.get_ventilation_and_cooling() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -250,6 +256,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): )if no_of_extension_4 > 0 else None, conservatory=conservatory, door=door, + ventilationAndCooling=ventilationAndCooling, ) @@ -583,7 +590,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): lst.append(dict_) until = index + 3 + i - def get_section_12(self): + def get_ventilation_and_cooling(self): data = self.raw_data[self.raw_data.index('12.0 Ventilation & Cooling'): self.raw_data.index('13.0 Lighting')] avoid = [ '12.0 Ventilation & Cooling', @@ -595,7 +602,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Space Cooling System Present", ] - self.two_columns_processor(data, sub_titles, avoid, 12) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + return VentilationAndCooling( + no_of_open_fireplaces=int(dict_.get("number_of_open_fireplaces", -1)), + ventilation_type=str(dict_.get("ventilation_type", "")), + space_cooling_system_present=True if dict_.get("space_cooling_system_present").upper() == "YES" else False + ) def get_section_13(self): data = self.raw_data[self.raw_data.index('13.0 Lighting'): self.raw_data.index('14.0 Main Heating1')] diff --git a/etl/transform/types.py b/etl/transform/types.py index 880de6b..dfb6f33 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -84,7 +84,7 @@ class Windows(BaseModel): u_value_w_m2_k: int g_value: int -class FirePlaces(BaseModel): +class VentilationAndCooling(BaseModel): no_of_open_fireplaces: int ventilation_type: str space_cooling_system_present: bool @@ -137,4 +137,5 @@ class PropertyDescription(BaseModel): ex3_property: Optional[PropertyDetail] = None ex4_property: Optional[PropertyDetail] = None conservatory: bool - door: Optional[Door] \ No newline at end of file + door: Optional[Door] + ventilationAndCooling: Optional[VentilationAndCooling] \ No newline at end of file From 48eb492bdff1d9e97d461c7cafcb5124c67bab88 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 15:38:53 +0000 Subject: [PATCH 4/8] finsihed lighting --- etl/pdfReader/sitenotes.py | 15 ++++++++++++--- etl/transform/types.py | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 4bb4547..e6058e8 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -2,7 +2,8 @@ from etl.pdfReader.reportType import ReportType from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, - Walls, Roofs, Floors, Door, VentilationAndCooling + Walls, Roofs, Floors, Door, VentilationAndCooling, + Lighting ) from datetime import datetime @@ -212,6 +213,8 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 12 ventilationAndCooling = self.get_ventilation_and_cooling() + lighting = self.get_lighting() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -257,6 +260,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): conservatory=conservatory, door=door, ventilationAndCooling=ventilationAndCooling, + lighting=lighting, ) @@ -609,7 +613,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): space_cooling_system_present=True if dict_.get("space_cooling_system_present").upper() == "YES" else False ) - def get_section_13(self): + def get_lighting(self): data = self.raw_data[self.raw_data.index('13.0 Lighting'): self.raw_data.index('14.0 Main Heating1')] avoid = [ "13.0 Lighting", @@ -619,7 +623,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Total number of light fittings", "Total number of L.E.L. fittings", ] - self.two_columns_processor(data, sub_titles, avoid = avoid, section=13) + dict_ = self.two_columns_processor(data, sub_titles, avoid = avoid) + print(dict_) + return Lighting( + total_no_of_light_fittings=int(dict_["total_number_of_light_fittings"]), + total_no_of_lel_fittings=int(dict_["total_number_of_l.e.l._fittings"]), + ) def get_section_14(self): data = self.raw_data[self.raw_data.index('14.0 Main Heating1'): self.raw_data.index('14.1 Main Heating2')] diff --git a/etl/transform/types.py b/etl/transform/types.py index dfb6f33..96eb249 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -138,4 +138,5 @@ class PropertyDescription(BaseModel): ex4_property: Optional[PropertyDetail] = None conservatory: bool door: Optional[Door] - ventilationAndCooling: Optional[VentilationAndCooling] \ No newline at end of file + ventilationAndCooling: Optional[VentilationAndCooling] + lighting: Optional[Lighting] \ No newline at end of file From 23d83605fbb3670936e02bf089725e8718f21801 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 15:49:32 +0000 Subject: [PATCH 5/8] wrote to do list on whats left --- etl/development.py | 2 +- etl/pdfReader/sitenotes.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/etl/development.py b/etl/development.py index e0f093a..da99b31 100644 --- a/etl/development.py +++ b/etl/development.py @@ -32,7 +32,7 @@ if __name__ == "__main__": # Current todo list # - [x] Finish off scraping for section that I need to finish -# - [] Pydantic format for deemed report +# - [in progress] Pydantic format for deemed report # - [] Generate deemed report # - [] Docker compose to an sql database in docker compose (2 hours, then work on sql) # - [] Deploy via terraform to aws (1 day) \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index e6058e8..d82cbbd 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -41,12 +41,13 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Saves windows till the end as that requires thought # self.get_section_11() - - # start section 12, should just be dict_ - # self.get_section_13() + # Heaters require thought so will complete later # self.get_section_14() # self.get_section_14_1() # self.get_section_14_2() + + + # These one are quick fixes can be done on train or one at a time # self.get_section_15_0() # self.get_section_15_1() # self.get_section_16() From f0c02bd035fffd4c809bae979fec4de9895bded9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 16:49:44 +0000 Subject: [PATCH 6/8] wate rheating --- etl/pdfReader/sitenotes.py | 17 +++++++++++++---- etl/transform/types.py | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index d82cbbd..4db2fcc 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -3,7 +3,7 @@ from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, Walls, Roofs, Floors, Door, VentilationAndCooling, - Lighting + Lighting, WaterHeating ) from datetime import datetime @@ -48,7 +48,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # These one are quick fixes can be done on train or one at a time - # self.get_section_15_0() # self.get_section_15_1() # self.get_section_16() # self.get_section_17() @@ -214,8 +213,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 12 ventilationAndCooling = self.get_ventilation_and_cooling() + # Section 13 lighting = self.get_lighting() + # Section 15.0 Water Heating + waterHeating = self.get_water_heating() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -262,6 +265,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): door=door, ventilationAndCooling=ventilationAndCooling, lighting=lighting, + WaterHeating=waterHeating, ) @@ -688,7 +692,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): ] self.two_columns_processor(data, sub_titles, avoid, 14.2) - def get_section_15_0(self): + def get_water_heating(self): data = self.raw_data[self.raw_data.index("15.0 Water Heating"):self.raw_data.index("15.1 Hot Water Cylinder")] sub_titles = [ "Heating Type", @@ -698,7 +702,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "15.0 Water Heating", "15.1 Hot Water Cylinder", ] - self.two_columns_processor(data, sub_titles, avoid, 15.0) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + + return WaterHeating( + heating_type=dict_.get("heating_type", ""), + fuel_type=dict_.get("fuel_type", ""), + ) def get_section_15_1(self): data = self.raw_data[self.raw_data.index("15.1 Hot Water Cylinder"):self.raw_data.index("16.0 Solar Water Heating")] diff --git a/etl/transform/types.py b/etl/transform/types.py index 96eb249..93339a5 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -110,6 +110,10 @@ class MainHeating(BaseModel): model_qualifer: str controls: MainHeatingSystemControls +class WaterHeating(BaseModel): + heating_type: str + fuel_type: str + class PropertyDetail(BaseModel): age_band: str dimensions: List[Dimension] = [] From 4079c0303fb528d8676ed4dcdd1c60219c9e4666 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 17:00:13 +0000 Subject: [PATCH 7/8] finsish water heating --- etl/development.py | 2 +- etl/pdfReader/sitenotes.py | 2 +- etl/transform/types.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/etl/development.py b/etl/development.py index da99b31..1e26f5f 100644 --- a/etl/development.py +++ b/etl/development.py @@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() -print(doc1.property_description.door) +print(doc2.property_description.waterHeating) # Transform diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 4db2fcc..52fad14 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -265,7 +265,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): door=door, ventilationAndCooling=ventilationAndCooling, lighting=lighting, - WaterHeating=waterHeating, + waterHeating=waterHeating, ) diff --git a/etl/transform/types.py b/etl/transform/types.py index 93339a5..6d70c9e 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -143,4 +143,5 @@ class PropertyDescription(BaseModel): conservatory: bool door: Optional[Door] ventilationAndCooling: Optional[VentilationAndCooling] - lighting: Optional[Lighting] \ No newline at end of file + lighting: Optional[Lighting] + waterHeating: Optional[WaterHeating] \ No newline at end of file From 652afd5fd348d555c0436907fdd61c78f8feba9b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 16 Mar 2025 17:56:21 +0000 Subject: [PATCH 8/8] hot water cylinder added --- etl/pdfReader/sitenotes.py | 17 +++++++++++++---- etl/transform/types.py | 9 ++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 52fad14..a001d05 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -3,7 +3,7 @@ from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, Walls, Roofs, Floors, Door, VentilationAndCooling, - Lighting, WaterHeating + Lighting, WaterHeating, HotWaterCylinder, ) from datetime import datetime @@ -48,7 +48,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # These one are quick fixes can be done on train or one at a time - # self.get_section_15_1() # self.get_section_16() # self.get_section_17() # self.get_section_18() @@ -219,6 +218,9 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 15.0 Water Heating waterHeating = self.get_water_heating() + # Section 15.1 Hot Water Cylinder + hotWaterCylinder = self.get_hot_water_cylinder() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -266,6 +268,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): ventilationAndCooling=ventilationAndCooling, lighting=lighting, waterHeating=waterHeating, + hotWaterCylinder=hotWaterCylinder, ) @@ -709,7 +712,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): fuel_type=dict_.get("fuel_type", ""), ) - def get_section_15_1(self): + def get_hot_water_cylinder(self): data = self.raw_data[self.raw_data.index("15.1 Hot Water Cylinder"):self.raw_data.index("16.0 Solar Water Heating")] sub_tites = [ "Volume", @@ -721,7 +724,13 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "16.0 Solar Water Heating", "15.1 Hot Water Cylinder" ] - self.two_columns_processor(data, sub_tites, avoid, 15.1) + dict_ = self.two_columns_processor(data, sub_tites, avoid) + return HotWaterCylinder( + volume=dict_.get("volume", ""), + insulation_type=dict_.get("insulation_type", ""), + insulation_thickness=dict_.get("insulation_thickness", ""), + thermostat=True if dict_.get("thermostat", "NO").upper() == "YES" else False, + ) def get_section_16(self): data = self.raw_data[self.raw_data.index("16.0 Solar Water Heating"):self.raw_data.index("17.0 Waste Water Heat Recovery System")] diff --git a/etl/transform/types.py b/etl/transform/types.py index 6d70c9e..f83d9b4 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -114,6 +114,12 @@ class WaterHeating(BaseModel): heating_type: str fuel_type: str +class HotWaterCylinder(BaseModel): + volume: str + insulation_type: str + insulation_thickness: str + thermostat: bool + class PropertyDetail(BaseModel): age_band: str dimensions: List[Dimension] = [] @@ -144,4 +150,5 @@ class PropertyDescription(BaseModel): door: Optional[Door] ventilationAndCooling: Optional[VentilationAndCooling] lighting: Optional[Lighting] - waterHeating: Optional[WaterHeating] \ No newline at end of file + waterHeating: Optional[WaterHeating] + hotWaterCylinder: Optional[HotWaterCylinder] \ No newline at end of file