From 4d30ef039a18e58f6789643b0879991385aa7f06 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 12 Mar 2025 14:36:27 +0000 Subject: [PATCH] save current progress --- etl/main.py | 1 + etl/pdfReader/sitenotes.py | 150 +++++++++++++++++++++++++++++++++++-- 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/etl/main.py b/etl/main.py index 7d70ed7..c98d63c 100644 --- a/etl/main.py +++ b/etl/main.py @@ -13,6 +13,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1) doc2 = pdfReader.get_reader() pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() +vars(doc1) def main(): diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index b7cbfc5..1120bbe 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -1,6 +1,6 @@ from etl.pdfReader.reportType import ReportType -class SiteNotes(): +class SiteNotesExtractor(): def __init__(self, data_list): self.raw_data = data_list @@ -12,7 +12,7 @@ class SiteNotes(): -class QuidosSiteNotes(SiteNotes): +class QuidosSiteNotes(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.QUIDOS_SITE_NOTE @@ -32,6 +32,14 @@ class QuidosSiteNotes(SiteNotes): self.get_section_7() self.get_section_8() self.get_section_9() + self.get_section_10() + # self.get_section_11() + self.get_section_12() + self.get_section_13() + self.get_section_14() + self.get_section_14_1() + self.get_section_14_2() + self.get_section_15_0() def get_summary_information(self): # Summary Information @@ -282,6 +290,13 @@ class QuidosSiteNotes(SiteNotes): self.two_column_with_extension_processor(data, sub_titles, titles, 8) def two_column_with_extension_processor(self, data, sub_titles, main_titles, section): + def get_value(key): + try: + index = proc_data.index(key) + value = proc_data[index + 1] + return None if value in sub_titles else value + except (ValueError, IndexError): + return None title = None proc_data = data @@ -290,20 +305,18 @@ class QuidosSiteNotes(SiteNotes): title = items.lower().replace(" ", "_").replace("-", "_") index = main_titles.index(items) if main_titles[index] in data: - print(main_titles[index]) proc_data = data[data.index(main_titles[index]):] continue else: break if title is None: continue - get_value = lambda key: None if proc_data[proc_data.index(key) + 1] in sub_titles else proc_data[proc_data.index(key) + 1] if items in sub_titles: setattr(self, f"section_{section}_{title}_{items.lower().replace(' ', '_').replace('-','_')}", get_value(items)) def get_section_9(self): data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')] - avoid = [ + sub_titles = [ "Floor Type", "Ground Floor Construction", "Ground Floor Insulation Type", @@ -320,13 +333,134 @@ class QuidosSiteNotes(SiteNotes): ] - self.two_column_with_extension_processor(data, avoid, titles, 9) + self.two_column_with_extension_processor(data, sub_titles, titles, 9) + + def get_section_10(self): + data = self.raw_data[self.raw_data.index("10.0 Doors"): self.raw_data.index("11.0 Windows")] + avoid = [ + "10.0 Doors", + "11.0 Windows", + ] + sub_titles = [ + "Number of Doors", + "Number of Insulated Doors", + "U-value (W/m²K)", + ] + self.two_columns_processor(data, sub_titles, avoid, 10) + + def two_columns_processor(self, data, sub_titles_to_gather, avoid, section): + def get_value(key): + try: + index = data.index(key) + value = data[index + 1] + return None if value in avoid else value + except (ValueError, IndexError): + return None + + for items in data: + if items in avoid: + continue + elif items in sub_titles_to_gather: + setattr(self, f"section_{section}_{items.lower().replace('-', '_').replace(' ','_')}", get_value(items)) + def get_section_11(self): + raise RuntimeError("Please complete me") + def get_section_12(self): + data = self.raw_data[self.raw_data.index('12.0 Ventilation & Cooling'): self.raw_data.index('13.0 Lighting')] + avoid = [ + '12.0 Ventilation & Cooling', + '13.0 Lighting' + ] + sub_titles = [ + "Number of Open Fireplaces", + "Ventilation Type", + "Space Cooling System Present", + ] -# validatin -# function of object of type this + self.two_columns_processor(data, sub_titles, avoid, 12) + def get_section_13(self): + data = self.raw_data[self.raw_data.index('13.0 Lighting'): self.raw_data.index('14.0 Main Heating1')] + avoid = [ + "13.0 Lighting", + "14.0 Main Heating1" + ] + sub_titles = [ + "Total number of light fittings", + "Total number of L.E.L. fittings", + ] + self.two_columns_processor(data, sub_titles, avoid = avoid, section=13) + def get_section_14(self): + data = self.raw_data[self.raw_data.index('14.0 Main Heating1'): self.raw_data.index('14.1 Main Heating2')] + main_titles = [ + "Main Heating Type", + "Product Database" + "Main Heating System Controls", + ] + sub_titles = [ + "Heating Source", + "Efficiency Source", + "Heating Fuel", + "Brand Name", + "Model Name", + "Model Qualifier", + "Control Type", + "Flue Type", + "Fan Assisted Flue", + "Heat Emitter Type", + "Electricity Meter Type", + "Mains Gas Available", + ] + self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.0) + def get_section_14_1(self): + data = self.raw_data[self.raw_data.index("14.1 Main Heating2"):self.raw_data.index("14.2 Secondary Heating Type")] + main_titles = [ + "Second Main Heating Type", + "Main Heating System Controls", + ] + sub_titles = [ + "Percentage of Heated Floor Area Served (%)", + "Heating Source", + "Efficiency Source", + "Heating Fuel", + "SAP 2009 Table 4a/4b", + "Heating Type", + "Heating Description", + "Control Type", + "Flue Type", + "Fan Assisted Flue", + "Heat Emitter Type", + ] + self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.1) + + def get_section_14_2(self): + data = self.raw_data[self.raw_data.index("14.2 Secondary Heating Type"):self.raw_data.index("15.0 Water Heating")] + avoid = [ + "14.2 Secondary Heating Type", + "15.0 Water Heating", + ] + sub_titles = [ + "Heating Type", + "Fuel Type", + ] + self.two_columns_processor(data, sub_titles, avoid, 14.2) + + def get_section_15_0(self): + data = self.raw_data[self.raw_data.index("15.0 Water Heating"):self.raw_data.index("15.1 Hot Water Cylinder")] + sub_titles = [ + "Heating Type", + "Fuel Type", + ] + avoid = [ + "15.0 Water Heating", + "15.1 Hot Water Cylinder", + ] + self.two_columns_processor(data, sub_titles, avoid, 15.0) + +# Extract +# Transform ( wiht validation pydantnic) +# Load \ No newline at end of file