mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
save current progress
This commit is contained in:
parent
2fbc330c7c
commit
4d30ef039a
2 changed files with 143 additions and 8 deletions
|
|
@ -13,6 +13,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1)
|
|||
doc2 = pdfReader.get_reader()
|
||||
pdfReader2 = pdfReaderToText(DATA_LOC_2)
|
||||
doc1 = pdfReader2.get_reader()
|
||||
vars(doc1)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from etl.pdfReader.reportType import ReportType
|
||||
|
||||
class SiteNotes():
|
||||
class SiteNotesExtractor():
|
||||
def __init__(self, data_list):
|
||||
self.raw_data = data_list
|
||||
|
||||
|
|
@ -12,7 +12,7 @@ class SiteNotes():
|
|||
|
||||
|
||||
|
||||
class QuidosSiteNotes(SiteNotes):
|
||||
class QuidosSiteNotes(SiteNotesExtractor):
|
||||
def __init__(self, data_list):
|
||||
super().__init__(data_list)
|
||||
self.type = ReportType.QUIDOS_SITE_NOTE
|
||||
|
|
@ -32,6 +32,14 @@ class QuidosSiteNotes(SiteNotes):
|
|||
self.get_section_7()
|
||||
self.get_section_8()
|
||||
self.get_section_9()
|
||||
self.get_section_10()
|
||||
# self.get_section_11()
|
||||
self.get_section_12()
|
||||
self.get_section_13()
|
||||
self.get_section_14()
|
||||
self.get_section_14_1()
|
||||
self.get_section_14_2()
|
||||
self.get_section_15_0()
|
||||
|
||||
def get_summary_information(self):
|
||||
# Summary Information
|
||||
|
|
@ -282,6 +290,13 @@ class QuidosSiteNotes(SiteNotes):
|
|||
self.two_column_with_extension_processor(data, sub_titles, titles, 8)
|
||||
|
||||
def two_column_with_extension_processor(self, data, sub_titles, main_titles, section):
|
||||
def get_value(key):
|
||||
try:
|
||||
index = proc_data.index(key)
|
||||
value = proc_data[index + 1]
|
||||
return None if value in sub_titles else value
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
title = None
|
||||
proc_data = data
|
||||
|
|
@ -290,20 +305,18 @@ class QuidosSiteNotes(SiteNotes):
|
|||
title = items.lower().replace(" ", "_").replace("-", "_")
|
||||
index = main_titles.index(items)
|
||||
if main_titles[index] in data:
|
||||
print(main_titles[index])
|
||||
proc_data = data[data.index(main_titles[index]):]
|
||||
continue
|
||||
else:
|
||||
break
|
||||
if title is None:
|
||||
continue
|
||||
get_value = lambda key: None if proc_data[proc_data.index(key) + 1] in sub_titles else proc_data[proc_data.index(key) + 1]
|
||||
if items in sub_titles:
|
||||
setattr(self, f"section_{section}_{title}_{items.lower().replace(' ', '_').replace('-','_')}", get_value(items))
|
||||
|
||||
def get_section_9(self):
|
||||
data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')]
|
||||
avoid = [
|
||||
sub_titles = [
|
||||
"Floor Type",
|
||||
"Ground Floor Construction",
|
||||
"Ground Floor Insulation Type",
|
||||
|
|
@ -320,13 +333,134 @@ class QuidosSiteNotes(SiteNotes):
|
|||
]
|
||||
|
||||
|
||||
self.two_column_with_extension_processor(data, avoid, titles, 9)
|
||||
self.two_column_with_extension_processor(data, sub_titles, titles, 9)
|
||||
|
||||
def get_section_10(self):
|
||||
data = self.raw_data[self.raw_data.index("10.0 Doors"): self.raw_data.index("11.0 Windows")]
|
||||
avoid = [
|
||||
"10.0 Doors",
|
||||
"11.0 Windows",
|
||||
]
|
||||
sub_titles = [
|
||||
"Number of Doors",
|
||||
"Number of Insulated Doors",
|
||||
"U-value (W/m²K)",
|
||||
]
|
||||
self.two_columns_processor(data, sub_titles, avoid, 10)
|
||||
|
||||
def two_columns_processor(self, data, sub_titles_to_gather, avoid, section):
|
||||
def get_value(key):
|
||||
try:
|
||||
index = data.index(key)
|
||||
value = data[index + 1]
|
||||
return None if value in avoid else value
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
for items in data:
|
||||
if items in avoid:
|
||||
continue
|
||||
elif items in sub_titles_to_gather:
|
||||
setattr(self, f"section_{section}_{items.lower().replace('-', '_').replace(' ','_')}", get_value(items))
|
||||
|
||||
def get_section_11(self):
|
||||
raise RuntimeError("Please complete me")
|
||||
|
||||
def get_section_12(self):
|
||||
data = self.raw_data[self.raw_data.index('12.0 Ventilation & Cooling'): self.raw_data.index('13.0 Lighting')]
|
||||
avoid = [
|
||||
'12.0 Ventilation & Cooling',
|
||||
'13.0 Lighting'
|
||||
]
|
||||
sub_titles = [
|
||||
"Number of Open Fireplaces",
|
||||
"Ventilation Type",
|
||||
"Space Cooling System Present",
|
||||
]
|
||||
|
||||
# validatin
|
||||
# function of object of type this
|
||||
self.two_columns_processor(data, sub_titles, avoid, 12)
|
||||
|
||||
def get_section_13(self):
|
||||
data = self.raw_data[self.raw_data.index('13.0 Lighting'): self.raw_data.index('14.0 Main Heating1')]
|
||||
avoid = [
|
||||
"13.0 Lighting",
|
||||
"14.0 Main Heating1"
|
||||
]
|
||||
sub_titles = [
|
||||
"Total number of light fittings",
|
||||
"Total number of L.E.L. fittings",
|
||||
]
|
||||
self.two_columns_processor(data, sub_titles, avoid = avoid, section=13)
|
||||
|
||||
def get_section_14(self):
|
||||
data = self.raw_data[self.raw_data.index('14.0 Main Heating1'): self.raw_data.index('14.1 Main Heating2')]
|
||||
main_titles = [
|
||||
"Main Heating Type",
|
||||
"Product Database"
|
||||
"Main Heating System Controls",
|
||||
]
|
||||
sub_titles = [
|
||||
"Heating Source",
|
||||
"Efficiency Source",
|
||||
"Heating Fuel",
|
||||
"Brand Name",
|
||||
"Model Name",
|
||||
"Model Qualifier",
|
||||
"Control Type",
|
||||
"Flue Type",
|
||||
"Fan Assisted Flue",
|
||||
"Heat Emitter Type",
|
||||
"Electricity Meter Type",
|
||||
"Mains Gas Available",
|
||||
]
|
||||
|
||||
self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.0)
|
||||
|
||||
def get_section_14_1(self):
|
||||
data = self.raw_data[self.raw_data.index("14.1 Main Heating2"):self.raw_data.index("14.2 Secondary Heating Type")]
|
||||
main_titles = [
|
||||
"Second Main Heating Type",
|
||||
"Main Heating System Controls",
|
||||
]
|
||||
sub_titles = [
|
||||
"Percentage of Heated Floor Area Served (%)",
|
||||
"Heating Source",
|
||||
"Efficiency Source",
|
||||
"Heating Fuel",
|
||||
"SAP 2009 Table 4a/4b",
|
||||
"Heating Type",
|
||||
"Heating Description",
|
||||
"Control Type",
|
||||
"Flue Type",
|
||||
"Fan Assisted Flue",
|
||||
"Heat Emitter Type",
|
||||
]
|
||||
self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.1)
|
||||
|
||||
def get_section_14_2(self):
|
||||
data = self.raw_data[self.raw_data.index("14.2 Secondary Heating Type"):self.raw_data.index("15.0 Water Heating")]
|
||||
avoid = [
|
||||
"14.2 Secondary Heating Type",
|
||||
"15.0 Water Heating",
|
||||
]
|
||||
sub_titles = [
|
||||
"Heating Type",
|
||||
"Fuel Type",
|
||||
]
|
||||
self.two_columns_processor(data, sub_titles, avoid, 14.2)
|
||||
|
||||
def get_section_15_0(self):
|
||||
data = self.raw_data[self.raw_data.index("15.0 Water Heating"):self.raw_data.index("15.1 Hot Water Cylinder")]
|
||||
sub_titles = [
|
||||
"Heating Type",
|
||||
"Fuel Type",
|
||||
]
|
||||
avoid = [
|
||||
"15.0 Water Heating",
|
||||
"15.1 Hot Water Cylinder",
|
||||
]
|
||||
self.two_columns_processor(data, sub_titles, avoid, 15.0)
|
||||
|
||||
# Extract
|
||||
# Transform ( wiht validation pydantnic)
|
||||
# Load
|
||||
Loading…
Add table
Reference in a new issue