junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

save progress - finished roofs and added more sectios

Browse source
This commit is contained in:
Jun-te Kim 2025-03-14 12:32:53 +00:00
parent df9d5fd49a
commit ba6cf1402d
3 changed files with 133 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
etl/development.py
View file

@ -15,7 +15,7 @@ pdfReader = pdfReaderToText(DATA_LOC_1)
doc2 = pdfReader.get_reader()
pdfReader2 = pdfReaderToText(DATA_LOC_2)
doc1 = pdfReader2.get_reader()
print(doc1.survey_information)
print(doc1.property_description.main_property)
# Transform

81
etl/pdfReader/sitenotes.py
View file

@ -1,5 +1,9 @@
from etl.pdfReader.reportType import ReportType
from transform.types import CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension
from transform.types import (
CompanyInfo, SurverySummaryInfo, AssessorInfo,
PropertyDescription, PropertyDetail, Dimension,
Walls, Roofs,
)
from datetime import datetime
class SiteNotesExtractor():
@ -32,8 +36,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
"""
self.transform_summary_information()
self.transform_sections()
# self.get_section_7()
# self.get_section_8()
# self.get_section_9()
# self.get_section_10()
# self.get_section_11()
@ -155,7 +157,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
# Section 2
data = self.raw_data[self.raw_data.index("2.0 Number Of"):self.raw_data.index("3.0 Date Built")]
data = self.get_data_between("2.0 Number Of","3.0 Date Built")
avoid = [
'2.0 Number Of',
@ -193,6 +195,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
# Section 5
conservatory = self.is_there_a_conservatory()
# Section 7
walls = self.get_walls()
# Section 8
roofs = self.get_roof()
self.property_description = PropertyDescription(
built_form = get_value("Built Form"),
@ -209,20 +217,28 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
terrain_type = get_value('Terrain Type'),
percentage_of_draught_proofed= int(get_value('Percentage of Draught Proofed(%)')),
main_property=PropertyDetail(
age_band= age_bands[0],
dimensions= dimensions["main"] if "main" in dimensions else [],
age_band=age_bands[0],
dimensions=dimensions["main"] if "main" in dimensions else [],
wall=walls[0],
roof=roofs[0],
)if no_of_main_property > 0 else None,
ex1_property=PropertyDetail(
age_band= age_bands[1],
dimensions= dimensions["ex1"] if "ex1" in dimensions else [],
wall=walls[1],
roof=roofs[1],
)if no_of_extension_1 > 0 else None,
ex2_property=PropertyDetail(
age_band= age_bands[2],
dimensions= dimensions["ex2"] if "ex2" in dimensions else [],
wall=walls[2],
roof=roofs[2],
)if no_of_extension_2 > 0 else None,
ex3_property=PropertyDetail(
age_band= age_bands[3],
dimensions=dimensions["ex3"] if "ex3" in dimensions else [],
wall=walls[3],
roof=roofs[3],
)if no_of_extension_4 > 0 else None,
conservatory=conservatory,
)
@ -320,7 +336,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
def get_section_6(self):
pass
def get_section_7(self):
def get_walls(self):
data = self.get_data_between('7.0 Walls','8.0 Roofs')
sub_titles = [
@ -345,9 +361,27 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
"Extension 3",
"Extension 4",
]
self.two_column_with_extension_processor(data, sub_titles, main_titles, 7)
lst = self.two_column_with_extension_processor(data, sub_titles, main_titles)
walls = []
for ext in lst:
walls.append(Walls(
construction=ext["construction"] if ext["construction"] is not None else "",
insulation=str(ext["insulation"]),
insulation_thickness_mm=str(ext["insulation_thickness(mm)"]),
wall_thickness_measured=True if ext["wall_thickness_measured"].upper() == "YES" else False,
wall_thickness_mm=int(ext.get("wall_thickness(mm)",-1)),
u_value_known=True if ext["u_value_known"].upper() == "YES" else False,
u_value_w_m2_k=float(ext.get("u_value_(w/m²k)", -1.0)),
dry_lining=True if ext.get("dry_lining", "NO").upper() == "YES" else False,
alternative_wall_present=True if ext.get("alternative_wall_present", "NO").upper() == "YES" else False,
))
return walls
def get_section_8(self):
def get_roof(self):
data = self.raw_data[self.raw_data.index('8.0 Roofs'): self.raw_data.index('9.0 Floors')]
sub_titles = [
"Construction",
@ -364,9 +398,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
"Extension 4",
]
self.two_column_with_extension_processor(data, sub_titles, titles, 8)
lst = self.two_column_with_extension_processor(data, sub_titles, titles)
roofs = []
for roof in lst:
roofs.append(Roofs(
construction=roof["construction"] if roof["construction"] is not None else "",
insulation_type=str(roof["insulation_type"]),
insulation_thickness=str(roof["insulation_thickness"]),
u_value_known=True if roof["u_value_known"].upper() == "YES" else False,
))
return roofs
def two_column_with_extension_processor(self, data, sub_titles, main_titles, section):
def two_column_with_extension_processor(self, data, sub_titles, main_titles):
def get_value(key):
try:
index = proc_data.index(key)
@ -374,22 +418,33 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
return None if value in sub_titles else value
except (ValueError, IndexError):
return None
title = None
proc_data = data
return_dict = {}
group_data = []
for items in data:
if items in main_titles:
title = items.lower().replace(" ", "_").replace("-", "_")
index = main_titles.index(items)
if main_titles[index] in data:
if return_dict:
group_data.append(return_dict)
return_dict = {}
proc_data = data[data.index(main_titles[index]):]
continue
else:
if return_dict:
group_data.append(return_dict)
break
if title is None:
continue
if items in sub_titles:
setattr(self, f"section_{section}_{title}_{items.lower().replace(' ', '_').replace('-','_')}", get_value(items))
return_dict.update({f"{items.lower().replace("?", "").replace(' ', '_').replace('-','_')}": get_value(items)})
if return_dict:
group_data.append(return_dict)
return group_data
def get_section_9(self):
data = self.raw_data[self.raw_data.index('9.0 Floors'): self.raw_data.index('10.0 Doors')]

64
etl/transform/types.py
View file

@ -41,6 +41,34 @@ class SurverySummaryInfo(BaseModel):
current_annual_emission_including_0925_multiplayer: str
current_annual_energy_costs: str
class Walls(BaseModel):
construction: str
insulation: str
insulation_thickness_mm: str
wall_thickness_measured: bool
wall_thickness_mm: Optional[int]
u_value_known: bool
u_value_w_m2_k: Optional[float]
dry_lining: bool
alternative_wall_present: bool
class Roofs(BaseModel):
construction: str
insulation_type: str
insulation_thickness: str
u_value_known: bool
class Floors(BaseModel):
floor_type: str
ground_floor_construction: str
ground_floor_insulation_type: str
floor_insulation_thickness_mm: Optional[float]
u_value_known: bool
class Doors(BaseModel):
no_of_doofs: int
nu_of_insulated_doors: int
u_value_w_m2_k: Optional[str] = None
class AssessorInfo(BaseModel):
accreditation_number: str
@ -48,9 +76,45 @@ class AssessorInfo(BaseModel):
phone_number: Optional[str] = None
email_address: Optional[EmailStr] = None
class Windows(BaseModel):
glazing_type: str
area_m2: float
roof_window: bool
orientation: str
u_value_w_m2_k: int
g_value: int
class FirePlaces(BaseModel):
no_of_open_fireplaces: int
ventilation_type: str
space_cooling_system_present: bool
class Lighting(BaseModel):
total_no_of_light_fittings: int
total_no_of_lel_fittings: int
class MainHeatingSystemControls(BaseModel):
control_type: str
flue_type: str
fan_assisted_flue: str
heat_emitter_type: str
electricity_meter_type: Optional[str]
mains_gas_available: Optional[bool]
class MainHeating(BaseModel):
heating_source: str
efficiency_source: str
heating_fuel: str
brand_name: str
model_name: str
model_qualifer: str
controls: MainHeatingSystemControls
class PropertyDetail(BaseModel):
age_band: str
dimensions: List[Dimension] = []
wall: Optional[Walls] = None
roof: Optional[Roofs] = None
class PropertyDescription(BaseModel):
built_form: str