From 867f086e1e487a860bc84f6cff96fdcad9fc6d40 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 17 Jun 2025 10:30:59 +0000 Subject: [PATCH 01/22] condition report saved --- etl/pdfReader/sitenotes.py | 51 ++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index d3d74f8..568d0e6 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -14,7 +14,8 @@ from etl.transform.conditionReportTypes import ( ExternalElevationGableOne, ExternalElevationGableTwo, ExternalElevationRear, ConservatoryOrOutbuilding, AccessAndElevations, Hallway, RoomInfo, WindowsInfo, VentilationInfo, LivingRoom, DiningRoom, Kitchen, Rooms, Utility, WC, Landing, Bedroom, Bathroom, LoftSpace, RoomInRoof, HeatingSystem, GeneralConditionHeatingSystem, - MainHeatingOne, MainHeatingTwo, SecondaryHeating, HeatingByRoom, Renewables, Occupant, EnergyUse, Heating, ShowerAndBath, FridgeAndFreezers, Cooker, TumbleDryer + MainHeatingOne, MainHeatingTwo, SecondaryHeating, HeatingByRoom, Renewables, Occupant, EnergyUse, Heating, ShowerAndBath, FridgeAndFreezers, Cooker, TumbleDryer, + GeneralInformation, OccupantAssessment ) from datetime import datetime from pprint import pprint @@ -92,25 +93,29 @@ class ConditionReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT - self.setup_condition_report() + _ = self.setup_condition_report() + pprint(_) def setup_condition_report(self): - assesor_details, inspection_and_project, the_property, main_elevation, elevations = self.get_section_1() + general_information = self.get_section_1() access_and_elevations = self.get_section_2() rooms = self.get_section_3() - general_condition_of_heating_system, main_heating_one, main_heating_two, secondary_heating, heating_by_room, renewables = self.get_section_4() - occupants, energy_use, heating, shower_and_bath, appliances, fridge_and_freezers, cooker, tumble_dryer = self.get_section_5() + heating_system = self.get_section_4() + occupant_assessment = self.get_section_5() site_name, reference_code, address, postcode = self.get_section_0() + return ConditionReportModel( project_site_name=site_name, property_reference_code=reference_code, property_address=address, postcode=postcode, - - + general_information=general_information, + access_and_elevations=access_and_elevations, + rooms=rooms, + heating_system=heating_system, + occupancy_assessment=occupant_assessment, ) - def get_section_0(self): data = self.get_data_between("Project Site Name", "1. General Information") site_name = self.get_next_value(data, "Project Site Name") @@ -127,7 +132,14 @@ class ConditionReport(SiteNotesExtractor): the_property = self.get_the_property() main_elevation = self.get_main_elevation() elevations = self.get_all_elevations() - return assessor_details, inspection_and_project, the_property, main_elevation, elevations + + return GeneralInformation( + assessor_details=assessor_details, + inspection_and_project=inspection_and_project, + the_property=the_property, + main_elevation=main_elevation, + elevations=elevations + ) def get_assessor_details(self): data = self.get_data_between("1.1 Assessor details","1.2 Inspection & Project") @@ -501,7 +513,15 @@ class ConditionReport(SiteNotesExtractor): secondary_heating = self.get_secondary_heating() heating_by_room = self.get_heating_by_room() renewables = self.get_renewables() - return general_condition_of_heating_system, main_heating_one, main_heating_two, secondary_heating, heating_by_room, renewables + + return HeatingSystem( + general_condition=general_condition_of_heating_system, + main_heating_one=main_heating_one, + main_heating_two=main_heating_two, + secondary_heating=secondary_heating, + heating_by_room=heating_by_room, + renewables=renewables + ) def get_main_heating_one(self): data = self.get_data_between("Main Heating 1", "Main Heating 2") @@ -589,7 +609,16 @@ class ConditionReport(SiteNotesExtractor): fridge_and_freezers = self.get_fridge_and_freezers() cooker = self.get_cooker() tumble_dryer = self.get_tumble_dryer() - return occupants, energy_use, heating, shower_and_bath, appliances, fridge_and_freezers, cooker, tumble_dryer + return OccupantAssessment( + occupant=occupants, + energy_use=energy_use, + heating=heating, + shower_and_bath=shower_and_bath, + appliances=appliances, + fridge_and_freezers=fridge_and_freezers, + cooker=cooker, + tumble_dryer=tumble_dryer + ) def get_occupants(self): data = self.get_data_between("Occupants", "Energy use") From 486de729fbaa3168224ec70554c8408fa768f63e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 17 Jun 2025 10:39:42 +0000 Subject: [PATCH 02/22] change name to models --- alembic/env.py | 4 ++-- etl/{load => models}/preSiteNoteTypes.py | 2 +- etl/{load => models}/topLevel.py | 0 etl/surveyedData/surveryedData.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename etl/{load => models}/preSiteNoteTypes.py (99%) rename etl/{load => models}/topLevel.py (100%) diff --git a/alembic/env.py b/alembic/env.py index d915dc1..172387d 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,8 +5,8 @@ from sqlalchemy import pool from alembic import context from sqlmodel import SQLModel -from etl.load.topLevel import * -from etl.load.preSiteNoteTypes import * +from etl.models.topLevel import * +from etl.models.preSiteNoteTypes import * import os diff --git a/etl/load/preSiteNoteTypes.py b/etl/models/preSiteNoteTypes.py similarity index 99% rename from etl/load/preSiteNoteTypes.py rename to etl/models/preSiteNoteTypes.py index 054dda4..55392af 100644 --- a/etl/load/preSiteNoteTypes.py +++ b/etl/models/preSiteNoteTypes.py @@ -5,7 +5,7 @@ from datetime import datetime from pydantic import EmailStr from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID -from etl.load.topLevel import BaseModel, Documents +from etl.models.topLevel import BaseModel, Documents class PreSiteNote(BaseModel, table=True): diff --git a/etl/load/topLevel.py b/etl/models/topLevel.py similarity index 100% rename from etl/load/topLevel.py rename to etl/models/topLevel.py diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 8525278..1c8e358 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,7 +1,7 @@ from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.pdfReader.reportType import ReportType import math -from etl.load.preSiteNoteTypes import ( +from etl.models.preSiteNoteTypes import ( AssessorInfo, CompanyInfo, PreSiteNotesSummaryInfo, PreSiteNote, @@ -10,7 +10,7 @@ from etl.load.preSiteNoteTypes import ( SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling, Door, Walls, Roofs, Floors, PropertyDetail, Windows ) -from etl.load.topLevel import( +from etl.models.topLevel import( Buildings, Documents ) import uuid From a9a479cf982c8948c48c4feaf2135a98b8cec585 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 17 Jun 2025 11:00:08 +0000 Subject: [PATCH 03/22] migrate and save new db structure --- alembic/env.py | 1 + ...6c4ea39b29_condition_table_from_osmosis.py | 34 ++++++ etl/models/conditionReport.py | 113 ++++++++++++++++++ migration_db.sh | 4 +- 4 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/956c4ea39b29_condition_table_from_osmosis.py create mode 100644 etl/models/conditionReport.py diff --git a/alembic/env.py b/alembic/env.py index 172387d..bbecde1 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -7,6 +7,7 @@ from sqlmodel import SQLModel from etl.models.topLevel import * from etl.models.preSiteNoteTypes import * +from etl.models.conditionReport import * import os diff --git a/alembic/versions/956c4ea39b29_condition_table_from_osmosis.py b/alembic/versions/956c4ea39b29_condition_table_from_osmosis.py new file mode 100644 index 0000000..d4b904a --- /dev/null +++ b/alembic/versions/956c4ea39b29_condition_table_from_osmosis.py @@ -0,0 +1,34 @@ +"""condition table from osmosis + +Revision ID: 956c4ea39b29 +Revises: 427e65da69c1 +Create Date: 2025-06-17 10:49:55.213111 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '956c4ea39b29' +down_revision: Union[str, None] = '427e65da69c1' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_foreign_key(None, 'documents', 'assessorinfo', ['assessor_id'], ['id']) + op.create_foreign_key(None, 'presitenote', 'assessorinfo', ['assessor_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'presitenote', type_='foreignkey') + op.drop_constraint(None, 'documents', type_='foreignkey') + # ### end Alembic commands ### diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py new file mode 100644 index 0000000..777a065 --- /dev/null +++ b/etl/models/conditionReport.py @@ -0,0 +1,113 @@ +# SQLModel mapping for ConditionReportModel using BaseModel +from sqlmodel import SQLModel, Field, Relationship +from typing import Optional, List +import uuid +from datetime import datetime + +class BaseModel(SQLModel): + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + + +class AssessorDetails(BaseModel, table=True): + assessor_name_and_id: str + elmhurst_id: str + + +class InspectionAndProject(BaseModel, table=True): + inspection_date: str + + +class TheProperty(BaseModel, table=True): + house_type: str + on_which_floor_is_the_flat_located: str + is_there_a_corridor: bool + is_it_heated: bool + it_there_a_balcony: bool + classification_type: str + orientation_front_elevation: str + orientation_in_degrees_front_elevation: str + exposure_zone: str + main_wall_construction: str + + +class ElevationInfo(BaseModel, table=True): + elevation_type: str + cavity_wall_depth: str + is_insulation_present: bool + insulation_type: str + + +class Elevation(BaseModel, table=True): + protected_conservatory_or_aonb: bool + material_type: str + are_there_any_visible_signs_of_existing_wall_insulation: str + does_the_existing_ground_level_on_any_elevation_bridge_the_dpc: bool + + +class PropertyAccess(BaseModel, table=True): + are_there_any_road_restriction_in_the_locality: bool + is_on_street_parking_available: bool + are_there_any_overhead_wires_or_cables: bool + is_the_access_gated: bool + is_there_restricted_space_for_contractors_to_access_the_wall_area: bool + is_there_restricted_space_for_contractors_to_access_the_roof_area: bool + is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation: bool + is_access_to_the_rear_provided_by_use_of_a_ginnel: bool + is_access_to_the_rear_provided_by_use_of_a_secured_alleyway: bool + + +class ExternalElevation(BaseModel, table=True): + structural_defects_of_elevation: str + does_any_structural_defect_need_resolving_before_retrofit: bool + are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework: bool + are_there_any_visible_signs_of_movement: bool + are_there_any_visible_signs_of_cracking_to_the_existing_external_finish: bool + + +class ConservatoryOrOutbuilding(BaseModel, table=True): + is_there_a_conservatory: bool + is_there_a_cellar_present: bool + is_there_an_outbuilding: bool + + +class GeneralConditionHeatingSystem(BaseModel, table=True): + is_the_heating_system_in_working_order: bool + does_the_occupant_have_a_smart_meter: bool + are_there_any_smart_monitoring_devices: bool + are_the_gas_and_electricity_meters_accessible: bool + dual_or_single_electric_meter: str + + +class SecondaryHeating(BaseModel, table=True): + is_there_a_secondary_heating: bool + fuel: str + electric_heating_type: str + gas_heating_type: str + + +class MainHeatingOne(BaseModel, table=True): + as_defined_by: str + fuel: str + type: str + + +class MainHeatingTwo(BaseModel, table=True): + is_there_a_main_heating_two: bool + + +class Heating(BaseModel, table=True): + room_stat_in_temperature_in_celsius: Optional[str] = None + room_stat_location: Optional[str] = None + is_the_heating_pattern_known: Optional[str] = None + + +class ConditionReport(BaseModel, table=True): + project_site_name: str + property_reference_code: str + property_address: str + postcode: str + + assessor_details_id: Optional[uuid.UUID] = Field(default=None, foreign_key="assessordetails.id") + inspection_id: Optional[uuid.UUID] = Field(default=None, foreign_key="inspectionandproject.id") + property_id: Optional[uuid.UUID] = Field(default=None, foreign_key="theproperty.id") + general_condition_id: Optional[uuid.UUID] = Field(default=None, foreign_key="generalconditionheatingsystem.id") diff --git a/migration_db.sh b/migration_db.sh index 08c8719..e8cadeb 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,3 +1,3 @@ -#poetry run alembic revision --autogenerate -m "Initial table" +poetry run alembic revision --autogenerate -m "condition table from osmosis added as i didn't add" -poetry run alembic upgrade head +#poetry run alembic upgrade head From 1eef91e63bff2ff4fe048731878eb62d960e88c4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 17 Jun 2025 12:47:28 +0000 Subject: [PATCH 04/22] added more logs and visibility for migration --- .../things_alembic_doesn't_automate_for.txt | 19 ++ ...ondition_table_from_osmosis_added_as_i_.py | 166 ++++++++++++++++++ etl/models/conditionReport.py | 1 + migration_db.sh | 4 +- 4 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 alembic/things_alembic_doesn't_automate_for.txt create mode 100644 alembic/versions/42a19efef660_condition_table_from_osmosis_added_as_i_.py diff --git a/alembic/things_alembic_doesn't_automate_for.txt b/alembic/things_alembic_doesn't_automate_for.txt new file mode 100644 index 0000000..b4f20e7 --- /dev/null +++ b/alembic/things_alembic_doesn't_automate_for.txt @@ -0,0 +1,19 @@ +Source: https://alembic.sqlalchemy.org/en/latest/autogenerate.html + +Autogenerate can not detect: + + Changes of table name. These will come out as an add/drop of two different tables, and should be hand-edited into a name change instead. + + Changes of column name. Like table name changes, these are detected as a column add/drop pair, which is not at all the same as a name change. + + Anonymously named constraints. Give your constraints a name, e.g. UniqueConstraint('col1', 'col2', name="my_name"). See the section The Importance of Naming Constraints for background on how to configure automatic naming schemes for constraints. + + Special SQLAlchemy types such as Enum when generated on a backend which doesn’t support ENUM directly - this because the representation of such a type in the non-supporting database, i.e. a CHAR+ CHECK constraint, could be any kind of CHAR+CHECK. For SQLAlchemy to determine that this is actually an ENUM would only be a guess, something that’s generally a bad idea. To implement your own “guessing” function here, use the sqlalchemy.events.DDLEvents.column_reflect() event to detect when a CHAR (or whatever the target type is) is reflected, and change it to an ENUM (or whatever type is desired) if it is known that that’s the intent of the type. The sqlalchemy.events.DDLEvents.after_parent_attach() can be used within the autogenerate process to intercept and un-attach unwanted CHECK constraints. + + + +Autogenerate can’t currently, but will eventually detect: + + Some free-standing constraint additions and removals may not be supported, including PRIMARY KEY, EXCLUDE, CHECK; these are not necessarily implemented within the autogenerate detection system and also may not be supported by the supporting SQLAlchemy dialect. + + Sequence additions, removals - not yet implemented. diff --git a/alembic/versions/42a19efef660_condition_table_from_osmosis_added_as_i_.py b/alembic/versions/42a19efef660_condition_table_from_osmosis_added_as_i_.py new file mode 100644 index 0000000..d3fc585 --- /dev/null +++ b/alembic/versions/42a19efef660_condition_table_from_osmosis_added_as_i_.py @@ -0,0 +1,166 @@ +"""condition table from osmosis added as i didn't add + +Revision ID: 42a19efef660 +Revises: 956c4ea39b29 +Create Date: 2025-06-17 12:45:14.976944 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + +# revision identifiers, used by Alembic. +revision: str = '42a19efef660' +down_revision: Union[str, None] = '956c4ea39b29' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('assessordetails', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('assessor_name_and_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('elmhurst_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('conservatoryoroutbuilding', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('is_there_a_conservatory', sa.Boolean(), nullable=False), + sa.Column('is_there_a_cellar_present', sa.Boolean(), nullable=False), + sa.Column('is_there_an_outbuilding', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('elevation', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('protected_conservatory_or_aonb', sa.Boolean(), nullable=False), + sa.Column('material_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('are_there_any_visible_signs_of_existing_wall_insulation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('does_the_existing_ground_level_on_any_elevation_bridge_the_dpc', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('elevationinfo', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('elevation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('cavity_wall_depth', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('is_insulation_present', sa.Boolean(), nullable=False), + sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('externalelevation', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('structural_defects_of_elevation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('does_any_structural_defect_need_resolving_before_retrofit', sa.Boolean(), nullable=False), + sa.Column('are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework', sa.Boolean(), nullable=False), + sa.Column('are_there_any_visible_signs_of_movement', sa.Boolean(), nullable=False), + sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_external_finish', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('generalconditionheatingsystem', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('is_the_heating_system_in_working_order', sa.Boolean(), nullable=False), + sa.Column('does_the_occupant_have_a_smart_meter', sa.Boolean(), nullable=False), + sa.Column('are_there_any_smart_monitoring_devices', sa.Boolean(), nullable=False), + sa.Column('are_the_gas_and_electricity_meters_accessible', sa.Boolean(), nullable=False), + sa.Column('dual_or_single_electric_meter', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('heating_from_condition_report', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('room_stat_in_temperature_in_celsius', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('room_stat_location', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('is_the_heating_pattern_known', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('inspectionandproject', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('inspection_date', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('mainheatingone', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('as_defined_by', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('mainheatingtwo', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('is_there_a_main_heating_two', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('propertyaccess', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('are_there_any_road_restriction_in_the_locality', sa.Boolean(), nullable=False), + sa.Column('is_on_street_parking_available', sa.Boolean(), nullable=False), + sa.Column('are_there_any_overhead_wires_or_cables', sa.Boolean(), nullable=False), + sa.Column('is_the_access_gated', sa.Boolean(), nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_wall_area', sa.Boolean(), nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_roof_area', sa.Boolean(), nullable=False), + sa.Column('is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation', sa.Boolean(), nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_ginnel', sa.Boolean(), nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_secured_alleyway', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('secondaryheating', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('is_there_a_secondary_heating', sa.Boolean(), nullable=False), + sa.Column('fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('electric_heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('gas_heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('theproperty', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('house_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('on_which_floor_is_the_flat_located', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('is_there_a_corridor', sa.Boolean(), nullable=False), + sa.Column('is_it_heated', sa.Boolean(), nullable=False), + sa.Column('it_there_a_balcony', sa.Boolean(), nullable=False), + sa.Column('classification_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('orientation_front_elevation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('orientation_in_degrees_front_elevation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('exposure_zone', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('main_wall_construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('conditionreport', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('project_site_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('property_reference_code', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('property_address', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('assessor_details_id', sa.Uuid(), nullable=True), + sa.Column('inspection_id', sa.Uuid(), nullable=True), + sa.Column('property_id', sa.Uuid(), nullable=True), + sa.Column('general_condition_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['assessor_details_id'], ['assessordetails.id'], ), + sa.ForeignKeyConstraint(['general_condition_id'], ['generalconditionheatingsystem.id'], ), + sa.ForeignKeyConstraint(['inspection_id'], ['inspectionandproject.id'], ), + sa.ForeignKeyConstraint(['property_id'], ['theproperty.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('conditionreport') + op.drop_table('theproperty') + op.drop_table('secondaryheating') + op.drop_table('propertyaccess') + op.drop_table('mainheatingtwo') + op.drop_table('mainheatingone') + op.drop_table('inspectionandproject') + op.drop_table('heating_from_condition_report') + op.drop_table('generalconditionheatingsystem') + op.drop_table('externalelevation') + op.drop_table('elevationinfo') + op.drop_table('elevation') + op.drop_table('conservatoryoroutbuilding') + op.drop_table('assessordetails') + # ### end Alembic commands ### diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 777a065..5ea84b6 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -96,6 +96,7 @@ class MainHeatingTwo(BaseModel, table=True): class Heating(BaseModel, table=True): + __tablename__ = 'heating_from_condition_report' room_stat_in_temperature_in_celsius: Optional[str] = None room_stat_location: Optional[str] = None is_the_heating_pattern_known: Optional[str] = None diff --git a/migration_db.sh b/migration_db.sh index e8cadeb..6e6a750 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,3 +1,3 @@ -poetry run alembic revision --autogenerate -m "condition table from osmosis added as i didn't add" +#poetry run alembic revision --autogenerate -m "condition table from osmosis added as i didn't add" -#poetry run alembic upgrade head +poetry run alembic upgrade head From cc934d859405ce16c95ce714a851aae02d9aa8c3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 17 Jun 2025 15:01:28 +0000 Subject: [PATCH 05/22] loading start --- etl/condition_report_etl.py | 10 ++++- etl/pdfReader/sitenotes.py | 3 +- etl/surveyedData/surveryedData.py | 70 +++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/etl/condition_report_etl.py b/etl/condition_report_etl.py index 5284172..fa9488c 100644 --- a/etl/condition_report_etl.py +++ b/etl/condition_report_etl.py @@ -1,6 +1,14 @@ +from pprint import pprint +from etl.db.db import get_db_session, init_db + from etl.surveyedData.surveryedData import surveyedDataProcessor condition_report_file_path = "/workspaces/survey-extractor/etl/files/osmosis_condition_report.pdf" sdp = surveyedDataProcessor("123 Fake Street", [condition_report_file_path]) +pprint(sdp.condition_report.master_obj) +init_db() +with get_db_session() as db_session: + sdp.load_condiion_report(db_session) -# TODO: add the ability to add document type, and sharepoint or s3 link so we can process access it again \ No newline at end of file +# TODO: add the ability to add document type, and sharepoint or s3 link so we can process access it again +# Terraform lambda set up to start this job from a s3 link \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 568d0e6..a70bacf 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -93,8 +93,7 @@ class ConditionReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT - _ = self.setup_condition_report() - pprint(_) + self.master_obj = self.setup_condition_report() def setup_condition_report(self): general_information = self.get_section_1() diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 1c8e358..3e1ca7b 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -10,6 +10,25 @@ from etl.models.preSiteNoteTypes import ( SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling, Door, Walls, Roofs, Floors, PropertyDetail, Windows ) +from pprint import pprint + +from etl.models.conditionReport import ( + AssessorDetails, + InspectionAndProject, + TheProperty, + ElevationInfo, + Elevation, + PropertyAccess, + ExternalElevation, + ConservatoryOrOutbuilding, + GeneralConditionHeatingSystem, + SecondaryHeating, + MainHeatingOne, + MainHeatingTwo, + Heating as HeatingFromConditionReport, + ConditionReport +) + from etl.models.topLevel import( Buildings, Documents ) @@ -38,6 +57,57 @@ class surveyedDataProcessor(): elif pdf.type == ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT: self.condition_report = pdf.get_reader() + def load_condiion_report(self, db_session): + # My task to complete load + # [] general_information = self.get_section_1() + # [] access_and_elevations = self.get_section_2() + # [] rooms = self.get_section_3() + # [] heating_system = self.get_section_4() + # [] occupant_assessment = self.get_section_5() + # [] site_name, reference_code, address, postcode = self.get_section_0() + + # return ConditionReportModel( + # project_site_name=site_name, + # property_reference_code=reference_code, + # property_address=address, + # postcode=postcode, + # general_information=general_information, + # access_and_elevations=access_and_elevations, + # rooms=rooms, + # heating_system=heating_system, + # occupancy_assessment=occupant_assessment, + # ) + # + # Load General Information: + # + # assessor_details = self.get_assessor_details() + # inspection_and_project = self.get_inspection_and_project() + # the_property = self.get_the_property() + # main_elevation = self.get_main_elevation() + # elevations = self.get_all_elevations() + + # return GeneralInformation( + # assessor_details=assessor_details, + # inspection_and_project=inspection_and_project, + # the_property=the_property, + # main_elevation=main_elevation, + # elevations=elevations + # ) + pprint(self.) + assessor_details = self.get_attribute_and_load( + self.condition_report.master_obj. + ) + + def load_general_information_from_condition_report(self, db_session): + # Assessors Information + self.upsert_record( + db_session=db_session, + model_class=AssessorDetails, + + + + ) + def load_pre_site_notes_summary_table(self, db_session): summary_data = self.pre_site_note.survey_information.model_dump() return self.upsert_record( From 629211d20626cc934c8bc1ab9789364944431b8c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 18 Jun 2025 10:21:20 +0000 Subject: [PATCH 06/22] renaming table works now --- .../versions/f2f205448dff_rename_ntables.py | 78 +++++++++++++++++++ etl/models/conditionReport.py | 2 +- etl/models/preSiteNoteTypes.py | 10 +-- etl/pdfReader/sitenotes.py | 6 +- etl/surveyedData/surveryedData.py | 15 ++-- etl/transform/conditionReportTypes.py | 5 +- etl/transform/preSiteNoteTypes.py | 6 +- migration_db.sh | 15 +++- 8 files changed, 117 insertions(+), 20 deletions(-) create mode 100644 alembic/versions/f2f205448dff_rename_ntables.py diff --git a/alembic/versions/f2f205448dff_rename_ntables.py b/alembic/versions/f2f205448dff_rename_ntables.py new file mode 100644 index 0000000..3324f1e --- /dev/null +++ b/alembic/versions/f2f205448dff_rename_ntables.py @@ -0,0 +1,78 @@ +"""rename ntables + +Revision ID: f2f205448dff +Revises: 42a19efef660 +Create Date: 2025-06-18 10:09:39.973162 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + +# revision identifiers, used by Alembic. +revision: str = 'f2f205448dff' +down_revision: Union[str, None] = '42a19efef660' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # Drop foreign keys referencing 'heating' + op.drop_constraint('propertydescription_main_heating_id_fkey', 'propertydescription', type_='foreignkey') + op.drop_constraint('propertydescription_main_heating2_id_fkey', 'propertydescription', type_='foreignkey') + + # Rename 'heating' table to 'heatingfrompresitenotes' + op.rename_table('heating', 'heatingfrompresitenotes') + + # Rename/re-add cleaned up columns (ensure you're within 63 char limit!) + op.drop_column('externalelevation', 'are_there_any_visible_signs_of_cracking_to_the_existing_externa') + op.drop_column('externalelevation', 'are_there_any_signs_of_water_penetration_caused_by_failed_rainw') + op.add_column('externalelevation', sa.Column( + 'water_penetration_from_failed_rainwater_goods', sa.Boolean(), nullable=False)) + op.add_column('externalelevation', sa.Column( + 'cracking_to_external_finish_visible', sa.Boolean(), nullable=False)) + + # Update propertyaccess fields + op.drop_column('propertyaccess', 'is_there_restricted_space_for_contractors_to_access_the_wall_ar') + op.drop_column('propertyaccess', 'is_there_restricted_space_for_contractors_to_access_the_roof_ar') + op.drop_column('propertyaccess', 'is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighb') + op.add_column('propertyaccess', sa.Column( + 'restricted_wall_access', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column( + 'restricted_roof_access', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column( + 'sufficient_boundary_width', sa.Boolean(), nullable=False)) + + # Create new foreign keys to heatingfrompresitenotes + op.create_foreign_key('propertydescription_main_heating_id_fkey', 'propertydescription', 'heatingfrompresitenotes', ['main_heating_id'], ['id']) + op.create_foreign_key('propertydescription_main_heating2_id_fkey', 'propertydescription', 'heatingfrompresitenotes', ['main_heating2_id'], ['id']) + + +def downgrade() -> None: + """Downgrade schema.""" + # Drop new foreign keys + op.drop_constraint('propertydescription_main_heating_id_fkey', 'propertydescription', type_='foreignkey') + op.drop_constraint('propertydescription_main_heating2_id_fkey', 'propertydescription', type_='foreignkey') + + # Rename 'heatingfrompresitenotes' back to 'heating' + op.rename_table('heatingfrompresitenotes', 'heating') + + # Revert column changes + op.drop_column('externalelevation', 'water_penetration_from_failed_rainwater_goods') + op.drop_column('externalelevation', 'cracking_to_external_finish_visible') + op.add_column('externalelevation', sa.Column('are_there_any_signs_of_water_penetration_caused_by_failed_rainw', sa.Boolean(), nullable=False)) + op.add_column('externalelevation', sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_externa', sa.Boolean(), nullable=False)) + + op.drop_column('propertyaccess', 'restricted_wall_access') + op.drop_column('propertyaccess', 'restricted_roof_access') + op.drop_column('propertyaccess', 'sufficient_boundary_width') + op.add_column('propertyaccess', sa.Column('is_there_restricted_space_for_contractors_to_access_the_wall_ar', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column('is_there_restricted_space_for_contractors_to_access_the_roof_ar', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column('is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighb', sa.Boolean(), nullable=False)) + + # Restore original foreign keys + op.create_foreign_key('propertydescription_main_heating_id_fkey', 'propertydescription', 'heating', ['main_heating_id'], ['id']) + op.create_foreign_key('propertydescription_main_heating2_id_fkey', 'propertydescription', 'heating', ['main_heating2_id'], ['id']) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 5ea84b6..cd3e7b1 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -95,7 +95,7 @@ class MainHeatingTwo(BaseModel, table=True): is_there_a_main_heating_two: bool -class Heating(BaseModel, table=True): +class HeatingFromConditionReport(BaseModel, table=True): __tablename__ = 'heating_from_condition_report' room_stat_in_temperature_in_celsius: Optional[str] = None room_stat_location: Optional[str] = None diff --git a/etl/models/preSiteNoteTypes.py b/etl/models/preSiteNoteTypes.py index 55392af..f97f3bc 100644 --- a/etl/models/preSiteNoteTypes.py +++ b/etl/models/preSiteNoteTypes.py @@ -127,7 +127,7 @@ class HeatingSystemControls(BaseModel, table=True): mains_gas_available: Optional[bool] = False -class Heating(BaseModel, table=True): +class HeatingFromPreSiteNotes(BaseModel, table=True): type: str heating_source: str efficiency_source: str @@ -252,8 +252,8 @@ class PropertyDescription(BaseModel, table=True): photovoltaic_panel_id: Optional[uuid.UUID] = Field(default=None, foreign_key="photovoltaicpanel.id") wind_turbine_id: Optional[uuid.UUID] = Field(default=None, foreign_key="windturbine.id") other_details_id: Optional[uuid.UUID] = Field(default=None, foreign_key="otherdetails.id") - main_heating_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heating.id") - main_heating2_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heating.id") + main_heating_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingfrompresitenotes.id") + main_heating2_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingfrompresitenotes.id") secondary_heating_type_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingtype.id") # Relationships @@ -275,8 +275,8 @@ class PropertyDescription(BaseModel, table=True): photovoltaic_panel: Optional["PhotovoltaicPanel"] = Relationship(back_populates="property_description") wind_turbine: Optional["WindTurbine"] = Relationship(back_populates="property_description") other_details: Optional["OtherDetails"] = Relationship(back_populates="property_description") - main_heating: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"}) - main_heating2: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"}) + main_heating: Optional["HeatingFromPreSiteNotes"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"}) + main_heating2: Optional["HeatingFromPreSiteNotes"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"}) secondary_heating_type: Optional["HeatingType"] = Relationship(back_populates="property_description") pre_site_notes: Optional["PreSiteNote"] = Relationship(back_populates="pre_site_note_description") diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index a70bacf..5bb3932 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -5,7 +5,7 @@ from etl.transform.preSiteNoteTypes import ( Walls, Roofs, Floors, Door, VentilationAndCooling, Lighting, WaterHeating, HotWaterCylinder, SolarWaterHeating, ShowerAndBaths, FlueGasHeatRecoverySystem, PhotovoltaicPanel, - WindTurbine, OtherDetails, Windows, Heating, HeatingSystemControls, + WindTurbine, OtherDetails, Windows, HeatingFromPreSiteNotes, HeatingSystemControls, HeatingType, Insulation ) from etl.transform.conditionReportTypes import ( @@ -14,7 +14,7 @@ from etl.transform.conditionReportTypes import ( ExternalElevationGableOne, ExternalElevationGableTwo, ExternalElevationRear, ConservatoryOrOutbuilding, AccessAndElevations, Hallway, RoomInfo, WindowsInfo, VentilationInfo, LivingRoom, DiningRoom, Kitchen, Rooms, Utility, WC, Landing, Bedroom, Bathroom, LoftSpace, RoomInRoof, HeatingSystem, GeneralConditionHeatingSystem, - MainHeatingOne, MainHeatingTwo, SecondaryHeating, HeatingByRoom, Renewables, Occupant, EnergyUse, Heating, ShowerAndBath, FridgeAndFreezers, Cooker, TumbleDryer, + MainHeatingOne, MainHeatingTwo, SecondaryHeating, HeatingByRoom, Renewables, Occupant, EnergyUse, HeatingFromConditionReport, ShowerAndBath, FridgeAndFreezers, Cooker, TumbleDryer, GeneralInformation, OccupantAssessment ) from datetime import datetime @@ -644,7 +644,7 @@ class ConditionReport(SiteNotesExtractor): def get_heating(self): data = self.get_data_between("Heating", "Shower & bath") - return Heating( + return HeatingFromConditionReport( room_stat_in_temperature_in_celsius=self.get_next_value(data, "Room Stat Temperature (in °C)", avoid=["Room Stat Location", '\xa0']), room_stat_location=self.get_next_value(data, "Room Stat Location", avoid = ["Is the heating pattern known?", '\xa0']), is_the_heating_pattern_known=self.get_next_value(data, "Is the heating pattern known?", avoid=["Shower & bath", '\xa0']), diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 3e1ca7b..f136483 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -5,7 +5,7 @@ from etl.models.preSiteNoteTypes import ( AssessorInfo, CompanyInfo, PreSiteNotesSummaryInfo, PreSiteNote, - PropertyDescription, Dimension, HeatingType, Heating, HeatingSystemControls, + PropertyDescription, Dimension, HeatingType, HeatingFromPreSiteNotes, HeatingSystemControls, OtherDetails, WindTurbine, PhotovoltaicPanel, FlueGasHeatRecoverySystem, ShowerAndBaths, SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling, Door, Walls, Roofs, Floors, PropertyDetail, Windows @@ -25,7 +25,7 @@ from etl.models.conditionReport import ( SecondaryHeating, MainHeatingOne, MainHeatingTwo, - Heating as HeatingFromConditionReport, + HeatingFromConditionReport, ConditionReport ) @@ -93,10 +93,14 @@ class surveyedDataProcessor(): # main_elevation=main_elevation, # elevations=elevations # ) - pprint(self.) + assessor_details = self.get_attribute_and_load( - self.condition_report.master_obj. + self.condition_report.master_obj.general_information, + "assessor_details", + AssessorDetails, + db_session ) + print("assessor details loaded please check") def load_general_information_from_condition_report(self, db_session): # Assessors Information @@ -136,6 +140,7 @@ class surveyedDataProcessor(): if found: print(f"Uploading to data base {found}") print(f"Uploaded to database with this dict {found.model_dump()}") + print(f"Pydantic model {pydanticModel}") if found.model_dump(): db = self.upsert_record( db_session=db_session, @@ -190,7 +195,7 @@ class surveyedDataProcessor(): if data: mainheating = self.upsert_record( db_session=db_session, - model_class=Heating, + model_class=HeatingFromPreSiteNotes, data_dict=data, lookup_field=None, additional_fields={"controls_id": mainheatingcontrols.id} diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index e2c6c55..3a099ba 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -250,7 +250,8 @@ class ShowerAndBath(BaseModel): do_you_know_the_no_of_showers_per_day_per_week: bool please_input_no_of_showers_and_specify_a_day_or_a_week: str do_you_know_the_number_of_baths_per_day_or_per_week: str -class Heating(BaseModel): + +class HeatingFromConditionReport(BaseModel): # TODO find one with an example of this one room_stat_in_temperature_in_celsius: Optional[str] = None room_stat_location: Optional[str] = None @@ -264,7 +265,7 @@ class EnergyUse(BaseModel): class OccupantAssessment(BaseModel): occupant: Occupant energy_use: EnergyUse - heating: Heating + heating: HeatingFromConditionReport shower_and_bath: ShowerAndBath appliances: Optional[Appliances] fridge_and_freezers: FridgeAndFreezers diff --git a/etl/transform/preSiteNoteTypes.py b/etl/transform/preSiteNoteTypes.py index 4a8b8fc..0e78efa 100644 --- a/etl/transform/preSiteNoteTypes.py +++ b/etl/transform/preSiteNoteTypes.py @@ -101,7 +101,7 @@ class HeatingSystemControls(BaseModel): electricity_meter_type: Optional[str] = "" mains_gas_available: Optional[bool] = False -class Heating(BaseModel): +class HeatingFromPreSiteNotes(BaseModel): type: str heating_source: str efficiency_source: str @@ -198,8 +198,8 @@ class PropertyDescription(BaseModel): photovoltaicPanel: Optional[PhotovoltaicPanel] windTurbine: Optional[WindTurbine] otherDetails: Optional[OtherDetails] - mainHeating: Optional[Heating] - mainHeating2: Optional[Heating] + mainHeating: Optional[HeatingFromPreSiteNotes] + mainHeating2: Optional[HeatingFromPreSiteNotes] secondaryHeatingType: Optional[HeatingType] # class PropertyReport(): diff --git a/migration_db.sh b/migration_db.sh index 6e6a750..dd6345b 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,3 +1,16 @@ -#poetry run alembic revision --autogenerate -m "condition table from osmosis added as i didn't add" +#poetry run alembic revision --autogenerate -m "rename ntables" + poetry run alembic upgrade head + +# See which hash I'm at +#poetry run alembic current + +# downgrade one version +#poetry run alembic upgrade head +#poetry run alembic downgrade -1 +#poetry run alembic upgrade +1 + + + + From 9bae34e617dd4c81c37d4c325bb8343308345926 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 18 Jun 2025 15:44:59 +0000 Subject: [PATCH 07/22] added the deletion of condition report --- ...a2bc48ac2_add_general_information_class.py | 73 ++++++++ ...89edb_delete_everythign_and_start_again.py | 176 ++++++++++++++++++ etl/condition_report_etl.py | 2 +- etl/models/conditionReport.py | 106 ----------- etl/surveyedData/surveryedData.py | 80 ++++++-- migration_db.sh | 7 +- 6 files changed, 318 insertions(+), 126 deletions(-) create mode 100644 alembic/versions/771a2bc48ac2_add_general_information_class.py create mode 100644 alembic/versions/881142a89edb_delete_everythign_and_start_again.py diff --git a/alembic/versions/771a2bc48ac2_add_general_information_class.py b/alembic/versions/771a2bc48ac2_add_general_information_class.py new file mode 100644 index 0000000..f5dbcf0 --- /dev/null +++ b/alembic/versions/771a2bc48ac2_add_general_information_class.py @@ -0,0 +1,73 @@ +"""add general information class + +Revision ID: 771a2bc48ac2 +Revises: f2f205448dff +Create Date: 2025-06-18 14:56:08.762223 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '771a2bc48ac2' +down_revision: Union[str, None] = 'f2f205448dff' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('generalinformation', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('assessor_details_id', sa.Uuid(), nullable=False), + sa.Column('inspection_and_project_id', sa.Uuid(), nullable=False), + sa.Column('the_property_id', sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(['assessor_details_id'], ['assessordetails.id'], ), + sa.ForeignKeyConstraint(['inspection_and_project_id'], ['inspectionandproject.id'], ), + sa.ForeignKeyConstraint(['the_property_id'], ['theproperty.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.add_column('conditionreport', sa.Column('general_information_id', sa.Uuid(), nullable=True)) + op.drop_constraint('conditionreport_general_condition_id_fkey', 'conditionreport', type_='foreignkey') + op.drop_constraint('conditionreport_assessor_details_id_fkey', 'conditionreport', type_='foreignkey') + op.create_foreign_key(None, 'conditionreport', 'generalinformation', ['general_information_id'], ['id']) + op.drop_column('conditionreport', 'general_condition_id') + op.drop_column('conditionreport', 'assessor_details_id') + op.add_column('externalelevation', sa.Column('are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework', sa.Boolean(), nullable=False)) + op.add_column('externalelevation', sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_external_finish', sa.Boolean(), nullable=False)) + op.drop_column('externalelevation', 'water_penetration_from_failed_rainwater_goods') + op.drop_column('externalelevation', 'cracking_to_external_finish_visible') + op.add_column('propertyaccess', sa.Column('is_there_restricted_space_for_contractors_to_access_the_wall_area', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column('is_there_restricted_space_for_contractors_to_access_the_roof_area', sa.Boolean(), nullable=False)) + op.add_column('propertyaccess', sa.Column('is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation', sa.Boolean(), nullable=False)) + op.drop_column('propertyaccess', 'sufficient_boundary_width') + op.drop_column('propertyaccess', 'restricted_wall_access') + op.drop_column('propertyaccess', 'restricted_roof_access') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('propertyaccess', sa.Column('restricted_roof_access', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('propertyaccess', sa.Column('restricted_wall_access', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('propertyaccess', sa.Column('sufficient_boundary_width', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.drop_column('propertyaccess', 'is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation') + op.drop_column('propertyaccess', 'is_there_restricted_space_for_contractors_to_access_the_roof_area') + op.drop_column('propertyaccess', 'is_there_restricted_space_for_contractors_to_access_the_wall_area') + op.add_column('externalelevation', sa.Column('cracking_to_external_finish_visible', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('externalelevation', sa.Column('water_penetration_from_failed_rainwater_goods', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.drop_column('externalelevation', 'are_there_any_visible_signs_of_cracking_to_the_existing_external_finish') + op.drop_column('externalelevation', 'are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework') + op.add_column('conditionreport', sa.Column('assessor_details_id', sa.UUID(), autoincrement=False, nullable=True)) + op.add_column('conditionreport', sa.Column('general_condition_id', sa.UUID(), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'conditionreport', type_='foreignkey') + op.create_foreign_key('conditionreport_assessor_details_id_fkey', 'conditionreport', 'assessordetails', ['assessor_details_id'], ['id']) + op.create_foreign_key('conditionreport_general_condition_id_fkey', 'conditionreport', 'generalconditionheatingsystem', ['general_condition_id'], ['id']) + op.drop_column('conditionreport', 'general_information_id') + op.drop_table('generalinformation') + # ### end Alembic commands ### diff --git a/alembic/versions/881142a89edb_delete_everythign_and_start_again.py b/alembic/versions/881142a89edb_delete_everythign_and_start_again.py new file mode 100644 index 0000000..22aa91d --- /dev/null +++ b/alembic/versions/881142a89edb_delete_everythign_and_start_again.py @@ -0,0 +1,176 @@ +"""delete everythign and start again + +Revision ID: 881142a89edb +Revises: 771a2bc48ac2 +Create Date: 2025-06-18 15:38:53.955796 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '881142a89edb' +down_revision: Union[str, None] = '771a2bc48ac2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.execute("DROP TABLE conditionreport CASCADE") + op.execute("DROP TABLE generalinformation CASCADE") + op.execute("DROP TABLE theproperty CASCADE") + op.execute("DROP TABLE assessordetails CASCADE") + op.execute("DROP TABLE inspectionandproject CASCADE") + op.execute("DROP TABLE propertyaccess CASCADE") + op.execute("DROP TABLE externalelevation CASCADE") + op.execute("DROP TABLE generalconditionheatingsystem CASCADE") + op.execute("DROP TABLE heating_from_condition_report CASCADE") + op.execute("DROP TABLE mainheatingone CASCADE") + op.execute("DROP TABLE mainheatingtwo CASCADE") + op.execute("DROP TABLE secondaryheating CASCADE") + op.execute("DROP TABLE conservatoryoroutbuilding CASCADE") + op.execute("DROP TABLE elevation CASCADE") + op.execute("DROP TABLE elevationinfo CASCADE") + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('inspectionandproject', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('inspection_date', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='inspectionandproject_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('elevation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('protected_conservatory_or_aonb', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('material_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_existing_wall_insulation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('does_the_existing_ground_level_on_any_elevation_bridge_the_dpc', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='elevation_pkey') + ) + op.create_table('mainheatingone', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('as_defined_by', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('fuel', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='mainheatingone_pkey') + ) + op.create_table('assessordetails', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('assessor_name_and_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('elmhurst_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='assessordetails_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('elevationinfo', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('elevation_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('cavity_wall_depth', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('is_insulation_present', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('insulation_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='elevationinfo_pkey') + ) + op.create_table('externalelevation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('structural_defects_of_elevation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('does_any_structural_defect_need_resolving_before_retrofit', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_signs_of_water_penetration_caused_by_failed_rainw', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_movement', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_externa', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='externalelevation_pkey') + ) + op.create_table('conditionreport', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('project_site_name', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('property_reference_code', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('property_address', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('postcode', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('inspection_id', sa.UUID(), autoincrement=False, nullable=True), + sa.Column('property_id', sa.UUID(), autoincrement=False, nullable=True), + sa.Column('general_information_id', sa.UUID(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['inspection_id'], ['inspectionandproject.id'], name='conditionreport_inspection_id_fkey'), + sa.ForeignKeyConstraint(['property_id'], ['theproperty.id'], name='conditionreport_property_id_fkey'), + sa.PrimaryKeyConstraint('id', name='conditionreport_pkey') + ) + op.create_table('theproperty', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('house_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('on_which_floor_is_the_flat_located', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('is_there_a_corridor', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_it_heated', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('it_there_a_balcony', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('classification_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('orientation_front_elevation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('orientation_in_degrees_front_elevation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('exposure_zone', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('main_wall_construction', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='theproperty_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('generalconditionheatingsystem', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_the_heating_system_in_working_order', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('does_the_occupant_have_a_smart_meter', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_smart_monitoring_devices', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_the_gas_and_electricity_meters_accessible', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('dual_or_single_electric_meter', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='generalconditionheatingsystem_pkey') + ) + op.create_table('mainheatingtwo', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_main_heating_two', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='mainheatingtwo_pkey') + ) + op.create_table('generalinformation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('assessor_details_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('inspection_and_project_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('the_property_id', sa.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['assessor_details_id'], ['assessordetails.id'], name='generalinformation_assessor_details_id_fkey'), + sa.ForeignKeyConstraint(['inspection_and_project_id'], ['inspectionandproject.id'], name='generalinformation_inspection_and_project_id_fkey'), + sa.ForeignKeyConstraint(['the_property_id'], ['theproperty.id'], name='generalinformation_the_property_id_fkey'), + sa.PrimaryKeyConstraint('id', name='generalinformation_pkey') + ) + op.create_table('heating_from_condition_report', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('room_stat_in_temperature_in_celsius', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('room_stat_location', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('is_the_heating_pattern_known', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name='heating_from_condition_report_pkey') + ) + op.create_table('propertyaccess', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('are_there_any_road_restriction_in_the_locality', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_on_street_parking_available', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_overhead_wires_or_cables', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_the_access_gated', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_wall_ar', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_roof_ar', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighb', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_ginnel', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_secured_alleyway', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='propertyaccess_pkey') + ) + op.create_table('secondaryheating', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_secondary_heating', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('fuel', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('electric_heating_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('gas_heating_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='secondaryheating_pkey') + ) + op.create_table('conservatoryoroutbuilding', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_conservatory', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_a_cellar_present', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_an_outbuilding', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='conservatoryoroutbuilding_pkey') + ) + # ### end Alembic commands ### diff --git a/etl/condition_report_etl.py b/etl/condition_report_etl.py index fa9488c..e826406 100644 --- a/etl/condition_report_etl.py +++ b/etl/condition_report_etl.py @@ -8,7 +8,7 @@ pprint(sdp.condition_report.master_obj) init_db() with get_db_session() as db_session: - sdp.load_condiion_report(db_session) + sdp.load_condition_report(db_session) # TODO: add the ability to add document type, and sharepoint or s3 link so we can process access it again # Terraform lambda set up to start this job from a s3 link \ No newline at end of file diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index cd3e7b1..1b24661 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -6,109 +6,3 @@ from datetime import datetime class BaseModel(SQLModel): id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - - -class AssessorDetails(BaseModel, table=True): - assessor_name_and_id: str - elmhurst_id: str - - -class InspectionAndProject(BaseModel, table=True): - inspection_date: str - - -class TheProperty(BaseModel, table=True): - house_type: str - on_which_floor_is_the_flat_located: str - is_there_a_corridor: bool - is_it_heated: bool - it_there_a_balcony: bool - classification_type: str - orientation_front_elevation: str - orientation_in_degrees_front_elevation: str - exposure_zone: str - main_wall_construction: str - - -class ElevationInfo(BaseModel, table=True): - elevation_type: str - cavity_wall_depth: str - is_insulation_present: bool - insulation_type: str - - -class Elevation(BaseModel, table=True): - protected_conservatory_or_aonb: bool - material_type: str - are_there_any_visible_signs_of_existing_wall_insulation: str - does_the_existing_ground_level_on_any_elevation_bridge_the_dpc: bool - - -class PropertyAccess(BaseModel, table=True): - are_there_any_road_restriction_in_the_locality: bool - is_on_street_parking_available: bool - are_there_any_overhead_wires_or_cables: bool - is_the_access_gated: bool - is_there_restricted_space_for_contractors_to_access_the_wall_area: bool - is_there_restricted_space_for_contractors_to_access_the_roof_area: bool - is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation: bool - is_access_to_the_rear_provided_by_use_of_a_ginnel: bool - is_access_to_the_rear_provided_by_use_of_a_secured_alleyway: bool - - -class ExternalElevation(BaseModel, table=True): - structural_defects_of_elevation: str - does_any_structural_defect_need_resolving_before_retrofit: bool - are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework: bool - are_there_any_visible_signs_of_movement: bool - are_there_any_visible_signs_of_cracking_to_the_existing_external_finish: bool - - -class ConservatoryOrOutbuilding(BaseModel, table=True): - is_there_a_conservatory: bool - is_there_a_cellar_present: bool - is_there_an_outbuilding: bool - - -class GeneralConditionHeatingSystem(BaseModel, table=True): - is_the_heating_system_in_working_order: bool - does_the_occupant_have_a_smart_meter: bool - are_there_any_smart_monitoring_devices: bool - are_the_gas_and_electricity_meters_accessible: bool - dual_or_single_electric_meter: str - - -class SecondaryHeating(BaseModel, table=True): - is_there_a_secondary_heating: bool - fuel: str - electric_heating_type: str - gas_heating_type: str - - -class MainHeatingOne(BaseModel, table=True): - as_defined_by: str - fuel: str - type: str - - -class MainHeatingTwo(BaseModel, table=True): - is_there_a_main_heating_two: bool - - -class HeatingFromConditionReport(BaseModel, table=True): - __tablename__ = 'heating_from_condition_report' - room_stat_in_temperature_in_celsius: Optional[str] = None - room_stat_location: Optional[str] = None - is_the_heating_pattern_known: Optional[str] = None - - -class ConditionReport(BaseModel, table=True): - project_site_name: str - property_reference_code: str - property_address: str - postcode: str - - assessor_details_id: Optional[uuid.UUID] = Field(default=None, foreign_key="assessordetails.id") - inspection_id: Optional[uuid.UUID] = Field(default=None, foreign_key="inspectionandproject.id") - property_id: Optional[uuid.UUID] = Field(default=None, foreign_key="theproperty.id") - general_condition_id: Optional[uuid.UUID] = Field(default=None, foreign_key="generalconditionheatingsystem.id") diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index f136483..7a4ad52 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -26,7 +26,9 @@ from etl.models.conditionReport import ( MainHeatingOne, MainHeatingTwo, HeatingFromConditionReport, - ConditionReport + ConditionReport, + GeneralInformation, + MainElevation, ) from etl.models.topLevel import( @@ -57,7 +59,7 @@ class surveyedDataProcessor(): elif pdf.type == ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT: self.condition_report = pdf.get_reader() - def load_condiion_report(self, db_session): + def load_condition_report(self, db_session): # My task to complete load # [] general_information = self.get_section_1() # [] access_and_elevations = self.get_section_2() @@ -93,25 +95,75 @@ class surveyedDataProcessor(): # main_elevation=main_elevation, # elevations=elevations # ) + self.load_general_information_from_condition_report(db_session) + def load_general_information_from_condition_report(self, db_session): assessor_details = self.get_attribute_and_load( self.condition_report.master_obj.general_information, "assessor_details", AssessorDetails, - db_session + db_session, + lookup_field="elmhurst_id", + ) + inspection_details = self.get_attribute_and_load( + self.condition_report.master_obj.general_information, + "inspection_and_project", + InspectionAndProject, + db_session, + ) + the_property = self.get_attribute_and_load( + self.condition_report.master_obj.general_information, + "the_property", + TheProperty, + db_session, ) - print("assessor details loaded please check") - def load_general_information_from_condition_report(self, db_session): - # Assessors Information - self.upsert_record( + main_elevation_info = self.get_attribute_and_load( + self.condition_report.master_obj.general_information.main_elevation, + "elevation_info", + ElevationInfo, + db_session, + ) + + main_elevation_in_db = self.upsert_record( + db_session=db_session + model_class=MainElevation, + data_dict={ + "elevation_info_id:": main_elevation_info.id, + }, + lookup_field=None + ) + + general_information = self.upsert_record( db_session=db_session, - model_class=AssessorDetails, - - - + model_class=GeneralInformation, + data_dict={ + "assessor_details_id": assessor_details.id, + "inspection_and_project_id": inspection_details.id, + "the_property_id": the_property.id, + }, + lookup_field=None, ) + + # elevations = [] + # for i,elevation in enumerate(self.condition_report.master_obj.general_information.elevations): + # data = elevation.model_dump() + + # elevation_from_db = self.upsert_record( + # db_session=db_session, + # model_class=ElevationInfo, + # data_dict = data + # lookup_field=None, + # additional_fields={} + # ) + # elevations.append(elevation_from_db.id) + + # pprint(self.condition_report.master_obj.general_information) + + # dimensions.append(dimension.id) + # print("Check database please") + def load_pre_site_notes_summary_table(self, db_session): summary_data = self.pre_site_note.survey_information.model_dump() return self.upsert_record( @@ -135,7 +187,7 @@ class surveyedDataProcessor(): lookup_field="UPRN", ) - def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session): + def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None): found = getattr(obj, attr_string, None) if found: print(f"Uploading to data base {found}") @@ -146,7 +198,7 @@ class surveyedDataProcessor(): db_session=db_session, model_class=pydanticModel, data_dict=found.model_dump(), - lookup_field=None + lookup_field=lookup_field ) return db return None @@ -178,7 +230,7 @@ class surveyedDataProcessor(): if data: mainheating2 = self.upsert_record( db_session=db_session, - model_class=Heating, + model_class=HeatingFromPreSiteNotes, data_dict=data, lookup_field=None, additional_fields= {"controls_id": mainheating2controls.id}, diff --git a/migration_db.sh b/migration_db.sh index dd6345b..ec9c0ce 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,7 +1,5 @@ -#poetry run alembic revision --autogenerate -m "rename ntables" - - -poetry run alembic upgrade head +#poetry run alembic revision --autogenerate -m "delete everythign and start again" +poetry run alembic upgrade +1 # See which hash I'm at #poetry run alembic current @@ -9,7 +7,6 @@ poetry run alembic upgrade head # downgrade one version #poetry run alembic upgrade head #poetry run alembic downgrade -1 -#poetry run alembic upgrade +1 From 68d6dbda44e56f3f1db7f089927b200bcf7d8ef5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 18 Jun 2025 18:24:43 +0000 Subject: [PATCH 08/22] save current working copy --- etl/models/conditionReport.py | 42 ++++++++++++ etl/surveyedData/surveryedData.py | 109 +++++++++++------------------- 2 files changed, 82 insertions(+), 69 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 1b24661..fcc5c4e 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -6,3 +6,45 @@ from datetime import datetime class BaseModel(SQLModel): id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + +class AssessorDetails(BaseModel, table=True): + assessor_name_and_id: str + elmhurst_id: str + +class InspectionAndProject(BaseModel, table=True): + inspection_date: str + +class TheProperty(BaseModel, table=True): + house_type: str + on_which_floor_is_the_flat_located: str + is_there_a_corridor: bool + is_it_heated: bool + it_there_a_balcony: bool + classification_type: str + orientation_front_elevation: str + orientation_in_degrees_front_elevation: str + exposure_zone: str + main_wall_construction: str + + +class ElevationInfo(BaseModel, table=True): + elevation_type: str + cavity_wall_depth: str + is_insulation_present: bool + insulation_type: str + + elevation_table: Optional["MainElevation"] = Relationship(back_populates="elevation_info") + +class MainElevation(BaseModel, table=True): + elevation_info_id: uuid.UUID = Field(foreign_key="elevationinfo.id") + + #SQLAlcemy things + elevation_info: ElevationInfo = Relationship(back_populates="elevation_table") + +class Elevation(BaseModel): + protected_conservatory_or_aonb: bool + material_type: str + visible_signs_of_existing_wall_insulation: str + ground_level_bridge_the_dpc: bool + + info: List[ElevationInfo] = Relationship(back_populates="") diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 7a4ad52..f405a91 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -15,20 +15,8 @@ from pprint import pprint from etl.models.conditionReport import ( AssessorDetails, InspectionAndProject, - TheProperty, - ElevationInfo, - Elevation, - PropertyAccess, - ExternalElevation, - ConservatoryOrOutbuilding, - GeneralConditionHeatingSystem, - SecondaryHeating, - MainHeatingOne, - MainHeatingTwo, - HeatingFromConditionReport, - ConditionReport, - GeneralInformation, - MainElevation, + TheProperty, ElevationInfo, + MainElevation ) from etl.models.topLevel import( @@ -98,71 +86,54 @@ class surveyedDataProcessor(): self.load_general_information_from_condition_report(db_session) def load_general_information_from_condition_report(self, db_session): - assessor_details = self.get_attribute_and_load( - self.condition_report.master_obj.general_information, - "assessor_details", - AssessorDetails, - db_session, + assessors_data = self.condition_report.master_obj.general_information.assessor_details.model_dump() + + assessor_details = self.upsert_record( + db_session=db_session, + model_class=AssessorDetails, + data_dict=assessors_data, lookup_field="elmhurst_id", ) - inspection_details = self.get_attribute_and_load( - self.condition_report.master_obj.general_information, - "inspection_and_project", - InspectionAndProject, - db_session, - ) - the_property = self.get_attribute_and_load( - self.condition_report.master_obj.general_information, - "the_property", - TheProperty, - db_session, - ) + inspection_data = self.condition_report.master_obj.general_information.inspection_and_project.model_dump() - main_elevation_info = self.get_attribute_and_load( - self.condition_report.master_obj.general_information.main_elevation, - "elevation_info", - ElevationInfo, - db_session, - ) - - main_elevation_in_db = self.upsert_record( - db_session=db_session - model_class=MainElevation, - data_dict={ - "elevation_info_id:": main_elevation_info.id, - }, + inspection_and_project = self.upsert_record( + db_session=db_session, + model_class=InspectionAndProject, + data_dict=inspection_data, lookup_field=None ) - general_information = self.upsert_record( + property_data = self.condition_report.master_obj.general_information.the_property.model_dump() + the_property = self.upsert_record( db_session=db_session, - model_class=GeneralInformation, - data_dict={ - "assessor_details_id": assessor_details.id, - "inspection_and_project_id": inspection_details.id, - "the_property_id": the_property.id, - }, - lookup_field=None, + model_class=TheProperty, + data_dict=property_data, + lookup_field=None ) + + # Main elevation + elevation_info = self.condition_report.master_obj.general_information.main_elevation.elevation_info.model_dump() + elevation_info = self.upsert_record( + db_session=db_session, + model_class=ElevationInfo, + data_dict=elevation_info, + lookup_field=None + ) + main_elevation = self.upsert_record( + db_session=db_session, + model_class=MainElevation, + data_dict={ + "elevation_info_id": elevation_info.id + }, + lookup_field=None + ) + + # elevations multiple + for i, elevation in enumerate() + # Other elevations - # elevations = [] - # for i,elevation in enumerate(self.condition_report.master_obj.general_information.elevations): - # data = elevation.model_dump() - - # elevation_from_db = self.upsert_record( - # db_session=db_session, - # model_class=ElevationInfo, - # data_dict = data - # lookup_field=None, - # additional_fields={} - # ) - # elevations.append(elevation_from_db.id) - - # pprint(self.condition_report.master_obj.general_information) - - # dimensions.append(dimension.id) - # print("Check database please") + print("Check database please") def load_pre_site_notes_summary_table(self, db_session): summary_data = self.pre_site_note.survey_information.model_dump() From 24d2550cb8deeefb3aab370658cea379f14590db Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 19 Jun 2025 10:29:41 +0000 Subject: [PATCH 09/22] elevation has worked --- etl/models/conditionReport.py | 15 ++++++----- etl/surveyedData/surveryedData.py | 45 +++++++++++++++++++++++-------- migration_db.sh | 5 ++-- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index fcc5c4e..a891c4b 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -26,25 +26,28 @@ class TheProperty(BaseModel, table=True): exposure_zone: str main_wall_construction: str - class ElevationInfo(BaseModel, table=True): elevation_type: str cavity_wall_depth: str is_insulation_present: bool insulation_type: str - elevation_table: Optional["MainElevation"] = Relationship(back_populates="elevation_info") + main_elevation: Optional["MainElevation"] = Relationship(back_populates="elevation_info") + + elevation_id: Optional[uuid.UUID] = Field(foreign_key="elevation.id") + elevation_table: Optional["Elevation"] = Relationship(back_populates="info") + class MainElevation(BaseModel, table=True): elevation_info_id: uuid.UUID = Field(foreign_key="elevationinfo.id") #SQLAlcemy things - elevation_info: ElevationInfo = Relationship(back_populates="elevation_table") + elevation_info: ElevationInfo = Relationship(back_populates="main_elevation") -class Elevation(BaseModel): +class Elevation(BaseModel, table=True): protected_conservatory_or_aonb: bool material_type: str visible_signs_of_existing_wall_insulation: str ground_level_bridge_the_dpc: bool - - info: List[ElevationInfo] = Relationship(back_populates="") + + info: List["ElevationInfo"] = Relationship(back_populates="elevation_table") diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index f405a91..8610b87 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -16,7 +16,8 @@ from etl.models.conditionReport import ( AssessorDetails, InspectionAndProject, TheProperty, ElevationInfo, - MainElevation + MainElevation, + Elevation, ) from etl.models.topLevel import( @@ -112,13 +113,17 @@ class surveyedDataProcessor(): ) # Main elevation - elevation_info = self.condition_report.master_obj.general_information.main_elevation.elevation_info.model_dump() - elevation_info = self.upsert_record( - db_session=db_session, - model_class=ElevationInfo, - data_dict=elevation_info, - lookup_field=None - ) + def upload_elevation_and_return_elevation(data): + return self.upsert_record( + db_session=db_session, + model_class=ElevationInfo, + data_dict=data, + lookup_field=None + ) + + elevation_data = self.condition_report.master_obj.general_information.main_elevation.elevation_info.model_dump() + elevation_info = upload_elevation_and_return_elevation(elevation_data) + main_elevation = self.upsert_record( db_session=db_session, model_class=MainElevation, @@ -128,11 +133,27 @@ class surveyedDataProcessor(): lookup_field=None ) - # elevations multiple - for i, elevation in enumerate() - # Other elevations + # elevations multiple, elevation definition first then elevationInfo + elevations_data = self.condition_report.master_obj.general_information.elevations.model_dump() + elevations = self.upsert_record( + db_session=db_session, + model_class=Elevation, + data_dict={ + "protected_conservatory_or_aonb": elevations_data.get("protected_conservatory_or_aonb"), + "material_type": elevations_data.get("material_type"), + "visible_signs_of_existing_wall_insulation": elevations_data.get("are_there_any_visible_signs_of_existing_wall_insulation"), + "ground_level_bridge_the_dpc": elevations_data.get("does_the_existing_ground_level_on_any_elevation_bridge_the_dpc"), + }, + lookup_field=None, + ) + for i, elevation in enumerate(self.condition_report.master_obj.general_information.elevations.info): + data = elevation.model_dump() + data.update({"elevation_id": elevations.id}) + elevation_info = upload_elevation_and_return_elevation(data) + + print(elevations.info) print("Check database please") def load_pre_site_notes_summary_table(self, db_session): @@ -505,6 +526,7 @@ class surveyedDataProcessor(): # Merge additional fields if provided if additional_fields: clean_data.update(additional_fields) + print(clean_data) if lookup_field is not None: lookup_value = clean_data.get(lookup_field) @@ -526,6 +548,7 @@ class surveyedDataProcessor(): # Filter out invalid fields that don't exist in the model class valid_fields = [field for field in clean_data if hasattr(model_class, field)] + print(f"valid fields {valid_fields}") clean_data = {field: clean_data[field] for field in valid_fields} print(f'clean data is {clean_data}') diff --git a/migration_db.sh b/migration_db.sh index ec9c0ce..0adb6c8 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,5 +1,6 @@ -#poetry run alembic revision --autogenerate -m "delete everythign and start again" -poetry run alembic upgrade +1 +#poetry run alembic revision --autogenerate -m "added new column" + +poetry run alembic upgrade head # See which hash I'm at #poetry run alembic current From db1635ce4c1908e1c5cad2405312bb40747fcf21 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 20 Jun 2025 16:05:01 +0000 Subject: [PATCH 10/22] access and elevations --- .vscode/settings.json | 1 + .../versions/544f060f1eb8_add_elevation.py | 132 ++++++++++++++++ ...1_is_there_an_fourth_external_elevation.py | 38 +++++ .../versions/ceffde5b28ac_added_new_column.py | 43 ++++++ etl/models/conditionReport.py | 76 ++++++++- etl/surveyedData/surveryedData.py | 144 +++++++++++++++--- etl/transform/conditionReportTypes.py | 1 - migration_db.sh | 4 +- 8 files changed, 411 insertions(+), 28 deletions(-) create mode 100644 alembic/versions/544f060f1eb8_add_elevation.py create mode 100644 alembic/versions/c373a9a083f1_is_there_an_fourth_external_elevation.py create mode 100644 alembic/versions/ceffde5b28ac_added_new_column.py diff --git a/.vscode/settings.json b/.vscode/settings.json index ea20d57..b323fda 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "python.REPL.sendToNativeREPL": true, "notebook.output.scrolling": true, "terminal.integrated.defaultProfile.linux": "bash", + "editor.rulers": [79], "terminal.integrated.profiles.linux": { "bash": { "path": "/bin/bash" diff --git a/alembic/versions/544f060f1eb8_add_elevation.py b/alembic/versions/544f060f1eb8_add_elevation.py new file mode 100644 index 0000000..e4a9167 --- /dev/null +++ b/alembic/versions/544f060f1eb8_add_elevation.py @@ -0,0 +1,132 @@ +"""add elevation + +Revision ID: 544f060f1eb8 +Revises: 881142a89edb +Create Date: 2025-06-19 09:54:43.770068 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '544f060f1eb8' +down_revision: Union[str, None] = '881142a89edb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('secondaryheating') + op.drop_table('heating_from_condition_report') + op.drop_table('generalconditionheatingsystem') + op.drop_table('conditionreport') + op.drop_table('mainheatingtwo') + op.drop_table('conservatoryoroutbuilding') + op.drop_table('mainheatingone') + op.drop_table('elevation') + op.drop_table('generalinformation') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('elevation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('protected_conservatory_or_aonb', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('material_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_existing_wall_insulation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('does_the_existing_ground_level_on_any_elevation_bridge_the_dpc', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='elevation_pkey') + ) + op.create_table('mainheatingone', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('as_defined_by', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('fuel', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='mainheatingone_pkey') + ) + op.create_table('conservatoryoroutbuilding', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_conservatory', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_a_cellar_present', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_an_outbuilding', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='conservatoryoroutbuilding_pkey') + ) + op.create_table('mainheatingtwo', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_main_heating_two', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='mainheatingtwo_pkey') + ) + op.create_table('conditionreport', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('project_site_name', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('property_reference_code', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('property_address', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('postcode', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('general_information_id', sa.UUID(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['general_information_id'], ['generalinformation.id'], name='conditionreport_general_information_id_fkey'), + sa.PrimaryKeyConstraint('id', name='conditionreport_pkey') + ) + op.create_table('generalconditionheatingsystem', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_the_heating_system_in_working_order', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('does_the_occupant_have_a_smart_meter', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_smart_monitoring_devices', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_the_gas_and_electricity_meters_accessible', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('dual_or_single_electric_meter', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='generalconditionheatingsystem_pkey') + ) + op.create_table('heating_from_condition_report', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('room_stat_in_temperature_in_celsius', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('room_stat_location', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('is_the_heating_pattern_known', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name='heating_from_condition_report_pkey') + ) + op.create_table('externalelevation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('structural_defects_of_elevation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('does_any_structural_defect_need_resolving_before_retrofit', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_signs_of_water_penetration_caused_by_failed_rainw', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_movement', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_externa', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='externalelevation_pkey') + ) + op.create_table('generalinformation', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('assessor_details_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('inspection_and_project_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('the_property_id', sa.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['assessor_details_id'], ['assessordetails.id'], name='generalinformation_assessor_details_id_fkey'), + sa.ForeignKeyConstraint(['inspection_and_project_id'], ['inspectionandproject.id'], name='generalinformation_inspection_and_project_id_fkey'), + sa.ForeignKeyConstraint(['the_property_id'], ['theproperty.id'], name='generalinformation_the_property_id_fkey'), + sa.PrimaryKeyConstraint('id', name='generalinformation_pkey') + ) + op.create_table('secondaryheating', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('is_there_a_secondary_heating', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('fuel', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('electric_heating_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('gas_heating_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='secondaryheating_pkey') + ) + op.create_table('propertyaccess', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('are_there_any_road_restriction_in_the_locality', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_on_street_parking_available', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('are_there_any_overhead_wires_or_cables', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_the_access_gated', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_wall_ar', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_restricted_space_for_contractors_to_access_the_roof_ar', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighb', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_ginnel', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('is_access_to_the_rear_provided_by_use_of_a_secured_alleyway', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='propertyaccess_pkey') + ) + # ### end Alembic commands ### diff --git a/alembic/versions/c373a9a083f1_is_there_an_fourth_external_elevation.py b/alembic/versions/c373a9a083f1_is_there_an_fourth_external_elevation.py new file mode 100644 index 0000000..eb3904a --- /dev/null +++ b/alembic/versions/c373a9a083f1_is_there_an_fourth_external_elevation.py @@ -0,0 +1,38 @@ +"""is there an fourth external elevation + +Revision ID: c373a9a083f1 +Revises: ceffde5b28ac +Create Date: 2025-06-20 15:08:31.752580 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'c373a9a083f1' +down_revision: Union[str, None] = 'ceffde5b28ac' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('externalelevationgabletwo', sa.Column('is_there_a_fourth_external_elevation', sa.Boolean(), nullable=False)) + op.drop_column('externalelevationgabletwo', 'do_all_answers_for_the_front_elevation_apply_to_this_wall') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('externalelevationgabletwo', sa.Column('do_all_answers_for_the_front_elevation_apply_to_this_wall', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.drop_column('externalelevationgabletwo', 'is_there_a_fourth_external_elevation') + op.add_column('externalelevation', sa.Column('are_there_any_visible_signs_of_cracking_to_the_existing_externa', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('externalelevation', sa.Column('any_signs_of_water_penetration_caused_by_failed_rainwater_goods', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.drop_column('externalelevation', 'are_there_any_visible_signs_of_cracking_to_the_existing_external_finish') + op.drop_column('externalelevation', 'any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework') + # ### end Alembic commands ### diff --git a/alembic/versions/ceffde5b28ac_added_new_column.py b/alembic/versions/ceffde5b28ac_added_new_column.py new file mode 100644 index 0000000..5b29461 --- /dev/null +++ b/alembic/versions/ceffde5b28ac_added_new_column.py @@ -0,0 +1,43 @@ +"""added new column + +Revision ID: ceffde5b28ac +Revises: 544f060f1eb8 +Create Date: 2025-06-19 10:23:37.567361 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + +# revision identifiers, used by Alembic. +revision: str = 'ceffde5b28ac' +down_revision: Union[str, None] = '544f060f1eb8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('elevation', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('protected_conservatory_or_aonb', sa.Boolean(), nullable=False), + sa.Column('material_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('visible_signs_of_existing_wall_insulation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ground_level_bridge_the_dpc', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.add_column('elevationinfo', sa.Column('elevation_id', sa.Uuid(), nullable=True)) + op.create_foreign_key(None, 'elevationinfo', 'elevation', ['elevation_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'elevationinfo', type_='foreignkey') + op.drop_column('elevationinfo', 'elevation_id') + op.drop_table('elevation') + # ### end Alembic commands ### diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index a891c4b..cd5aab5 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -3,9 +3,7 @@ from sqlmodel import SQLModel, Field, Relationship from typing import Optional, List import uuid from datetime import datetime - -class BaseModel(SQLModel): - id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) +from etl.models.topLevel import BaseModel, Documents class AssessorDetails(BaseModel, table=True): assessor_name_and_id: str @@ -25,7 +23,7 @@ class TheProperty(BaseModel, table=True): orientation_in_degrees_front_elevation: str exposure_zone: str main_wall_construction: str - + class ElevationInfo(BaseModel, table=True): elevation_type: str cavity_wall_depth: str @@ -51,3 +49,73 @@ class Elevation(BaseModel, table=True): ground_level_bridge_the_dpc: bool info: List["ElevationInfo"] = Relationship(back_populates="elevation_table") + +class GeneralInformation(BaseModel, table=True): + assessor_detail_id: uuid.UUID = Field(foreign_key="assessordetails.id") + inspection_and_project_id: uuid.UUID = Field(foreign_key="inspectionandproject.id") + the_property_id: uuid.UUID = Field(foreign_key="theproperty.id") + main_elevation_id: uuid.UUID = Field(foreign_key="mainelevation.id") + elevations_id: uuid.UUID = Field(foreign_key="elevation.id") + + assessor_details: AssessorDetails = Relationship() + inspection_and_project: InspectionAndProject = Relationship() + the_property: TheProperty = Relationship() + main_elevation: MainElevation = Relationship() + elevations: Elevation = Relationship() + +class PropertyAccess(BaseModel, table=True): + are_there_any_road_restriction_in_the_locality: bool + is_on_street_parking_available: bool + are_there_any_overhead_wires_or_cables: bool + is_the_access_gated: bool + is_there_restricted_space_for_contractors_to_access_the_wall_area: bool + is_there_restricted_space_for_contractors_to_access_the_roof_area: bool + more_than_1_5_meters_in_width_to_fence_or__along_the_full_gable_elevation: bool + is_access_to_the_rear_provided_by_use_of_a_ginnel: bool + is_access_to_the_rear_provided_by_use_of_a_secured_alleyway: bool + + +class ExternalElevation(BaseModel, table=True): + structural_defects_of_elevation: str + does_any_structural_defect_need_resolving_before_retrofit: bool + any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework: bool + are_there_any_visible_signs_of_movement: bool + are_there_any_visible_signs_of_cracking_to_the_existing_external_finish: bool + +class ExternalElevationFront(BaseModel, table=True): + external_elevation_id: uuid.UUID = Field(foreign_key="externalelevation.id") + external_elevation: ExternalElevation = Relationship() + +class ExternalElevationRear(BaseModel, table=True): + do_all_answers_for_the_front_elevation_apply_to_this_wall: bool + external_elevation_id: Optional[uuid.UUID] = Field(foreign_key="externalelevation.id") + external_elevation: Optional[ExternalElevation] = Relationship() + +class ExternalElevationGableOne(BaseModel, table=True): + do_all_answers_for_the_front_elevation_apply_to_this_wall: bool + external_elevation_id: Optional[uuid.UUID] = Field(foreign_key="externalelevation.id") + external_elevation: Optional[ExternalElevation] = Relationship() + +class ExternalElevationGableTwo(BaseModel, table=True): + is_there_a_fourth_external_elevation: bool + external_elevation_id: Optional[uuid.UUID] = Field(foreign_key="externalelevation.id") + +class ConservatoryOrOutbuilding(BaseModel, table=True): + is_there_a_conservatory: bool + is_there_a_cellar_present: bool + is_there_an_outbuilding: bool + +class AccessAndElevations(BaseModel, table=True): + property_access_id: uuid.UUID = Field(foreign_key="propertyaccess.id") + external_elevation_front_id: uuid.UUID = Field(foreign_key="externalelevationfront.id") + external_elevation_back_id: uuid.UUID = Field(foreign_key="externalelevationrear.id") + external_elevation_gable_one_id: uuid.UUID = Field(foreign_key="externalelevationgableone.id") + external_elevation_gable_two_id: uuid.UUID = Field(foreign_key="externalelevationgabletwo.id") + conservatory_or_out_building_id: uuid.UUID = Field(foreign_key="conservatoryoroutbuilding.id") + + property_access: PropertyAccess = Relationship() + external_elevation_front: ExternalElevationFront = Relationship() + external_elevation_back: ExternalElevationRear = Relationship() + external_elevation_gable_one: ExternalElevationGableOne = Relationship() + external_elevation_gable_two: ExternalElevationGableTwo = Relationship() + conservatory_or_out_building: ConservatoryOrOutbuilding = Relationship() diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 8610b87..367e7c7 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -18,6 +18,15 @@ from etl.models.conditionReport import ( TheProperty, ElevationInfo, MainElevation, Elevation, + GeneralInformation, + PropertyAccess, + ExternalElevation, + ExternalElevationFront, + ExternalElevationRear, + ExternalElevationGableOne, + ExternalElevationGableTwo, + ConservatoryOrOutbuilding, + AccessAndElevations, ) from etl.models.topLevel import( @@ -50,7 +59,7 @@ class surveyedDataProcessor(): def load_condition_report(self, db_session): # My task to complete load - # [] general_information = self.get_section_1() + # [x] general_information = self.get_section_1() # [] access_and_elevations = self.get_section_2() # [] rooms = self.get_section_3() # [] heating_system = self.get_section_4() @@ -69,22 +78,104 @@ class surveyedDataProcessor(): # occupancy_assessment=occupant_assessment, # ) # - # Load General Information: - # - # assessor_details = self.get_assessor_details() - # inspection_and_project = self.get_inspection_and_project() - # the_property = self.get_the_property() - # main_elevation = self.get_main_elevation() - # elevations = self.get_all_elevations() - - # return GeneralInformation( - # assessor_details=assessor_details, - # inspection_and_project=inspection_and_project, - # the_property=the_property, - # main_elevation=main_elevation, - # elevations=elevations + # Load Assessor Information: + # return AccessAndElevations( + # property_access=pa, + # external_elevation_front=front, + # external_elevation_back=back, + # external_elevation_gable_one=one, + # external_elevation_gable_two=two, + # conservatory_or_out_building=co, # ) - self.load_general_information_from_condition_report(db_session) + general_information = self.load_general_information_from_condition_report(db_session) + _ = self.load_access_and_elevations_from_condition_report(db_session) + pprint(_) + + def load_access_and_elevations_from_condition_report(self, db_session): + pa = self.get_attribute_and_load( + self.condition_report.master_obj.access_and_elevations, + "property_access", + PropertyAccess, + db_session=db_session, + additional_fields={ + "more_than_1_5_meters_in_width_to_fence_or__along_the_full_gable_elevation": self.condition_report.master_obj.access_and_elevations.property_access.is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation + } + ) + print(f"hello junte {pa.id}") + + + # Front + obj = self.condition_report.master_obj.access_and_elevations.external_elevation_front + + external_elevation = self.get_attribute_and_load( + obj, + "external_elevation", + ExternalElevation, + db_session=db_session, + additional_fields={ + "any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework": obj.external_elevation.are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework, + }, + ) + + external_elevation_front = self.upsert_record( + db_session=db_session, + model_class=ExternalElevationFront, + data_dict={ + "external_elevation_id": external_elevation.id + }, + lookup_field=None, + ) + + def external_elevation_load_for_side(obj, pydanticModel, different="do_all_answers_for_the_front_elevation_apply_to_this_wall"): + external_elevation = None + data_dict={ + different: getattr(obj, different), + } + + if getattr(obj, different) is False: + if getattr(obj, "external_elevation") is not None: + external_elevation = self.get_attribute_and_load( + obj, + "external_elevation", + ExternalElevation, + db_session=db_session, + additional_fields={ + "any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework": obj.external_elevation.are_there_any_signs_of_water_penetration_caused_by_failed_rainwater_goods_or_pipework, + }, + ) + data_dict.update({"external_elevation_id": external_elevation.id}) + return self.upsert_record( + db_session=db_session, + model_class=pydanticModel, + data_dict=data_dict, + ) + # Back + back = external_elevation_load_for_side(self.condition_report.master_obj.access_and_elevations.external_elevation_back, ExternalElevationRear) + # One + gable_one = external_elevation_load_for_side(self.condition_report.master_obj.access_and_elevations.external_elevation_gable_one, ExternalElevationGableOne) + # Two + gable_two = external_elevation_load_for_side(self.condition_report.master_obj.access_and_elevations.external_elevation_gable_two, ExternalElevationGableTwo, different="is_there_a_fourth_external_elevation") + # CO + co = self.get_attribute_and_load( + self.condition_report.master_obj.access_and_elevations, + "conservatory_or_out_building", + ConservatoryOrOutbuilding, + db_session=db_session, + ) + + return self.upsert_record( + db_session=db_session, + model_class=AccessAndElevations, + data_dict={ + "property_access_id": pa.id, + "external_elevation_front_id": external_elevation_front.id, + "external_elevation_back_id": back.id, + "external_elevation_gable_one_id": gable_one.id, + "external_elevation_gable_two_id": gable_two.id, + "conservatory_or_out_building_id": co.id, + } + ) + def load_general_information_from_condition_report(self, db_session): assessors_data = self.condition_report.master_obj.general_information.assessor_details.model_dump() @@ -153,8 +244,18 @@ class surveyedDataProcessor(): data.update({"elevation_id": elevations.id}) elevation_info = upload_elevation_and_return_elevation(data) - print(elevations.info) - print("Check database please") + return self.upsert_record( + db_session=db_session, + model_class=GeneralInformation, + data_dict={ + "assessor_detail_id": assessor_details.id, + "inspection_and_project_id": inspection_and_project.id, + "the_property_id": the_property.id, + "main_elevation_id": main_elevation.id, + "elevations_id": elevations.id, + }, + lookup_field=None, + ) def load_pre_site_notes_summary_table(self, db_session): summary_data = self.pre_site_note.survey_information.model_dump() @@ -179,7 +280,7 @@ class surveyedDataProcessor(): lookup_field="UPRN", ) - def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None): + def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None, additional_fields={}): found = getattr(obj, attr_string, None) if found: print(f"Uploading to data base {found}") @@ -190,7 +291,8 @@ class surveyedDataProcessor(): db_session=db_session, model_class=pydanticModel, data_dict=found.model_dump(), - lookup_field=lookup_field + lookup_field=lookup_field, + additional_fields=additional_fields ) return db return None @@ -517,7 +619,7 @@ class surveyedDataProcessor(): db_session, model_class, data_dict, - lookup_field, + lookup_field=None, update_if_exists: bool = False, additional_fields: dict = None ): diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index 3a099ba..0b33da8 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -51,7 +51,6 @@ class PropertyAccess(BaseModel): is_there_restricted_space_for_contractors_to_access_the_roof_area: bool is_there_more_than_1_point_5_meters_in_width_to_fence_or_neighbouring_boundary_along_the_full_gable_elevation: bool is_access_to_the_rear_provided_by_use_of_a_ginnel: bool - is_access_to_the_rear_provided_by_use_of_a_ginnel: bool is_access_to_the_rear_provided_by_use_of_a_secured_alleyway: bool class ExternalElevation(BaseModel): diff --git a/migration_db.sh b/migration_db.sh index 0adb6c8..00eede9 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,6 +1,6 @@ -#poetry run alembic revision --autogenerate -m "added new column" +poetry run alembic revision --autogenerate -m "access and elevations" -poetry run alembic upgrade head +#poetry run alembic upgrade head # See which hash I'm at #poetry run alembic current From 743e27cfe413931354e0c4ae150cc0db551f999d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 23 Jun 2025 09:56:19 +0000 Subject: [PATCH 11/22] updated aidan automation --- etl/osmosis_data/asset_list.xlsx | Bin 11718 -> 27461 bytes ...osmosis_monday_to_sharepoint_automation.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx index d891cb51fa1b88279d37a20c0d2be33a60a3be4f..ceada9c65be1198d05dd8027d96138f840418fd9 100644 GIT binary patch literal 27461 zcmeFZcQjmI`!}qFBzlA)L`Xz82!iN>Nc0-L_uhLa>gbUnx)ELU-g_Hu^e!Qy6LqxV zJ#v5V-@Tsqxz~FCc>a6WV=ZUSoW0LJdtd$fT>Ffo4CXx&G&Hma!2feJ>Qr6G0dN+H zj)q1Ae45xBD>~UbII|l%IIwxx+N8zFDRgn*g&w@W4r4^Ve)9ft%DC*Bj1E0t)Kbpm z-U}@O>CVX;Z@kA?j=#*VYN*?4=c2^f<>Q=aO4cjKmC?McFx+ZzV-JQu6eyJ zSih%fx4RzV`GMs57c@-{3}=_k=58k2n zUkQYj2Af2u$C+zFkFme5z=%0<0)xgzq%W7`sk94`VHzLk3(a>&h zG0_zN7ve|@191FEkZ5$9%ZtU*Q#ME5Xn2YomiJZRAO59^*Zs3MfnuH-V7&yq&MHrWJL-0W*$m zS_;Fr^dLVHMcxnz+?S@SsB=(mw3gqFPWoKF)2ONb&T>{HpPjIcdQC;8@O~LzTpp_J#e)wI}Y5ZEeusA$fj5r)J{2MSw{lyzW3G2O?F`vR|U&#f-nFUWDk<-DtS> zY-*;JiziL8_`l|oYR8#j?S(lU4v3J%y+aG)*CAO-aq6cFa=#KZpua435pUlLX z?NxbHQFI{$Uv!;L4}H$GE*U?v3y8X7IYFb^Ac6DLD=OFMICc6}#P z8|Qz2eRaoKw!14O@j~`pG(@3DX%rzyye@@LT&?E0=DhTiM%67F-5*9d?Nk^E^S7-U zN+K^Lx5x1bJeCPT#T`xC;jc95=~BGtnO_d~D(81!{a!!QQ)R+@;viel9sZ#NI-?HZ zb%}%LOfJL~z0Lk6k?HIa$GP=P=vxY=@|!uwlQ=k!8)v|>c$kxL>e9e@>T{_1kd zxHT8qlo_7ng+0%K+sC!XJHb+eWG}y+N;aWAcT4_KV7-OV;2TcmS7i8QN;&L5zbTWo z$-3uXBDiHGV;d3ihDawvH!Q}wL*a@8^tl1b6fX$ebe{?yZ1(nHhDX>rYk$j92 zKA8~S-SC^opD=Xq_W*ar5Fm^rj(6Ews4y!|R96pedE`~nx&^87UnyR_&fnouzBFy) z37>Hwp_P0&MqnEcBh}f122Ct|{niH4;l*S6ZmtI zl97q-lYjF?PoGI+Sj7IB5AhgsWlWMgXye)j>p>3rH?+Wv*Grn;PAciF;LmLO3W8HH z1~*5&2c2ZGAhiar`kC;{)Jt zMLn^8YnP|z2a@#lw%b0(`V-UM^+S)t{*NsU2Zc9Znrz2V$E+vMKyOYtCf|OndvBwH ztjsUWa&m(QAL+fbgkf8C0rcu<&zlzsnE5o^C7&KjU6-!xyC)y}=UfN?lJ{>bjsu+E z{XIt;>NtB;$%2M<_x~8U0w}(%sfnc_yQQt6x#=r52Rri!MR_TF9E!Vp@ISnlP)0+; zGyu+5A7BGVMvQOUXlN{+A0)(7J@odLG##FvF6}#RfA`v1Oe=EUZt)%+E_7y-Z<(7l zqIsxILqz;u1H%wso91O;c?jLhmvLARNYJB@zWOHG%kFEwE9wumYgs)%+s;A3b4M@O zj}X^h&uma7ACXb-zE@nM6i2Z>Dk!**MqT^o-P|Be%>kQ*h89sjbo=)t%T=-c?gGto z5{|pi_Ybh}?mmg%q620RE#TAtZN4MN|0T=+fjB&Q<8@MyTUl8-H#t46g_b6MH`6mTdw7rMPofg7+pHTksGWotfPsW&Fo#L^avwpZ)=IFrw@6BwCa z4t#x9*S2KS=zd^_?m&RW3Ri3omu= zx|LW%ge51jy9^3nAF-_U%GI7wb4_)PPYX5oqmXZ!4l=hAxHbgi$fkKghLLj zPXM4)zdZ>DjW<6Ek&)U(<@mRJMJcg=?^~;wlR<8`> zz8K9)DuW%*Yu7jx{NN5%E9+RTvpoTWxtik6gssV`Z=>u{IPmy+E%*)UJ}W=_t>4s6 zfe11)vzE=k)s=zOd}LDFWszUcmw<%@sa=V>o}8M}yuMnO=n_^QfG$hHG+`I&lKyg$Bl_{;PG+Rga{^w! zMYm25dW&A18({k9B0FIHj*)NvvQ&eGlmmb0`WTO#Z-_YQtjf-}BG;f>18z>$q%Djp zapZU3vVFqiQ*F96{sn};n`4o+T&3cs>f5u2eTD9N+%oh^0Dy5Qb^l~@D1*Jlu*`A*3N2~!eZOT>iF?9 zK8JSLjCNV1AsyL`%`F+d_T6aeDN>hsR~-uu`;Ji_RV%XFxkU51}pINKKf9|GdJ1Fednh zOeb+bB~QDW+O(If3hZdfoD-RS#?_oPMl6Ymzw};vMkPe8M?=!!Pn;686@{(svLxWX zWvN+d5<<7HKgm3D!#|YSHoF>|uYQ0I752r}mX?9D8qQx{e=|ZpgJ+4+vd)){)akcn zLpy?{w6FM?1A}tuN^fj(G^1tKGHeWzk`OBUO-PdsOJ@uU2P>G#5YkcMH zdEq8Q?t2JB?Obl%a#0mn83?(|T0qY7DSHoA^i3_@L1opG`pAIw-BUvBhQsVU=cUmg zpRw2Pb*{Xyln%8n?1!Z!ZICO$zh?8~Qd#Yr58qX7Ej86iVvdF2l44F|1Ptp2DSeXtCawc>{L_6^3ligLrL;1mkes9FKA`iutAO6^YC zii`$;Y-vzt1-z#GD`M}^#TT*yazF22)Y>k{L;RnS7I9oc98QuWK=_N?iWEX#&AH!x zK2swxI||+LB-ADQkNV5t|`krXZ?1Y#Y)hq!!EqCl%*& z8-Xt+c6MQbhU8p>b6wAY`Bvo@D92?8ui>gHq%=Qo`OZPZBvrV3c7Jq;L)m$u8aVHw zYB%|PmJ9Pu*LsJcjE0zAq;#aqAMpDU1}oQ#S*>U6$~Rrlj@GZ`(+a8zJY4q2`(SJ2 zB8HKYR|0k9eRl9cHEVDP{=uH=$@Ps_G$rrc_&iLnOev>V^Mv1arF8=KB4>2t*T4ig zWqSjXnAiVFDf_H1dj`zbvh3xzCspR%l+(lg%!?_UT`T@!9&?6}4&_T(UCp@HNmz%H zEH6%XZkC+RY_{DEP7XTiL97}E7t?jEQ#O95#rm4>VxQSITy7VGFE75R>sdBz7ZtN! z-rD;er-D#z*sBzABgwD2oKUAT+Sg}Z&-RctD{n*Wt*cl>Cce;w6Pi>c0HSjON^ss?}M82o!y)@pNsV*Fu%R-eO)qt-`2xu zdkw#n9BR$q!>Q&GFm?kdyS|<0W_B3s&E@6=OfPfg`Zq6hRQPIW6*oyi6@7v%+>5&C z+!xS=y7QgY*wHpS`!-bVdd-Yp)%nf=m*2Gq1g9dhVMQKs^=2Zd+GdUu^6UUvym|5Z zHOby%ELJKU4P%n>MNnb^x)5Nfrb%!#pnpYcR-Ms+tQC7B3@~1zfbmkt+MTUf1L1$G z@yGZ1SfXS5u*z)MO9pTZV6U`1y^F$UKb3FW+irFj0Ru4~%jv%SR%X@+jg-`zRktY@ z+j;}r=2Gffsrr4Oo#=OT^O=2LFn`S6g34`{Tay1dR&{c?|I*pV{i;t)%*z%^uMlgh ziYocb-CO!umx|GnDuOkc&&A(qT#Lw`bz+~SROXrf~5h7 z0znWknj5bgSM-TQ^(99B$h-$~nZoeZvQbzMVC$+GrISKO4Vd|34j<=fu8zN8_i7eS zvYxgnuNGJe!@5`=4c1XCe*r6uOoaz~oaP553Oz9~E%GtdYi~`d^!V)%_f)gmBEE<4 zlka}ptsey2hhp|}uXFW&XUb77lgp*^ZGC}7JL>v)R2$}ZTkyyCzc6;1>!+zK)?N7XI4rAk70J}LcdI_#wcgIk38?MUfvoiocZ z&Zb1BHs_~cD6^|pB~^zLtbp5w(e8-H;G?=W)G+a>=Q&*e#wcRvd);#7If~IsSWZde z>fN0?nkL68sepnK%K^rgy!y*#);re(W+cbKD%nzO4^nI&BL*xAx2SSQL}VVpR%k!q zNo$6_HnqdhEAnpepYV4Q^EgaQhN5HQvhuUZ{amQCZ=+Mmr#SmM_~SViXir^*g78^U ztp^34RXnzkJj7E1h}fZ$u_}QleXqPnFk}+%B2%xl$hR7SlZVz;na}b~I|yfbAS>E}9M`3M0LI zt3|+rkIrHnPQ5$y_2n!lD+iR`#ZXwkL7xA>eY7OnxWveW>jXCz^QnkTJrJQV(N`lX z^OBCUfvEGJTHkg#)!vxIJq672HHkPWHU6j5TO~{-ln$o;o`U~AlClnM(~+4-@56_P z_1~%XwG#M?C{=Mu|NT03muJizl)QG;9*(%lnaUfn4Ev{uHSd#6!$a=e{nDoWN(o(R zf5DU^a<_EYS^g(d4$`ywjp98QK|kh9k&FW5_f=dCN5_iwdF!&8^F7j*1DGA5!{*p? zI6m*9!Qqusb$$Qk=?>Te2+sk&PyjI_cp~TpU^H>wnwkxCL(45+Kyq^9j?qs9 z2F8_BE;mQL#SuwRj@I|C2Z&MPTD-G?P%#tc#>3J)ep3I2fmJf(*wIqHLS}_Wir3K; zN$<&;*>HyaLuViz>CsmJTg8FpNKa;zRQ_~+!r$z|URDB(cr{qUJdWsvHFzr zMC;_GwpNtH)r2dqwu$8LVQrhkJ{6s21o1me%-i~kHG0kY=t;RQ70=Ta8YTE}n4}6w z|CT1)y@>fef>bPpQLO@27$%@ckYEfD92Qud_fq(HhzkUsCUU1Kwz%3UZ5lb6HICKv zyMXI!sXkaGXhx~B`yS@4AEDY8q%cpYZ_oRn`q!&u(J+x2x0}Prj1Bj;t8_{IPahUB z@=z9EFBmdBI$=A0qe4i)Q$K`25lH>_Zo9Jx;u`7}0{&+$@>nqqs(3gKOB_)DpMT*>fqhC$bO{j)+a z;5CyZRj}^kKQsKw=?{y#f&p&WE{xge4DX(#0J zT`n}=DiwPJouAi&WNPK@sfr4L5BBQVtv&WtJ1xH2`M0X%jg;HWRlY7R(bu;*yi$8k z_?Rb)kfrjDgM;uo&FeE>)m{+LSXlOf$ri@T60;tzr>3M7hvHMXh$yIIByr|V6vcRrRmP}_h?SjK@zz0sE~OCMf(naVy-^0^plzz^FcM4+f=bkdl4DTZqWGCP1SLFCu_#^@?;Ct zB61p%T$*XeT1pw-C)8Swk%yF=blgU*7ZORe#LW8}8fIf9VdH4GMVeN5hQK;QJ?x=?7S>4**VrHqHUH#0Au{m05nmtY?snYeNIXjmN2?S!4A*x4L?0 zHJ@HWv(Y64U}h-3zLr)|W8?fPvTXgR>#2|z(26#a+Sb21{I#kbw&${Dfna~BYj0a; zfl!9zn>N;@gWSU@9+c@c^sbJo^PZoE@Y|_2tId)w@d?P5-aol;^4pmv)x35r(X1Lt z_UVV)j^wmM5Wylzf2eb6|0|! zF!Kv99aYCMzrGkM>j>-p#RgCdn_ru7Yrq!v4?h7w37g#ZD!X4h{dx~+eemb%^dXCZ zmW>>5>shSIiQuJ?sI-g>Sg9;C&+pxmJlY=a-#`Yim0%XegdKF>)--A?RxXnAr+_Jq zzyN*1cn2in&-DRC`})Q@BlmlW1-WegYU~c2omW#!8a{l5>eFQvKiDJ}U8tDg2|R1pIc#?eVmzB)#n#?@olOckj?>#NT{t;EJc`({+%0-;1E zYS$cUWBCfNO`lr7-KEQ>Ej<>;tM-{ZyVXzR&f|~mCf(tWpapM>Df!@l{gi^-g^Z2b zb#@>ft-o$51%R-G4Y4HuwbYyd(@+d3B}w^jrMp0aZSnW@zak0{(0M5B&CK{m>AFNf z7{U4#gX79tPrDBMx2rXwj9UBiB7V0w%rZd62*C~&@;zz19zylyBM3@I#YwWGQYd8_|B{%cXBZDF!ubPL1oK6Bf~_l zJAg}%)!MDBtXP!}R8T&E^g~na8V6SER9sR+BO`He*}#8uykS6aGYbMe|E1O>;sQFvd!w_b-Y8_uwB+Wx0;PMF!C$0;yeDtY)m2E$J<@PQ|>A78$f{&P6 z{T})XY7@!iUH0gD0l0gRdAN*OK;Qu`0JzR4AoSt^ z5~06##QM&@&o>5ak*VCGcY*oYz?8n5iZyg9%>3e^F_ji==}jlKCj5VnF((kl^ct6E zls1UWK3!pbEX7z#K8Iw$<7h&iM<-4=Hw{m4Md8!dBEFY+wH7!gwJ<*=&znZ3IZ!#^ zTw3g}&M2)Ajuw$=k*aw>W(4GSQ`1;BqZutw*)9If;}X=VRsFldPpA-5v%V1=2zzUz z+&knm6!`L(P9{m4%`5W^H(?J(IXJ5KKv|8do5h#ry^MW*^VTJedCa|sN?ulFrUQq> z@N6yy);X&KK$1b{nOE1BGi|jd(wG(iOGZU*7hhZ`w*xS&6~G6C?SpC`UmXfQDY*)j z>laS}5wGj9-^eM<`#SFQXGY(@rRQLV!47?52ZY2PHALX&ZH#8xM-&VCHiZMNOHOyu z)_(mO0X&#c*aET9D|tn2{Y`e6WS z1x}RO|E(mj9yZlD?)DTx&&D41iD#w&iUCT`Kg(a=Xww>blV^&5B@KWNykoc^#F#4M zQaeW^6n=*JK}1ov#VegoMQeFQUqLQj!1E}Pj&13W+v+z<=*2OaHY`$gHb8jW@h*yj zmTA}`j#*Ne0a(y&SHmIrM|{W{>%Z}fPCa(HuR>1Wrio|B^oJ_Wcd8tID$C@ZcpLo4 z`U#@SY%^>WPP$Cft>#I**MK^tX;thQb|t)bVPr^jU?3K!1|Sz9%yuW|^O9m|E#^gF z_T8IqC-u2wsJ2XIjiBmHLhe<^2dHcf5~J2>T^fR2=_eqmvc&JYZSwHS=tEm(qsx|2 zgceADw8>K1CL7Hj13h8iRcGs%InZQvc+Opax(} zjhEhUGm{qVnucL0JW8G+7-gi$$(T5ZwDELjM*Exd#t4+9w7wID^aik2a3d-ZM2i@8 z6+clBK}{-aXjpizMli4fRv{<1prGHW5Wu#?(IH_NP+c(~VU!YLlfHBEC7vYZAXtFu zr++CY#M$Q#71O>0!q~B7RpN1Q8MOFa_f3@$2$GV$1$V6&9=wk#h&66WtLa@&5a|>A zbFY(K`!|Ye0GvL%N;tS#@fhs5MK=HEr!jzHP!~KC)UX{;)m{1^(-DpUk&^0;b|Tim z+I3LvlUP}WOkrb$DxX~8(>V@C1g$wRc?GiS6XvPf4?q#UX4mB(YlZ5aF-`uzRQ_@mzI!~k&P7V(u&RA z9tsO?n&QBUNbOqg5b@EqvK(mcXkzAk!Dh1-e?W;9V`^a(6>kr0r?7LpY(){C)j`X; zQEDET>z>&GA~k9uu%0fhlK{hrc(RcGf#${EM-}eslhw&l8TuQ=$nP<0i^umNZkre8 zKCc#5%ha_=%QIRowSTi~7;Q~9gaPG6bdc{eZ2V`I2+!?ztBnMgZ@)GBq~_*!=KW!| zrqpXvWz$j%3)&1F00Ns~a4Z(n3Td}MBlmk;`+6U@rCU6wBF^xtKNc11#jB_5u`iCq z;n53vHQlZiq~)^tG=tTa6Rb)4ehokdYY~u-UX@4BoT5yw)_fxPnO?5a6ohe-br8_KH zNpZ==19n#CQokoi#sbeJNE}G2-^dgq*@mZ{g-;`MaGoXf{m#F zkpra4a$ssh@#=|?SAt{@%S4aE(R9p`m=^zOxqtS$&2|c}>ryep z{X2uteOg{b^FG|}K7$q;tJd(w3NK*xf7`Us4dyDhbB9P{))lAI^n>PwOLXz8r4QH6_egcdlc*lfBi?KyIay(ZCY*@|OBCP+9jsZa_CkvyW zXxSYc1r|exJiscM17Hm{kT4KHm8mRtN}%Hc)lUknU1G=W$-h-!3c6h~tJX15jS$(} zS^vHmUFi7ucc9M20_HE1O?n=3RW(Ur4pz$wZAPzKa6;F4uPVV7D6u$v_Xf|pitp(|gZ9IoqP zmFYqC>mxL$48fn0eM}l8Rd9U~IPhCG-*Ms@fc6q?4BYLLqQd)Vi&C)@e$`Pe84R+m zq#Ovxw%eO45a}y@9T;ryHJU#F&H_0a0Or!)CAFK_t&RI%pJnRt%2|5<{%U^LWb+pv z>^T^f+V}|xAJK+{Z=Rtq7bxYh&aZrW_?R{0M(;?!x#eVB2DIeSkeAoHT1d3wceMvv z`lHE>^$I8~jsn@)2BSfP7&?4c5EamxJHsM9zCjiWWWLsqfCABO^IeiQ0|Z5PK@wT4 zOld$3F9qX>rkZ)KS#gu4lmet9R1 zlBaBedqt0DRLPdW&?`zko6EU zz4Gqd^>08p&j~i5;J8Aa1L!D%%&l0xtacN-OY6EFSTh87)3NyD_sVY6i$8()pOv4Z zE={!Bs|A}^Vih#dANxM%G492K`y!Xj53e3VMih9@s5lG)fY1|2r6k~qz?O$aSmzHw zjD52em6eSU_?aj-VIYdOTMAlYp|8|BYGz z4)p$CB@--01G%s}isJ%%FY5aXe#m8TcEMJ2d){hUZ#$;G@#KG>A1G#rl8ieL8D~6o zza4aP+e;EtrsMnk!l#k`{alXNN0b!*ZHr^%jegu3WpF`zkE)MR>big_yT(mlZ{|1W z{B;pt)Z|gvpda+}sX42o<)9|;SM@OUh~%A0B9Mf3=JACvv9s6NEH&25b?1(TfwIRO zN7DsnV1Z?KXsB1(+2bTTlby0Kg~hiFDMi1LYmbqc$`(Z_F*!h>#c9<2KxQ;U2+aJ} zdb*^-!do|KdmAXRr2Dv}Cf2+7@KfV&-eERdD{XD2*XM^IUH99@y_L3=)!4p`KR>&m z^(TT_Bwb`|)Jw^;6N#7W@FZ>6G8zBKUAY#t6s{BU0ZWeeJxDhetX-?2=XYTWx2;<; zQ$qJTol*yTiFoo8F8$$;!uhl0zG920JTc7hSOvTZZ$axUuyy9gi<~w^CIi5_D7J4i z84Lr}PlCIu6#zP*#pTQ2g=6~XTCxB>qXr^nH|>d>g1Xs{w$eB!VQW@1zqiUxJO;@O z=rt-*-E#JKJQ5@DfAsA}A`ZmwdRA?EdI*LN)WBJrp4vR%tZ}( zM!B6ux3RFjPU76}wbu53pT}tM9Qd3Ete)+^FF{!~?e2DLNb(MSRBb*vZfyJvLKC$s zWNbcPg@E+yeU^QZy0~K`ciVcH?hf~H$ekiEu+coNE>GToz+5XWQvNb!%fYNzSRtUL zF$%OXer9H7Rw{m%^7GEzO*ibFnX)=Gx$FRR(F_=H@XHE#p7eJ%($dmkF0-MEp^U2b zN<;CAb!Ai^94(G}seYD>EmMlxcM^j;?CwdCz~LegBUA^tAJ~p1H-`T?$FA82pucu+ zG^IIu%HIuJXepxT;?`OJbcs$XYd(^OL_$aJ0N1+>URV})yROjCEHJR}z6SzBqM@Zp zL7ui84YEM3%UWvZd`=KjAQ}Q-k!WZR&v3}K9{=snH~N*!oR!-Dx?=qfC+Eq{&HXRE z{_lpw|6gsy3}D?|s2f$*(y}W2{=EhbZS3itSZLl!SEbnh!B)fnPcb>VMOVg_EwYs+ zWCO$y?SrJeMCn_Dp#Swn4TbP5Mj~L3;$vWQIzG?~*y8K#;%Q^*e7BXq`J=Y=0tbm7 z^{&W+mZsL1BMAmg+;aFKc6jq0dZgNJ-7c>xmD6HSNUDnjK9>?XI8?O6nbmhgfHz$t zDf-7|{B{b{6XV;?uJA$Gby?`Eo3l86=O`<26Z!fC?p|-LHTbW!umT=$`GdMdihD$r zF^AL1R$Gm(wYfo)WJNIIWfEyzBhkrRZS|Cd=Gw4(?wF&g4M7|c0-tjG^)O$A-&7Q* z$wowy*BYjM^@W7Q_e1M)Ra0#E6m74W1{BvuT>>bC$SrF|cOO74)rvcH&`OIVgG^7} zEQK^Vd~lP%_*M7(B6|mmp_TVC`n`(!gp}Dt*yjSp@kvG%0ZXH&Cy_6UGRJ6t#_3Tv zeRvs>mF|=Kgd);$zj zXo?VS{~A61rg}UMjXHo|jo49({6UV{`m}I#(GT~Ws6ZzfSX#XDhfdpHVUJN523&&| zEw*(BJrCVfXc`_Da@2W7WjorAgwp-lPNLFd(71l36($sfA9!?8Ucf|fe!v&~UhNP? zYMx6n`LP_I=7~hLT8dC0q&C5pucD<64_=bi;a9Y9?{!a;4qvc@{fQ5S@UJ2T#Q+4K zSlD|vZe@p=(CnliLrRxPj+g%T^rH;5%M`P7c=#-X#NW_*m-U=O(s`fmZ??WfnDx~g z9ueZBC3`yCFpWZEE2?sAYlBHQOVU6leZyTpO`~FWM!C%eD)@FU2cLbhd~W2Uvobe= zToCxg^Sd;3fi%M+Uo*;oxkMN|&1?B%sAX#LtGvY~Yv_e;Ge4j@KkuWV5&lbc-0hvLjqL5M@06!R&DK7LkLbp49Cg3jJyMPXeX=?T ztxqy1Iyylk4guMM#iJ(U(NLd)AaXu20hFw*I=5- z=XhVSt(}zq{B|_-TQGszQ>4S>#8?pJAs3ze$49H9A|vi^g@qm*nxKhJfQP;-+gdE> zwwk`)8ihN_A+5?JT8w*oqSc$A7jf9wi8?{y2N~*qHt2Mh7)5 zdD;MRxNvfjCz~XLv@7=5+55}jTk<*bHr$RIV?23rC)uP3D+bX>ipc;n?+8?#Ut7uiMGe%`0#CT&ee| zK6^BZ${wdeJ2|d0iT9w%@G6GPN}#{@R6k0@E--(j%!NBC38h1tIw*SbA0lche7gU}!3embkX7wCp-Q{K|JKgmJa^6LkUn-aHF zp!M^9hI?-uSXa{DOpnW@B@Ruzj2;-ht&$R5Hzby~RljP;RknROoSCIrruk%oAnmC* zhR;eubOd$I6Ga!UG)*>wIMTih>-d+c@o(gzOapoY+X0E0&DpI&X=bXY#KNSaUK(Lu zLj+BI!%mBQCYQR!xHAI|bG;L7D$VEzEN(C-H*eVL5!|YZUkS;-w76oL&n#8U7a>0r zJH4}+DR=41X?V+-br$C!L||&AVCrEs~#bP#kvbY?g1MHHW>C-H~*6dh`E6>Ljfd6;i+&lVK)!`3_rWtKND zlCQMLbo3)VR2f9m|HS_OjAcD{-xTN`g9}LN;C+`Y-j!EF!3ej}pu!?ypkyB3^jJQL zn{>sNYhv_y|1c-9vDAm)!~?3fFy-Il9d1C~=NBIH9i`P7=#lG;MpzFKkKQcJG~Pz`oO-^eODSPo44I4D;Q z#pl_){Y^c};XQ9HS!>u_)3A8pM&L*77tRq^nm3PKlAdGlUp9hppFXd1=(>80b1Oxi}dP(bMQF zrWrVpDLv;S)3Y-~mtJ>yRPPT>!kpF7U{|R;aqd zcGC5B;m3ChJX?KVjn_xi6nn7N{5iMGnLp+1zOgipgCA3H6fxqB{mFu(h>%i@SBdWA zn$LKCm%s_@{5jxQZg2le#<%$;s|8V#dq31)c?HZ~rGA?Y4b?k&{q}(z`+VZ8oPJyJ zR0q_Im5-z0LC4o%2Fj~r*woF**`?RbnUA0FzR2>etuTQ&IZwXGXq&=bKD*;=b2_2; z;!?=cb+M`5O~%MUTWj0R^(*>LhsCz23H`pquDDP+?~}=AD8c=+h0!&?E2D^frg-xR zhYvQ|5i+08cBMZ%Hs8KUqJNYk8*zq8cwd)VEh1&f8N;4xV!=uGRBpbKNv`+M^fQKB z-lv=oh?J-`I6Y~K>_e%~MX#UqCCTk)47Ol+8+LU~t4^ z;C23(4t!{aUgx!r4>4B!k}#oboB+IjJNyj0mp983>NxcXoq*Lr^qh6Ah?e>M#ZUA z$2d#*Ukt`8#q?*H>bOdLp|$@+yDtI;#w>d3$@^5v^zM<&QXk*I2JarJ7E|IR@#SC5 zn(N`r+Q1`K-&77hRI80Djwx_5`SQ5P=7~c3%c8}NBs_Z6Ibe9$V1)bsbTInzSeGc7 z^O)aqor|SVok3&$6W-Z|hTo^s=|^hvB~EX#!E0JnO6%Rs3?_GT18!E6|7UKGi__x8 z{?3Qt;~U!Gzw-fZp~tq@Y0>b^F;`j_CY*T=bn*De0AWUY76jvI{heRc;8(VoOSKW$ zpzpJf_kpQeyH4Wvl{mfoavW{lXU%S^v_8aaem7ZQg(G*@|E@4ib3N#v6;=&m{j)-| zFSvhK_ySnrPg(4LuKsLs68i$?DpUEmyeX;LFxs^}CibYfSxZW_;g4Pe4KUa50CgH1 zGs6B(Nod0O3ofU|xrY5|CS8$^#(E@hxdvElh+64Ci^VP0`Dd|Y9~+2+o7MHa_f+pz zxG@3r5G+=a{Bk8Y*)d%Bmu)*Q_pdiA{^3z+iBm3@Piv3SX$@_W&IYnp^~7VZ4uaV? zpSX*h`qkt%;O^Z(m&Y@1C%kN_46oEDc>;nzH$%q#Qb^5R6+v?N?}&T!6AAB=CP%1wdcT4ZGLHI=a#i^m zU9p9aS~Aeu;t-fTAYGg?gRVz~&;NHH(LqMkDh~vqy)^&xnx>0|sjVsd-{W7mxT`q~ zq2wiLBfAvAb9TRey%tNi{A1MS^U_O`^u(t%wYy3mxi}M=DIa6dy*X88{Fa;a)?O%a zToe<(cQ)Zdzqk=sY%ePYxj(}#8!xUAPQ3z>dSk8B4X^vz;kNZ4dUsXx3~UU&BRyqpYT$yNSNPMdJ97i6U9?aTR`b{GGPw7+*@v%=3; z@o({?7+2JbD(JE+*19U+hKFn_?^8XWKQhdCCGc$3dmPC~OzdZIL5JEPs{;`AkMt7j z*qY*YS+|@GvIagH2t2CAaikeMu1|MF03yv84rxMgBY!a7}H>1O%nJ2P1%`r2JH zGJAT8btl$}5+9uEL)k)VtlS&sNliL8V;(Q#1&g&LFpVn3;TP{JtH^4+;9b{>+UFET z3&2o*`C)aS2Tx(_L1jIJ@y(C!1-gj`#P*@Y^ak~+O!(4j=ViiyH+i&?wH%z^zUUf6 zXizGjc^GvM$ktx7WKHqXuxAK)-~8D+4r>$gKiRzK(ywf&VV1GGPp6`?KNYKgi$akt z>$lzf**@gmnq)Za>~6chfG4%x9s-4z)kRYFR^OwetK*DA*o zTh>)zJtgF0LTd=jgTpVpaJ?69!KSpU{6^x{YpPa=$uCYP{|+hoQudb;UXe`4m&UB0 zWPbC9ol=Ok;oz+iTs_n6=1Guo+8l3ykaNWgxy?o^3OdirCiHjv(GxtEjCT+IK;{!W zCmgTdda=(gq=0U47uPN4G8rY24ARlrAt`)N7PQ!4A4yzima!ZoRS`}m?94o zZOhH=9fLeTj1iAkzFkmAEJ^F~t*5DarN(~sq@<_;Pq2I;WE<^>#xUJy{BiVVpd<9eF6vR4~{l76q+m6xt=HP!?Ajr z-n#qVYeq!f=O|cEYGi1zLLYf?p7J`Gbup0^yyi$R!oWEnJBFtvZo1T-7AbqG>cK^y zW&AD7m4!5O?E97cu}5t5guHn}Nn=VOE>^;Y2a`^;c#?G9rk2pb^7qLdzfDzl%_MBb zxe0PZSu;u{S8^qNg101k=AZQDzG@&`G!Tm0N!eQO9M^F;Y)MHy*3W)OwycMV&5MPa zJx$y&Ce5$@+8kAiGxGJF*JrN6ujq|8XlE-G`4j-FPomPIX+-@vp{E>+PS;~YU` zKUoEap;wx{jbFniITad=8YrE(wxbCjSxdfPs1Ky4R~hRTu)CTi)Zp2fVEsJ^p zk+9ZUiYN?J?Zj!D@@=Qe({m`WF>R19ZDYTRBFxE zKtMd@Dc$^Y(iei+!2!-3;*`^DmhXzJs#030=3;$-r!dKLZtMDvPw23eL8L!@NXu1} zbw@h={Bm%ydCKg*d!V3~gFj!s@}}Zj`y^V%%W=y7L>>dk;%Ck12P@5Q7{IrFOg8tI zq-Vj26ozd?s=>RB;72_`exYn*Ve(q zA&(#sS^XihnxB`etvYi)jt7ru(ywX;vovkptU}Ccgc$F&$mfjhPV3s9k#YwHefkKO@8Te8radHt9{lL8 zfCY?aeY;0Ac0X6hAxcY}_O*PW?mhp}7ks3yXPZ=``yN7bd~KJzJgZ&%J_}8OQKPg> zCE91j(=GVFjQpITAAX9jeH_knccW=5NmB3G`mpay)a~j=GxOYr{mT)&W{<*u+5S82 zubLHCxvcelxr@CK9+tLR9v8~o97Z+W34SIhY8k7jxNvB>=D=IvI{F(~WxucXuWWMO z^kFqdr_-vNXgP1{_v>x-kJR3jf~v?~r6$gVmBhA;rrRQ+SKwGM zunaX^o2L&RkMhV+=yg>iq>kZj^5eI08yw@rj-|B={>|esXe@6$6fv2GJ#0y%|Kp5a zhniSkyc7B(#+{u}1@Xre@1>uoh$ZeuoQxF&a)92o zirWA`)X`h>T3+5yd|<$qsCU3J<~h#=UT3}auUGwMDa`NIcN?EkXF4LQ{6!M~pZ3ne zE6Q$b;B=RO2q+;rfJh@smvna{-Q|EZ5=wVSOCyN1fG{K7EhW+&(k0C|@?PH2dA;}k z0pESrnza~U|JHN%InRFf*=L`9(4spKDNSuCld0lfqjbpL!BVU>2uQ|nDs_l4a%yY1 z4ft@GjvO*g-TLR~9yq{=IttR86Ohl(+84&o&w=)~nzlA9mOxY6Yp2R4f)@u8W87lg$Cq&Z$yGjdM~Y!z?_sXcf9aT zZ|E{0&`ep*1I{s)o{1Uxjh5^1D4VY=_gTw*Mn0FFkCq=BHTAv_Z}_Vku<_UNjtoE6 zt>`tCx_?edZZAAWPvLjxO1B+T=$O}T8ghI89G2YG2EjFuzjQ)AKi8ptZ@_h_s_Rx@ zg*S#Yu@Z{^K)e)nJxk2d&cB`gl>zw&hMnkX;j3kS%PchiD?9Z!9=MYgyr5K2XWNfgliJDL;shqaGcG!7lr z)U~-^u{lj$#FyJuCbU1T&eOjlN@Eb9EKlB(t8`FUD^p`UkE zKCJBagbNNzS~Ux>(q+=&94VdR*`K3C8IDb2gOD{0e2zbL7yHj=a6{+2{R9vG)#rvw zT)(Y~tbE(?cXg7fM-RW6>DYimFRC&CjHBX2dco579&Gv4-!G^>k6N6(&!daUSyn`X z6S#lei*+`af5EMsE*0~dQ@?OIK9iBAZ5VfuJ4aFr#6+jj$WhtU=LNQ=W*0I3iQhv&lgRO zAAI}#iCa*?zR0>cx!Tf)CD-|z!Bh+({ai$FZxOy9u~HO%?48zr4#c5s zU<`eb_or+TtHv@ev_}^eFp?~>%rdMLHzwZ1j}mxxUkZ8Ox-Dj)n^Fjcd)oh_;9pdT!~91QRh&ED`(@1`V=w_kF3^#HX`;wl9q48Opr; z@ZJIEJ-jm*9j&BdgH?1csp8T168yj*kS_SE2B+I%EytnYR=s2$6JeFYS$$m{uwGdp z%AyGkArwcR@RJkq%k7d|PCvfyPT9%9Gwih`ZRqjr z{Pw~F_~)#{eF8Z6K2I6yf*KKp(WgC`((L01tMSrCSA$!!-_^Tc(Iigcqg;KVi40x? zsQb{fN#Wrpe|P8xyVX1uk_dVUg&?(ke;m#=R~!aPW#o6y0}c;H8~&o8QkhTc|Ubmaq3 zI-X#BpW44NHIT;<6l&PaZm{7{4dcBZ`g~s0%+UZJ}zqXJ@gU)(qJ- zL1lmN%v*bhT53s>SR$15sMEWk77nkWHu?^{EW_?Aio7~Bu(YOITr1sy7lKd~4JQ0Y zI<}C$0eIDe`nEx9d97sjRX(ai-8WH$ltjyo%Dajf_ zB=mT;49Un5*T=fNR#Cya+}wWALBdxxm$PAC6ze-udnl)?-}af3IjXF>Rb^noE?hqR zwA;(m3qS$(8zwSg8RMV=XA*2hr8mc97VmU$RJ)Q6RQv9edmT6hZHdI66A88+(0-;= zKYaI`5xW=1Dkzgbe63385y*I~MI1Z|sPSA*OMjSNJ0ap$|1=WhCmM>POJKmT^depw zkudk;(IA;gOg){OUjA;A{6c(Mn+)#<{fP^iA|*5+uDFP1x3FdHLTX+g5Wfaz^Z_Pn zvT==OiZS097G^CQ0R^=^Mi9ISazKq2W!Mn$^D$tly>)DXOy$cS>~O(BT_39?;aQY) zrj!Uk+udY`mTLsQ@G{xH|U z7R1rW2qzJuCMr|Knm2ShMxrd?=#dRlr@%=^7kWpm%~49d$z-Eg&yf6Iy(UAjIDBb}d{(v_a8{zE@k8cPCIBZPTgEoC%S37SW3MP~(k=yd5R( zn|T3*%Kin3Tf6(-^$(gzk&VggZ%?}kvJWATltqbeJ~OUZk6sE$Wb7#aP}ZeAK%ufl z)!P!}Jly%U^pZ=UVnnszhNJnvNZ6af~*iaXb_5G)fj3Ijbb; zgJKK;vL$g`AYYfdAOZ&-1lf#+NhLUX)=g!~F~)HmJ2c-wxN0YK!k76>NON+X8|&>9 zIKg!9&yOB1stkYKJSk9Zv6pI)V-;`(e>y%8f1ib>HIcdrcpk^L0*~|X?h*xfd7gbm zvZ7`cXJqZx+9#tAsOs+BO3x!S-Ss|nIS=(@&CJ@7*BD>wp=PkGk6|8JK1N|a)vO+g zsx~Ff@=*iLXxCr_7ukNd;jPPjzy#kUI= zwpM+psP7pUKClj+jFTTcq;qqXBN?Aw+Ih<|o>HmEK=aBN2<|ER7FZc~v?bFqz*!Yj zlpVpHlC@`>^8GOmDv(1tp{mfm+_J5pYp?A!W5IYgLc*dFaZd>iC3?NYV=B+3R8NaE z?k6~2O=iNFDqDkD0UjBw&vpIo*z1b4{$+rc+@3uI6T?n45 zY$s@TjHG|PPcHu;-`JQeO=YGL8BCz*EEGWsPb3ZhaNHNt{0rG4FCqo7Z|v@?oUnzy zH~yb1;zatKzGP9H8Zc8_K@wH|>F+Mesi#-}XMgwF4a5qU!-RP4V%vUK_c!?SzG(|u z5L)Z(6wh+y=;&Cu0wLK>KkK%h`rnD3CL2=IDi9D^5 zoR!`+RH{OTX;*A7w0^Mi^wDaV!43gKQv|)aNYRj};}|v(Qx!`f;V`3=lc|By_Ld!z z^D!jD0NPlTe-)nv3O;skf;1KYnRk-Ip}|9)`@afu{yj&zuEt53g#-^NwhH;*|KC?Y ze`5#&8y3oJ-7Ih1WLlpqUi6x)77-}8X7(eHouk!KB^hvfM5Ms6&4;-9n0eXzSo@IFh7n>9EmZFGYU zD}hOxxZ5P-UQrx-piU}r_kdig-)|9rK@y^R9LgAGbnPK4y^hg}R z=I{tX;a0=O$j+W_A8a@|I4nbKH8Qq053!i_#R_x*TX{QFDJQt@1*<7h-AxqxqHpa& zfYN=NHq4Y=5l3A*Ng}PN?%S=ys$xNBZ>|h44z;nn9eVZf&h9xe=k;>Uvt;C*%<$Rg zw=$=iMCS09w`GGvZT5qV8Zcj*TkFJo2v>)DpOsudG6A5*;HDDwOpZB#6!LJJKuXAx zLK+GQK>cL}eoaTZ9s-~^IqU11)czN!3ujq0Q?=~t2ka8#SYF)iA+$x z+_Ai*&EAWw6cw4k*1Ml}*ktcmBwt!%r(R%>|4iKar$!G*5F~{l8kK<*=KW{zhvPcE zE2hp4qDpZe+JT2M-d)@_sN$n2fB`JM!we(%DicZR*#45(yvR%;Uf@E5hpS}ig#NPS zK866Xxknrwp>2b7ei;AJDj`?#pv!84%dQoa%rZXJ7-%>WdPFSrXErDyK8z5 zPw2k%wpiNIM1_3`>B0LnA05krQEU9L(bKPQ<+)QI!Aj(qrnLT1Wd9>>Zf-FjKn@YqvA#wH!jx`X615~X`cKCFf;yG9>F79xumUgU&Bxkpw_`$s&H3oDsBn$@ zD!6v+QlwwB8!(N%DmGPf$>kgEl^Shrv+F7>*eJYs*gMcMp-F z*>`1Dxp-f0mo`OI8Mx^US2!#T(IW`jhb&3Z_4pu_M+D<1m@+5gSHU#sPR_WhUP z0JZPWuI)SUr+x1RK>GLjqVB_I;w~BmC- zd8-g!*x0Cy$~H3p!Rc20+;zGSY%zTT{m-Td5Nk98u|^O-0PZj2e%*49Y>>1s zh%?&*=AB@q`eMT(zA9&O0ne6nVfZL1DyR11$5UZoW)-=6YRQXqx1>X}@@&^^JP#NA z1aWk~RI{Nv^(L8%ni|wwu})9JHzb>UscZ_rzdz&%XI#qsdMFT5G@ljlkoM{&v9pAF zs~`U-T}vL#cqd}*18LksTHW|1vez@DioKkzeUfOr&k?f(jzK3N?niPZmfi05ow?LZ zQF&s0QCi1Q<09+(Tv_UwxipQX-d$@!U1wCrh~)Iz7jIX$tp}BrpVGccMoe2i+2BF0 z1m%c7C-9j}dg{5t3S!62*R*5l%=*Dr#&6uG(fSwcJ+?DY&3XnqT?KAY7K0)VCxE{)iSXgy6qeQ+$lJwx( z{aChcu&0-KvHijY!RVcJ`xkIaaN3!-;OBkXiqPbU^bicsyOq1$Px&Sd z^byMlX?V$hNRxU7`kp_IQVM{O@nQj!g=Cy{=AMs~*w0)Yi2Ur5<^| zX#IkU7tn0B!`eU9(nRKTEnsPiE z_MPMca71e6pw4U02b`;s?$y>gvri{}u=`jF#MKTXo07;!=u=?v01lxb3$*8io^#jKD_2sVzFkFG9Ac6TRAPmuI?F#T%vFE`%}Z z={c@mF{qpv#vUK=W$9l=D}J@kC5$CB#KhQBi?oEso$%(jppn(f z5F4OZSp`eo{;ZqjIkI53dhKVa9UT%PQxXy4QaM#5LJM-SA&XlL_%D}=-euYBM+`bJ zm=QBPyjXvw-q&9vR2`llVdqS&|AB)cwSfZ;V2Yf^yr*?r3C+CKiLF1+?df~BK6XPT zSv!8VPudql_U?xscyj%314u4}t72Xlwey}e?^mBJ9`zjjJ)ML{V7^|igsfWr;WQJ- zFMqlI*7d*6Hc^!MUHbRqK7J8|BZKtV-_kdSf53{v9xwp~tV22*;?DlJ<0fE%u!kx@ zff5iO`B&htM=bnB`S<1Tp_B#4mM;hewirI_ZNP4>fdbmtuL01lHn5_wn+u?#@Z8s; zzxwvD!mugRP+^P5e+&Po1Zr3yY!)&UX!Zma2+c``rNHK?K`9|Z*A&>SHCO=bN;?#Q zE_n_3$GST#0d`dvN|=#?T_U|+9fbwJuBt)-8Pc!-=o%|51$MO(N{NtxrCd+oVF9qg zE+`;M4Hj^H8^2L*{L9~?NKghGoR|8qlt0CkZtm!Y^?NMhSC+r#pIOkz#0~4l)Wc3_ zp|lX~>wf+>(_2^$>@*I_nbi69o_5Yo7*_) zCR^$nw#iTWkW0+|k&UQ}L(8!BO;z@`!RG}UU6E#V&2AsvT$-uFL(Ua$>o~DZ zbQm;Kg}ljq#%2HtW+4?g#jF;??j3^I-(>nv+Q=5~C|5CwI+h6y>QW;j4dn1B%;{IM zyHiEwdAMkOJaT(_Yq(qF5A})MEOJKSHlW3>pRg{`ck9ktM9_L$$X~e!9Io{2^12?3 zY-;)*oyP8ZvOdBB08dX)0EK@G%SsjIm*-$#lL1E^5;!b%Yz#rROpMRp{|(3g!!h_b z)r+EJ<=dH&{f{N?1A49|7bDR`q#cDN>dBQoyd)Q1REB+fMY!1fng~sqz!&PBXQRh` z-~1w9*j^9$cx!aYepu4GE`gX7Q93O4FRkWE-}^e8yS+L-BCTADrQts)h3%WvFhj~-(Wa2<}pgPH&itxioq}NU1tlqrshN-n|IYe=&4DNUXJ zLXz}4;!DtJJsJpA--;+oAMv(B8TG2@^Odmb5=p-+O#4pUD6@pwgcN-19a&=iw$ZL} zF1FduplTh#hhl-Rs)hxlmHy|fbO~9TAJbZ+zVFtfuoSI%b6A<)q#EvT&1w&+iO>PX z{KXlCdocO%DEh@EKKL+aiYtzyOJXibj;s!S*O^=hOqn3UDq+}ZhQh=`L^_9QYLf_# zT)0vboth&{tWEvwsN-Vw#HdHPd!*QOM1Tt8a3!9wQHty#l9Z)@nzV8nJ9&BY?$|pS z{`G^FWJc?mvkL@I4V8Q(dXH-@-Ha`K?bAi)&(eMF@{YiAEd1~kQE{S;1oWBM_1C~H z$}bNn4QSfxR;)cPafW_8)5H?Ht)rB)GOzLcpy+gtJ%#HdGPX$lcMp<&DqoZnicdcj zl1wPhFO-#R?AKa+f_^x;d9dLpSt?VUO2?eH$*`z$^?ZM+0HqJ&F~zj|9zS>yN7m|* z)gv0djWVo!D41n-jpOrMX%h8R;f{^w3F5(fqLx8C~c0}d$Na4t>yn&N9&s#`;4 z>>*k)q?0%8%cD=??g8UE!FnGs$>V&mB2L~_q^rodD2z>!uiV9g4sKb>yfeS9FIZW5 zg>w7n)d4N)*^;RelA5Dbj?_pG1e$o7WP^y(SykSL?<<7eSU5I+Fz0e`c2ToirBJb6 zZhY|cpI&i)8^Jzg8O|;vpu@-%QPJ2Ft7CblANH-EJ1o390w-iA&G7Qd&hI7-f}{I$ zp>=dVPw&P7FF5-C8K2$)(Q#+sCw1T?MgV|^0LSMadF<~|`mdY@0bXK)qx}E&Rir2@ z-NB63i1Zl1?3m<$jXrC~L~)?9{{p_Zf@YqIoXz8WnUJbj%B5Ie0b*1+RBlOyctn1{338Z@=&m2{q9cvsNIghdvdHd+VB- z5?U+Bi=@7F$tEue>aUXb2@2}YKy8WzlQ0!AtT}_rqW3>5H`*!{kHqFPG zbo2}#L6Q%}88EstK~8twDs~C~v*F%I)_KUR_1i7-CQ+d@NdX7Ho4*}dK+ii;Pnsy( z&CG;)mE%=Ak-I*c=J}uW{pZI<8@=6=%V10Y(`9x)#b4%vr*p$M0017C;?F7F*2K`z z&X(!djpcc6Pae~W$Ye$XoLos?^0K=JK&pySSWIzjT^`yzEt%nGeX07WsW!OTbccdx zkEt_26&cNXZ{$C8f0L*A6LpBrOe(Ppf$~#UarA8L`Sh@Tw9=X{bBsx`cT{T2_;%-b z+)Zm;iX{M2xb-@UliUtQUz;=CFTxxSc6nk7h<-9^o^VVQXnHMCO)njVFfM}cf`fP! zsJpKrss5pM`coUe^+>SS#htkQV}YP(HO0rAmnaK{CLErc=dC0TZE zm1TsSx34&B^&XCGI<-|OACkh$r}+|on#2s}pv#xPkB+b~{C+aDq~e!Tx|YlJqz% zE9)eXD(8}*wHvW7op{0Azf6>$j&sN9Rvqv?y-K#ljc;p4U+|Tfy@q!jcS=0l5z-?# z2R}XiGHZ0wqQ!zL@@JjS6R3&IM{E!>drt^niMMvP7%1y-UTpze6m=MNrB<5tGgY*m zz)vD;ren{hMt3*Q8z<+c`}_U0fHWP_!_(D=<2cgeyR@eBh!;$o*n z>;Bl%!6RE8|1G_!`ZSgGt8Ww--1lBDeZSmeMdFQQWilZl<4{txCJMw7vd7pXPFAIbQLvNV(R13SA6S96IY0ldy^|u72z!MEI&>UVSLoW9JF%`Rny6NMOK4 z(=*_UE+tAHXOzz}UDoLeb5RyV3<2shS7bV%Q015CTw(GsxPU)sOj)SGG$#AtC!6?E zgjq6)%DR@8y8!+{iGt4b$S&WP?O;j52xznLKjkV*LKTF8`tu`=da@*8j%%`KPMP2O zvA?B~rF9A*x8r-6AK)uO@FLt;{h2U}tW)1U0Le+o7!uE-z`*2Rq=$HvMP zEM1Ut$RC7QWd59Lr1$OzA~0tlD!NTKFiY2eY}JxV)&=FoTkg&T35e8Z zehXuN4o|`gR!!>oGwp*gsr{yl0Dr9JhZk(y&oOr(+SKjjDJ~tD)m_voj1~HiLAh_ zEq%;`)I6{PJYS7Uz*aT}v+}o;i}?%dN}#|{Ou&7n{VZz(JNSu)28RiltT!v+*%>NK zN5CRRNtP>vhlLG@iTiAerLk9L!Nci)R2Ut<@$qY zFxPyBe-s&ej6Q)~8ti_tHD1r!L+aE#_mhCNq5`&mHyhX)Coyoz+Mip$TyOzQlqgyL zrz;mFu$zIw|6hMnK2ydP;4&ilv71p1&LNMja9KefKXXpow<-pY2hNJ@-@A$ZpKd@2 z?3&NuH)u{k&OcNx{?iowS+igjH~kTTRWgbH>z~qEubSAwBm+^;B;Xb}KxSEiRYsr} zL%{e9dqmTc(I3onqx^U^tEq_QMg29iiKqOLyJ3}b5JMcd1Gd9;s5Li}B04?0)%J*X zC8NC+`0HndenW}k`&cI*zkLO=862ufCq%L7?PFd4Y7cFzN#|tIYvuxW2vTH{z6dnW z8Ovb9e}Hq%Ey3S^%YXRUs^q(pkO4^wtGW1{!gdZ~Xc5$s-Jlcc-h?vYt1jKu5(tbv zB%W|C>qKrS;Bb9g>-SjgS7k};X;6>%^5j;gGXxYFO$tgu5{4x;0RZR}&u6W_7B_Y#hUSJ$zrKI1YYsF9BM7Upo8fw%ga)*x26kL^c}8-% zE{|MRW3)CTB%}SchpdDY>*=r{5Vt3!y&-|4(%wrh)WNVOUa++r5iFlx%0311*@!2u z1_w7j3PtHTGOkvV74icoPB|^gE&HL8`lp^VFm)HL}qYP;68~6{dQUcn9}9X_HVX^3zzCOJK<{4lw1v@ZS>FL`ZpoZ&PE1kV#g1& z9;rdozQrC;pG<FTG#^Dv7ohkOg>oMS<;8UfM=96({2$xLIw&*3_;7POuGU!8ZkCDxt1H8 z9xp<+pPWxSahXz|kcdYOvshJP7s%ye4QojsAG@C(E*y87pC0F5=sfm@#TnigdOki* zRxdw9yQrJ`(P%&36sqZXUi@V8ynZCrc~fx(of^A}Nq`w7*F)(tpzkE4kC-Hw5S%0_ zLm|U}K;&%^u=gdHex;Xy4Yx!dgeia>Y51|_uFx1~x?G|N%YNXVIhbLEWxHtDZyfOA z;v*%JylU?lf313qbp%V3)gXUM(B{f0t>3wn7A+k550g~ZWa@@j2=67Hx`q*~`4)|% zH5~YJ*jg!dsL%1m++|Xz5=Qc|CHzLn!&;flaD}ZZ4U^V$^M(`l^8+^8#M{bR282CC z;tjj*oU*MB6nR=%=m_uO3wy+Vf*6bp7bIv;&ZC&a#ce6FrMJF#Z8PvZko$LR+1;^d zc5^CHyBv)k@JFY6UL&!2U`}ir%qV z0G-bpS}jKzI`GkulBlUczNV368sr^B*MBYwGMr9LAKKaZ9;<3Xir#iYnTqL?>bCc3 zWINyR%;($}zO0Yj8fFR#)LqadANZM7a+;%_=4YT3oxyNV)bXlqLsP8N6nD^%ncN%b z1`EF<}JLDhJWNM=3$h_gbvgF_>FO6COV`Rm@h$ zS|TOR`)pOETQ~Z+NBZRC>83Z(Xz=HpVjdsf(NpUin-2DTVJ{{{_|9?0f}EHd4V1CC z^8!N0-8z4QM5@M`=^<;Kw9E@0qDU6;q-NF{$R1E(qLixrT=fN?^Fi50wi+l~B%^Q+Uww?P94`qUyfCS8O9f(3k;dP@=D>8QV1h zMxE`Yhq@gR{hY=_mkMtVz*O^liz}B;^+m~ao^fdUOE`|(k9Rb-7hUypJjavcccHy~ z=z!u*EqO=w>BflMG)AOKu9FI}Hf~BJx}?viF*GLz(3b7acCmLyg4kAE7xuYU`C{@( zdgTO&1jUyDCzWy((>74jeF;94I~Y?=@F7EQ*^(U^Va~%=S>?!`vOKQZKnMX$Y?Ni# zPn`Lb*or7WLRmwSioV{w&IyL^Fj=@bKeKH2rlblo*mzTF=b67D(zxOQ+-TXaitE)A zHYzx{8Vjtlb3mdIkj5F(3A>*6t$d~9T|b<8Lz!3>;}lBXK~OB@>L^D^L<*fyxQ#W` zUX@T3*lfnDl=@KowuFLIKUSQ3S+Q@H&o>FhP`Z1;++3lqKMfzb&a7=$@woK3;jqc$ z`&2C5Ww+c9paxLPGU*6(yXtU*QK*Moscu+%=Q4B%iXw(CuuXEhYHSuQjvvU(!9U*8 zO{qA0pET!M1TR%DZyT{#mGpTW!?OOuci|#$lD8wLl-=7Bb*SHX-t87SsvAuXU5%di z&!#T=_Ac9xX-3zHvl_XDguFnWF_mC@B$Fbw+jM z!{MxtOL@OoIZQqFP~VK%MFB%Y%nVVvg`ce1sFicLO%s26s z7k8`MLh@qPBRx%ScB-0o_9s_{~6Z8cl6SM87A{l`Qve@Q)-4?gr#1Haor`6JQW+Bt&^ zZJ&=WvQ*5?=9$qR1!g?aZjbsL*69(dq{k-|cKXtZ4+{1AgYvXPv$BS|9$n=fw29tS zjd}#h=Z@PBFTT_oD^x|SFzm7^W+l@4jK`v~7hI^~-zu6#;j+ePfZR)gv)ybv5ud#3Bb(%*pGCY+iUy%1G5uihOc%4p2Tut^jDXd6i9YNj zbfdC+QZ&iIvm68|2Ex7TD5N6?VjS_ZPCFVeQ^GKlpg9we-bCWtsG&Qmh>h$~ z)#)QhtDq7y?XqgSTmvZ+XA#JaD{?Ueg!S?{%`>?ybQVy5qZ_xpS#h#h_7XLm9_0IF zpV{&;rh*gArP+6z`42%=%3iPAUdc-{R0yMH`3~8Wh*9ppEFsKga7pjl!6nsH>g_C9 z4r%=M)s2*%2Yw}`B&(D zr#opa5x2HtwXJOUE=TRyKIOJ zpZI!~LzWDaKr(QB!ecJHobVZ;Te-j6PqZC?OvzK1i5VOaZwG4#51CyKI_J%P!`z>ZCa zJ{X7>z|{<=v;xJJPN?CG_}$D1Mh3B*$w6mry*F;PH8Xd4@qIe~hCo>Nsy&9Eur
HMz4G_i2SlN&4rH_mtQMCSK5{vZ3 z*f@`fYzBMoKNqjsx&@s{FaQ8D4gi4lr%#*c+87!r+1UUsjDNXysJg7}nmAfxF6~oB zNO?JP6q;h+B@Q0)<|hW&h9X-`6CC9!IDAQm`{T1$4JtGm1l9((vhwo7`_l|1&5wld z&Z$&za(ZRox$W9ZP}Q@;e|?bMc6NUX7|)=;`q~mPCc0!NU@&##a(8Dr{UH|fl4@3V zJ94{QuznHb82Ka3T{42zd(*f;Pq16Q^4RDd79!Mi-Q2k00-cg4KCvL zGG66ZtAc(Fyv6X{2MgN9tB3X;W(adVCUlP9EBdhAK02Pw?^^)xq0cn|K3l-D+SnOUrIi&%oQCL`> zh{|Ay=Iw|-`0$zE$8h)1AD|*M)0ozO>pFkFa;&W{*h}zJ$K) zg6C9FNinnTiuIX<#|W1J0H!T?hm8ki7L#Efh9R?JhSbBK6;XXRD0K-m;$Gkb%OIQ& zEV{f5o?;bPlO)YB+2#F8p+VeEJ&!{@d7M%7BkgjOb;FRpC7)u}Z&VWIx-*@|BZ?|= z%FEx9l}26HD^`po;JxTeY*<&LfQDTNM#l8?N7*_mg%2)Xv#>kW5ItXybLW_Dy2ehF zd*~y^%P!?Qdufq^=#`c401C%CR0P<)B<_bPujfn=goS>L24N(MvMe8(gi?{fzR^;} z+m`#5f54+*q&{eYZuU)Qm3^KJCef+SUn#*E$dw3c&UQimio?1@d;Wv!5nq)zYZ;Mc z{>7fYkyC(1!TD=KZaG$Ujzwn;CBtT_AXZF$71ed*hm=t)>jJcojD8r$g?q!b^%T@| zS{tMg*VGnX)-+qE z1=J^>1}`%ol8^(HEuNfAagj?Vh>X#XD^kde!n%p$cQ!ypxs6S?Z$X;TG)wE!8z?Gc zZjA-XbGi@x4H-IBDSPSJO}?j_1}*HFeka*&d{Tay{U$g^FHn>Qy}E!=>BVvQ1u^v1 z*bw2>JT!u-8biWdroP-22rRi%p8E*6vK#2%xSC=}++j*MAz2EwAlVrbB?D_rqgREq z&-a=Ay?$CSRZNZNei+x*tezG+mgu)ULsU*fJ%dDMUD|{D=?Kvya>Qj~93upBd&~!t zZ@=*hc(HFA%rr*dEr<$QOcEh3TJ7iYV1N5|&$ayQVpBQtp<>Nyp;(Y)vOR4$?zLcx z?diQ;lYlni#t`Rre=6MZ(GFYyuR+NaJ>H9#;b*0MIy&y{jsf+l8b{I6I&0eSec^b?npe+0d!#^A*pT;-X!f zi2_4}sYpenmiYB>QMsB%BXS;>g_pcK30+Nh1K*?H)ARwk^s$MdLWFC=Q2(B0X@jx_ zERtNda6~etYgUYLo z`FdUAhPt=Bd>nk*`yS$HYYDC_`45vhNFVr06nscw2A=BC!EfdaEcF#^EUj#r^et@+ z|E&4??+ZC_mJ5y2l5S_l^E-xldfV%sR5dOuGQSE}Z$rlYz5%JXo?*r<(!k;FD(6tO z@Jysf)ieScqlNZ!UVrurkn6VS`U4c}hb#2z6zop{{{C zo?J%Lscg>c0`eo4L?615RK>*-S}5_nPZlErDt1R@Rn%>1B_W9VE}gK&IZ^Hhv^>uR zR&~n^vZ$S?4JX;)&=j#38#0H4!qO^z?@9HvZ_Lp!QVOe$)anZ=yEk3LZ?4;_ao`lBU_&WmC!-sAyOi>I^*pf4{dpxv30uxH zBl{gg-I5|N67no6AbS_-NvNHd>Ow?S8O!GAXOz7PEc$XV9Z@wpG3&Ayw&Ch@D=N82 z^hW1AaL!n8zu^&@{~o0GTduTT*pl@i3roc#tY^hbdjiZ zw5npJc?Mg}C!W%=js^TR&SOImf$=KmTpTK7Z(&(OtQ8qN$3)yN7IjXCiaaSw@i*e@ z6Kh8q)6}eavnWU-nB>RN=$M{@NU&7ow9G=7-N#FP0hjrINn1MfF=O;odGIx#@kYy-jdJ<1>=Ef-?h66Q0ABL`=Ze{Xnd@Ko|4=`xApP$Q{=K&EU%=;k z95^QaR_XUU@b`MBzn~4^jplFFPrrlzy%^&!C;;#g@fY}iDb4tu&+l*O|Kb#Z{69kc z<9+?_Tz)?a|BDL<`#&Cx|IXm|E%sjwREU2u_-n)cJM{OR%wJF%vOl1|?`wYN@bBsK vFFXKnNeKY_BbojV|Mw~I?{I7Czrp{U5EZ0hz}5u-5WycWaK2`ydv5(d`Kk9i diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 313bccf..4e83612 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -13,7 +13,7 @@ from tqdm import tqdm osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) -parent_folder = "/Osmosis ACD/Osmosis ACD Projects/Stonewater/Stonewater Property ID Folders/12. Decent Homes" +parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders" asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1") From 7f204e328c07d6564274954c185cea046ad8e659 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 26 Jun 2025 10:39:54 +0000 Subject: [PATCH 12/22] hallway finsihed --- .vscode/settings.json | 2 +- etl/models/conditionReport.py | 31 ++++++++ etl/surveyedData/surveryedData.py | 113 +++++++++++++++++++++++++----- migration_db.sh | 4 +- 4 files changed, 130 insertions(+), 20 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b323fda..27782c1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "python.REPL.sendToNativeREPL": true, "notebook.output.scrolling": true, "terminal.integrated.defaultProfile.linux": "bash", - "editor.rulers": [79], + "editor.rulers": [67], "terminal.integrated.profiles.linux": { "bash": { "path": "/bin/bash" diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index cd5aab5..cf38a25 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -119,3 +119,34 @@ class AccessAndElevations(BaseModel, table=True): external_elevation_gable_one: ExternalElevationGableOne = Relationship() external_elevation_gable_two: ExternalElevationGableTwo = Relationship() conservatory_or_out_building: ConservatoryOrOutbuilding = Relationship() + +class VentilationInfo(BaseModel, table=True): + is_there_a_ventilation_system_present_in_the_room: bool + any_damp_mould_or_excessive_condensation_within_the_room: bool + are_there_sufficient_undercuts_on_the_closed_door: str + is_there_any_open_flue_heating_appliances_within_the_room: bool + +class WindowsInfo(BaseModel, table=True): + does_the_room_have_any_windows: bool + condition_of_the_windows: Optional[str] = None + do_the_windows_have_trickle_vents: Optional[bool] = None + are_the_windows_openable: Optional[bool] = None + input_trickle_vent_product_code_or_measurement: Optional[str] = None + +class RoomInfo(BaseModel, table=True): + overall_condition_of_the_room: str + does_the_room_have_any_defects: str + are_there_any_sloped_ceiling_areas: Optional[bool] = None + + windows_info_id: uuid.UUID = Field(foreign_key="windowsinfo.id") + ventilation_info_id: uuid.UUID = Field(foreign_key="ventilationinfo.id") + + windows_info: WindowsInfo = Relationship() + ventilation_info: VentilationInfo = Relationship() + +class Hallway(BaseModel, table=True): + is_there_a_hallway: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + + \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 367e7c7..73a797c 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -27,6 +27,10 @@ from etl.models.conditionReport import ( ExternalElevationGableTwo, ConservatoryOrOutbuilding, AccessAndElevations, + Hallway, + VentilationInfo, + WindowsInfo, + RoomInfo ) from etl.models.topLevel import( @@ -60,7 +64,7 @@ class surveyedDataProcessor(): def load_condition_report(self, db_session): # My task to complete load # [x] general_information = self.get_section_1() - # [] access_and_elevations = self.get_section_2() + # [x] access_and_elevations = self.get_section_2() # [] rooms = self.get_section_3() # [] heating_system = self.get_section_4() # [] occupant_assessment = self.get_section_5() @@ -78,18 +82,96 @@ class surveyedDataProcessor(): # occupancy_assessment=occupant_assessment, # ) # - # Load Assessor Information: - # return AccessAndElevations( - # property_access=pa, - # external_elevation_front=front, - # external_elevation_back=back, - # external_elevation_gable_one=one, - # external_elevation_gable_two=two, - # conservatory_or_out_building=co, + # hallway = self.get_hallway() + # living_room = self.get_living_room() + # dining_room = self.get_dining_room() + # kitchen = self.get_kitchen() + # utility = self.get_utility() + # wc = self.get_wc() + # landing = self.get_landing() + # bedrooms = self.get_bedrooms() + # bathrooms = self.get_bathroom() + # loft_space = self.get_loft_space() + # room_in_roof = self.get_room_in_roof() + + # return Rooms( + # hallway=hallway, + # living_room=living_room, + # dining_room=dining_room, + # kitchen=kitchen, + # utility=utility, + # wash_chamber=wc, + # landing=landing, + # bedrooms=bedrooms, + # bathrooms=bathrooms, + # loft_space=loft_space, + # room_in_roof=room_in_roof, # ) general_information = self.load_general_information_from_condition_report(db_session) - _ = self.load_access_and_elevations_from_condition_report(db_session) - pprint(_) + access_and_elevations = self.load_access_and_elevations_from_condition_report(db_session) + rooms = self.load_rooms_from_condition_report(db_session) + + pprint(rooms) + + def load_rooms_from_condition_report(self, db_session): + self.load_hallway_from_condition_report(db_session) + + def load_hallway_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.hallway.is_there_a_hallway: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.hallway + ) + + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "hallway", + pydanticModel=Hallway, + db_session=db_session, + additional_fields={ + "is_there_a_hallway": self.condition_report.master_obj.rooms.hallway.is_there_a_hallway, + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_room_info_from_condition_report(self, db_session, obj): + room_info = obj.room_info + ventilation_info = room_info.ventilation_info + ventilation = self.get_attribute_and_load( + room_info, + "ventilation_info", + VentilationInfo, + db_session, + additional_fields={ + "any_damp_mould_or_excessive_condensation_within_the_room": ventilation_info.are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room + } + ) + + windows = self.get_attribute_and_load( + room_info, + "windows_info", + WindowsInfo, + db_session, + ) + + room_info = self.get_attribute_and_load( + obj, + "room_info", + RoomInfo, + db_session, + additional_fields={ + "windows_info_id": windows.id, + "ventilation_info_id": ventilation.id, + } + ) + + return room_info + + def load_ventilation_info_from_condition_report(self, db_session, obj): + self.get_attribute_and_load( + obj + ) def load_access_and_elevations_from_condition_report(self, db_session): pa = self.get_attribute_and_load( @@ -176,7 +258,6 @@ class surveyedDataProcessor(): } ) - def load_general_information_from_condition_report(self, db_session): assessors_data = self.condition_report.master_obj.general_information.assessor_details.model_dump() @@ -623,12 +704,13 @@ class surveyedDataProcessor(): update_if_exists: bool = False, additional_fields: dict = None ): - clean_data = data_dict + def remove_nested_dicts(data: dict): + return {k: v for k, v in data.items() if not isinstance(v, dict)} + clean_data = remove_nested_dicts(data_dict) # Merge additional fields if provided if additional_fields: clean_data.update(additional_fields) - print(clean_data) if lookup_field is not None: lookup_value = clean_data.get(lookup_field) @@ -664,9 +746,6 @@ class surveyedDataProcessor(): return new_record - - - def load_assessor_table(self, db_session): company = self.load_company_table(db_session) assessor_data = self.pre_site_note.assessor_information.model_dump() diff --git a/migration_db.sh b/migration_db.sh index 00eede9..d0e2adf 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,6 +1,6 @@ -poetry run alembic revision --autogenerate -m "access and elevations" +#poetry run alembic revision --autogenerate -m "add column in ventilation info" -#poetry run alembic upgrade head +poetry run alembic upgrade head # See which hash I'm at #poetry run alembic current From 95cc1821c71ec853bd82364dcf10e3db56cebed0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 26 Jun 2025 15:36:05 +0000 Subject: [PATCH 13/22] bathrooms --- etl/models/conditionReport.py | 48 ++++++- etl/surveyedData/surveryedData.py | 183 +++++++++++++++++++++++++- etl/transform/conditionReportTypes.py | 22 ++-- migration_db.sh | 2 +- 4 files changed, 241 insertions(+), 14 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index cf38a25..23ceda6 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -149,4 +149,50 @@ class Hallway(BaseModel, table=True): room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") room_info: Optional[RoomInfo] = Relationship() - \ No newline at end of file +class LivingRoom(BaseModel, table=True): + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class DiningRoom(BaseModel, table=True): + is_there_a_dining_room: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class Kitchen(BaseModel, table=True): + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + is_there_a_cooker_hood_present_in_the_room: bool + +class Utility(BaseModel, table=True): + is_there_a_utility_room: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class WC(BaseModel, table=True): + is_there_a_seperated_wc: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class Landing(BaseModel, table=True): + is_there_a_landing: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class LoftSpace(BaseModel, table=True): + is_the_main_loft_space_accessible: str + is_there_more_than_one_loft_space: bool + +class RoomInRoof(BaseModel, table=True): + is_there_a_room_in_roof: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class Bedroom(BaseModel, table=True): + double_or_single_bedroom: str + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() + +class Bathroom(BaseModel, table=True): + is_this_an_ensuite_bathroom: bool + room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") + room_info: Optional[RoomInfo] = Relationship() \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 73a797c..19a46b4 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -30,7 +30,17 @@ from etl.models.conditionReport import ( Hallway, VentilationInfo, WindowsInfo, - RoomInfo + RoomInfo, + LivingRoom, + DiningRoom, + Kitchen, + Utility, + WC, + Landing, + LoftSpace, + RoomInRoof, + Bedroom, + Bathroom ) from etl.models.topLevel import( @@ -115,6 +125,177 @@ class surveyedDataProcessor(): def load_rooms_from_condition_report(self, db_session): self.load_hallway_from_condition_report(db_session) + self.load_living_room_from_condition_report(db_session) + self.load_dining_room_from_condition_report(db_session) + self.load_kitchen_from_condition_report(db_session) + self.load_utility_room_from_condition_report(db_session) + self.load_wash_chamber_from_condition_report(db_session) + self.load_landing_from_condition_report(db_session) + self.load_loft_space_from_condition_report(db_session) + self.load_room_in_roof_from_condition_report(db_session) + self.load_bedrooms_from_condition_report(db_session) + self.load_bathrooms_from_condition_report(db_session) + + def load_bathrooms_from_condition_report(self, db_session): + bathroom_ids = [] + for i, bathrooms in enumerate(self.condition_report.master_obj.rooms.bathrooms): + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.bathrooms[i] + ) + bathroom = self.upsert_record( + db_session=db_session, + model_class=Bathroom, + data_dict=bathrooms.model_dump(), + additional_fields={ + "room_info_id": room_info.id, + } + ) + bathroom_ids.append(bathroom.id) + return bathroom_ids + + def load_bedrooms_from_condition_report(self, db_session): + bedroom_ids = [] + for i, bedrooms in enumerate(self.condition_report.master_obj.rooms.bedrooms): + room_info = self.load_room_info_from_condition_report( + db_session, + bedrooms + ) + bedroom = self.upsert_record( + db_session=db_session, + model_class=Bedroom, + data_dict=bedrooms.model_dump(), + additional_fields={ + "room_info_id": room_info.id, + } + ) + bedroom_ids.append(bedroom.id) + return bedroom_ids + + def load_room_in_roof_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.room_in_roof.is_there_a_room_in_roof: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.room_in_roof, + ) + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "room_in_roof", + RoomInRoof, + db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_landing_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.landing.is_there_a_landing: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.landing, + ) + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "landing", + Landing, + db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_loft_space_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "loft_space", + LoftSpace, + db_session, + ) + + def load_wash_chamber_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.wash_chamber.is_there_a_seperated_wc: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.wash_chamber, + ) + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "wash_chamber", + WC, + db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_utility_room_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.utility.is_there_a_utility_room: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.utility + ) + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "utility", + Utility, + db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_kitchen_from_condition_report(self, db_session): + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.kitchen, + ) + + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "kitchen", + pydanticModel=Kitchen, + db_session=db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_dining_room_from_condition_report(self, db_session): + room_info = None + if self.condition_report.master_obj.rooms.dining_room.is_there_a_dining_room: + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.dining_room + ) + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "dining_room", + pydanticModel=DiningRoom, + db_session=db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) + + def load_living_room_from_condition_report(self, db_session): + room_info = self.load_room_info_from_condition_report( + db_session, + self.condition_report.master_obj.rooms.living_room, + ) + + return self.get_attribute_and_load( + self.condition_report.master_obj.rooms, + "living_room", + pydanticModel=LivingRoom, + db_session=db_session, + additional_fields={ + "room_info_id": room_info.id if room_info else None, + } + ) def load_hallway_from_condition_report(self, db_session): room_info = None diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index 0b33da8..e064694 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -153,17 +153,17 @@ class RoomInRoof(BaseModel): room_info: Optional[RoomInfo] class Rooms(BaseModel): - hallway: Hallway - living_room: LivingRoom - dining_room: DiningRoom - kitchen: Kitchen - utility: Utility - wash_chamber: WC - landing: Landing - bedrooms: List[Bedroom] - bathrooms: List[Bathroom] - loft_space: LoftSpace - room_in_roof: RoomInRoof + hallway: Hallway #loaded + living_room: LivingRoom # loaded + dining_room: DiningRoom # loaded + kitchen: Kitchen # loaded + utility: Utility # loaded + wash_chamber: WC # loaded + landing: Landing # loadded + loft_space: LoftSpace #loaded + room_in_roof: RoomInRoof # loaded + bedrooms: List[Bedroom] # loaded + bathrooms: List[Bathroom] # loaded class GeneralConditionHeatingSystem(BaseModel): is_the_heating_system_in_working_order: bool diff --git a/migration_db.sh b/migration_db.sh index d0e2adf..7a2b5dd 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,4 +1,4 @@ -#poetry run alembic revision --autogenerate -m "add column in ventilation info" +#poetry run alembic revision --autogenerate -m "add utility" poetry run alembic upgrade head From 50600b5192c9fef0bccc992eee5030c9be36b79a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 26 Jun 2025 16:43:20 +0000 Subject: [PATCH 14/22] rooms have been completed --- etl/models/conditionReport.py | 40 ++++++++++++++++++++- etl/models/preSiteNoteTypes.py | 1 - etl/surveyedData/surveryedData.py | 60 +++++++++++++++++++++---------- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 23ceda6..fcd7067 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -191,8 +191,46 @@ class Bedroom(BaseModel, table=True): double_or_single_bedroom: str room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") room_info: Optional[RoomInfo] = Relationship() + rooms_id: uuid.UUID = Field(foreign_key="rooms.id") + rooms: Optional["Rooms"] = Relationship(back_populates="bedrooms") + class Bathroom(BaseModel, table=True): is_this_an_ensuite_bathroom: bool room_info_id: Optional[uuid.UUID] = Field(foreign_key="roominfo.id") - room_info: Optional[RoomInfo] = Relationship() \ No newline at end of file + room_info: Optional[RoomInfo] = Relationship() + + rooms_id: uuid.UUID = Field(foreign_key="rooms.id") + rooms: Optional["Rooms"] = Relationship(back_populates="bathrooms") + + +class Rooms(BaseModel, table=True): + hallway_id: uuid.UUID = Field(foreign_key="hallway.id") + hallway: Hallway = Relationship() + + living_room_id: uuid.UUID = Field(foreign_key="livingroom.id") + living_room: LivingRoom = Relationship() + + dining_room_id: uuid.UUID = Field(foreign_key="diningroom.id") + dining_room: DiningRoom = Relationship() + + kitchen_id: uuid.UUID = Field(foreign_key="kitchen.id") + kitchen: Kitchen = Relationship() + + utility_id: uuid.UUID = Field(foreign_key="utility.id") + utility: Utility = Relationship() + + wash_chamber_id: uuid.UUID = Field(foreign_key="wc.id") + wash_chamber: WC = Relationship() + + landing_id: uuid.UUID = Field(foreign_key="landing.id") + landing: Landing = Relationship() + + loft_space_id: uuid.UUID = Field(foreign_key="loftspace.id") + loft_space: LoftSpace = Relationship() + + room_in_roof_id: uuid.UUID = Field(foreign_key="roominroof.id") + room_in_roof: RoomInRoof = Relationship() + + bedrooms: List[Bedroom] = Relationship(back_populates="rooms") + bathrooms: List[Bathroom] = Relationship(back_populates="rooms") diff --git a/etl/models/preSiteNoteTypes.py b/etl/models/preSiteNoteTypes.py index f97f3bc..dbd9f5c 100644 --- a/etl/models/preSiteNoteTypes.py +++ b/etl/models/preSiteNoteTypes.py @@ -326,7 +326,6 @@ class CompanyInfo(BaseModel, table=True): assessors: List[AssessorInfo] = Relationship(back_populates="company") - class Insulation(BaseModel, table=True): type: str diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 19a46b4..4f23c8a 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -40,7 +40,8 @@ from etl.models.conditionReport import ( LoftSpace, RoomInRoof, Bedroom, - Bathroom + Bathroom, + Rooms ) from etl.models.topLevel import( @@ -121,22 +122,41 @@ class surveyedDataProcessor(): access_and_elevations = self.load_access_and_elevations_from_condition_report(db_session) rooms = self.load_rooms_from_condition_report(db_session) - pprint(rooms) - def load_rooms_from_condition_report(self, db_session): - self.load_hallway_from_condition_report(db_session) - self.load_living_room_from_condition_report(db_session) - self.load_dining_room_from_condition_report(db_session) - self.load_kitchen_from_condition_report(db_session) - self.load_utility_room_from_condition_report(db_session) - self.load_wash_chamber_from_condition_report(db_session) - self.load_landing_from_condition_report(db_session) - self.load_loft_space_from_condition_report(db_session) - self.load_room_in_roof_from_condition_report(db_session) - self.load_bedrooms_from_condition_report(db_session) - self.load_bathrooms_from_condition_report(db_session) + hallway = self.load_hallway_from_condition_report(db_session) + living_room = self.load_living_room_from_condition_report(db_session) + dining_room = self.load_dining_room_from_condition_report(db_session) + kitchen = self.load_kitchen_from_condition_report(db_session) + utility_room = self.load_utility_room_from_condition_report(db_session) + wash_chamber = self.load_wash_chamber_from_condition_report(db_session) + landing = self.load_landing_from_condition_report(db_session) + loft_space = self.load_loft_space_from_condition_report(db_session) + rir = self.load_room_in_roof_from_condition_report(db_session) - def load_bathrooms_from_condition_report(self, db_session): + rooms = self.upsert_record( + model_class=Rooms, + data_dict=self.condition_report.master_obj.rooms.model_dump(), + db_session=db_session, + additional_fields={ + "hallway_id": hallway.id, + "living_room_id": living_room.id, + "dining_room_id": dining_room.id, + "kitchen_id": kitchen.id, + "utility_id": utility_room.id, + "wash_chamber_id": wash_chamber.id, + "landing_id": landing.id, + "loft_space_id": loft_space.id, + "room_in_roof_id": rir.id, + } + ) + + bedrooms = self.load_bedrooms_from_condition_report(db_session, rooms.id) + bathrooms = self.load_bathrooms_from_condition_report(db_session, rooms.id) + + return rooms + + + def load_bathrooms_from_condition_report(self, db_session, rooms_id): bathroom_ids = [] for i, bathrooms in enumerate(self.condition_report.master_obj.rooms.bathrooms): room_info = self.load_room_info_from_condition_report( @@ -149,12 +169,13 @@ class surveyedDataProcessor(): data_dict=bathrooms.model_dump(), additional_fields={ "room_info_id": room_info.id, + "rooms_id": rooms_id, } ) bathroom_ids.append(bathroom.id) return bathroom_ids - def load_bedrooms_from_condition_report(self, db_session): + def load_bedrooms_from_condition_report(self, db_session, rooms_id): bedroom_ids = [] for i, bedrooms in enumerate(self.condition_report.master_obj.rooms.bedrooms): room_info = self.load_room_info_from_condition_report( @@ -167,6 +188,7 @@ class surveyedDataProcessor(): data_dict=bedrooms.model_dump(), additional_fields={ "room_info_id": room_info.id, + "rooms_id": rooms_id, } ) bedroom_ids.append(bedroom.id) @@ -885,9 +907,9 @@ class surveyedDataProcessor(): update_if_exists: bool = False, additional_fields: dict = None ): - def remove_nested_dicts(data: dict): - return {k: v for k, v in data.items() if not isinstance(v, dict)} - clean_data = remove_nested_dicts(data_dict) + def remove_nested_dicts_and_lists(data: dict): + return {k: v for k, v in data.items() if not isinstance(v, (dict, list))} + clean_data = remove_nested_dicts_and_lists(data_dict) # Merge additional fields if provided if additional_fields: From 01f3a0cec66638a6bd7c5f6c8688606f9bab26e2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 27 Jun 2025 16:09:20 +0000 Subject: [PATCH 15/22] save current progress but need to finsih section 4 --- etl/models/conditionReport.py | 63 ++++++++++++++- etl/surveyedData/surveryedData.py | 127 ++++++++++++++++++++++-------- 2 files changed, 158 insertions(+), 32 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index fcd7067..3321999 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -1,5 +1,5 @@ # SQLModel mapping for ConditionReportModel using BaseModel -from sqlmodel import SQLModel, Field, Relationship +from sqlmodel import SQLModel, Field, Relationship, Column, JSON from typing import Optional, List import uuid from datetime import datetime @@ -234,3 +234,64 @@ class Rooms(BaseModel, table=True): bedrooms: List[Bedroom] = Relationship(back_populates="rooms") bathrooms: List[Bathroom] = Relationship(back_populates="rooms") + +class GeneralConditionHeatingSystem(BaseModel, table=True): + is_the_heating_system_in_working_order: bool + does_the_occupant_have_a_smart_meter: bool + are_there_any_smart_monitoring_devices: bool + are_the_gas_and_electricity_meters_accessible: bool + dual_or_single_electric_meter: str + +class MainHeatingOne(BaseModel, table=True): + as_defined_by: str + fuel: str + type: str + +class MainHeatingTwo(BaseModel, table=True): + is_there_a_main_heating_two: bool + +class SecondaryHeating(BaseModel, table=True): + is_there_a_secondary_heating: bool + fuel: str + electric_heating_type: str + gas_heating_type: str + +class HeatingByRoom(BaseModel, table=True): + rooms_heated_by_main_system_one: List[str] = Field(sa_column=Column(JSON)) + rooms_heated_by_main_system_two: List[str] = Field(sa_column=Column(JSON)) + rooms_heated_by_secondary_heating: List[str] = Field(sa_column=Column(JSON)) + are_there_any_partially_heated_rooms: bool + partially_heated_rooms: Optional[List[str]] = Field(sa_column=Column(JSON)) + are_there_any_unheated_rooms: bool + unheated_rooms: List[str] = Field(sa_column=Column(JSON)) + + +class Renewables(BaseModel, table=True): + is_there_any_renewable_energy_system_in_place: bool + suitable_roof_orientation_for_solar_pv_water: str + is_there_a_water_tank: bool + type: str + size: str + tank_location: str + is_the_tank_insulated: bool + type_of_insulation: str + thickness_of_insulation_in_mm: int + +class HeatingSystem(BaseModel, table=True): + general_condition_id: uuid.UUID = Field(foreign_key="generalconditionheatingsystem.id") + general_condition: GeneralConditionHeatingSystem + + main_heating_one_id: uuid.UUID = Field(foreign_key="mainheatingone.id") + main_heating_one: MainHeatingOne + + main_heating_two_id: uuid.UUID = Field(foreign_key="maineheatingtwo.id") + main_heating_two: MainHeatingTwo + + secondary_heating_id: uuid.UUID = Field(foreign_key="secondaryheating.id") + secondary_heating: SecondaryHeating + + heating_by_room_id: uuid.UUID = Field(foreign_key="heatingbyroom.id") + heating_by_room: HeatingByRoom# + + renewables_id: uuid.UUID = Field(foreign_key="renewables.id") + renewables: Renewables \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 4f23c8a..5807fe1 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -41,7 +41,14 @@ from etl.models.conditionReport import ( RoomInRoof, Bedroom, Bathroom, - Rooms + Rooms, + GeneralConditionHeatingSystem, + MainHeatingOne, + MainHeatingTwo, + SecondaryHeating, + HeatingByRoom, + Renewables, + HeatingSystem ) from etl.models.topLevel import( @@ -76,7 +83,7 @@ class surveyedDataProcessor(): # My task to complete load # [x] general_information = self.get_section_1() # [x] access_and_elevations = self.get_section_2() - # [] rooms = self.get_section_3() + # [x] rooms = self.get_section_3() # [] heating_system = self.get_section_4() # [] occupant_assessment = self.get_section_5() # [] site_name, reference_code, address, postcode = self.get_section_0() @@ -92,35 +99,87 @@ class surveyedDataProcessor(): # heating_system=heating_system, # occupancy_assessment=occupant_assessment, # ) - # - # hallway = self.get_hallway() - # living_room = self.get_living_room() - # dining_room = self.get_dining_room() - # kitchen = self.get_kitchen() - # utility = self.get_utility() - # wc = self.get_wc() - # landing = self.get_landing() - # bedrooms = self.get_bedrooms() - # bathrooms = self.get_bathroom() - # loft_space = self.get_loft_space() - # room_in_roof = self.get_room_in_roof() + # class HeatingSystem(BaseModel): + # heating_by_room: HeatingByRoom + # renewables: Renewables - # return Rooms( - # hallway=hallway, - # living_room=living_room, - # dining_room=dining_room, - # kitchen=kitchen, - # utility=utility, - # wash_chamber=wc, - # landing=landing, - # bedrooms=bedrooms, - # bathrooms=bathrooms, - # loft_space=loft_space, - # room_in_roof=room_in_roof, - # ) general_information = self.load_general_information_from_condition_report(db_session) access_and_elevations = self.load_access_and_elevations_from_condition_report(db_session) rooms = self.load_rooms_from_condition_report(db_session) + heating_system = self.load_heating_system_from_condition_report(db_session) + + + def load_heating_system_from_condition_report(self, db_session): + general_condition = self.load_general_condition_of_heating_system_from_condition_report(db_session) + main_heating_one = self.load_main_heating_one_from_condition_report(db_session) + main_heating_two = self.load_main_heating_two_from_condition_report(db_session) + secondary_heating = self.load_secondary_heating_from_condition_report(db_session) + heating_by_room = self.load_heating_by_room_from_condition_report(db_session) + renwables = self.load_renewables_from_condition_report(db_session) + + return self.get_attribute_and_load( + self.condition_report.master_obj, + "heating_system", + HeatingSystem, + db_session, + additional_fields={ + "general_condition_id": general_condition.id, + "main_heating_one_id": main_heating_one.id, + "main_heating_two_id": main_heating_two.id, + "secondary_heating": secondary_heating.id, + "heating_by_room_id": heating_by_room.id, + "renewables_id": renwables.id, + }, + ) + + def load_renewables_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "renewables", + Renewables, + db_session, + ) + + def load_heating_by_room_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "heating_by_room", + HeatingByRoom, + db_session, + exclude_list=False + ) + + def load_secondary_heating_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "secondary_heating", + SecondaryHeating, + db_session, + ) + + def load_main_heating_two_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "main_heating_two", + MainHeatingTwo, + db_session, + ) + + def load_main_heating_one_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "main_heating_one", + MainHeatingOne, + db_session, + ) + + def load_general_condition_of_heating_system_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.heating_system, + "general_condition", + GeneralConditionHeatingSystem, + db_session, + ) def load_rooms_from_condition_report(self, db_session): hallway = self.load_hallway_from_condition_report(db_session) @@ -564,7 +623,7 @@ class surveyedDataProcessor(): lookup_field="UPRN", ) - def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None, additional_fields={}): + def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None, additional_fields={}, exclude_list=True): found = getattr(obj, attr_string, None) if found: print(f"Uploading to data base {found}") @@ -576,7 +635,8 @@ class surveyedDataProcessor(): model_class=pydanticModel, data_dict=found.model_dump(), lookup_field=lookup_field, - additional_fields=additional_fields + additional_fields=additional_fields, + exclude_list=exclude_list, ) return db return None @@ -905,10 +965,15 @@ class surveyedDataProcessor(): data_dict, lookup_field=None, update_if_exists: bool = False, - additional_fields: dict = None + additional_fields: dict = None, + exclude_list=True ): def remove_nested_dicts_and_lists(data: dict): - return {k: v for k, v in data.items() if not isinstance(v, (dict, list))} + if exclude_list: + return {k: v for k, v in data.items() if not isinstance(v, (dict, list))} + else: + return {k: v for k, v in data.items() if not isinstance(v, (dict))} + clean_data = remove_nested_dicts_and_lists(data_dict) # Merge additional fields if provided From 8e16dba6f1954f4664764428f0522747242a9918 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 30 Jun 2025 10:25:34 +0000 Subject: [PATCH 16/22] section 4 compelted --- etl/models/conditionReport.py | 14 +++++++------- etl/surveyedData/surveryedData.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 3321999..fde26f8 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -279,19 +279,19 @@ class Renewables(BaseModel, table=True): class HeatingSystem(BaseModel, table=True): general_condition_id: uuid.UUID = Field(foreign_key="generalconditionheatingsystem.id") - general_condition: GeneralConditionHeatingSystem + general_condition: GeneralConditionHeatingSystem = Relationship() main_heating_one_id: uuid.UUID = Field(foreign_key="mainheatingone.id") - main_heating_one: MainHeatingOne + main_heating_one: MainHeatingOne = Relationship() - main_heating_two_id: uuid.UUID = Field(foreign_key="maineheatingtwo.id") - main_heating_two: MainHeatingTwo + main_heating_two_id: uuid.UUID = Field(foreign_key="mainheatingtwo.id") + main_heating_two: MainHeatingTwo = Relationship() secondary_heating_id: uuid.UUID = Field(foreign_key="secondaryheating.id") - secondary_heating: SecondaryHeating + secondary_heating: SecondaryHeating = Relationship() heating_by_room_id: uuid.UUID = Field(foreign_key="heatingbyroom.id") - heating_by_room: HeatingByRoom# + heating_by_room: HeatingByRoom = Relationship() renewables_id: uuid.UUID = Field(foreign_key="renewables.id") - renewables: Renewables \ No newline at end of file + renewables: Renewables = Relationship() \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 5807fe1..2dcdab1 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -117,6 +117,15 @@ class surveyedDataProcessor(): heating_by_room = self.load_heating_by_room_from_condition_report(db_session) renwables = self.load_renewables_from_condition_report(db_session) + print("hello junte") + print(general_condition.id) + print(main_heating_one.id) + print(main_heating_two.id) + print(secondary_heating.id) + print(heating_by_room.id) + print(renwables.id) + print("goodbye junte") + return self.get_attribute_and_load( self.condition_report.master_obj, "heating_system", @@ -126,9 +135,9 @@ class surveyedDataProcessor(): "general_condition_id": general_condition.id, "main_heating_one_id": main_heating_one.id, "main_heating_two_id": main_heating_two.id, - "secondary_heating": secondary_heating.id, + "secondary_heating_id": secondary_heating.id, "heating_by_room_id": heating_by_room.id, - "renewables_id": renwables.id, + "renewables_id": renwables.id }, ) From f9242b8aaf08afa11cecff9da3dea4b65a64b82b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 30 Jun 2025 11:03:51 +0000 Subject: [PATCH 17/22] occupant information --- etl/models/conditionReport.py | 14 +++++++++++- etl/surveyedData/surveryedData.py | 38 ++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index fde26f8..3eefb15 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -294,4 +294,16 @@ class HeatingSystem(BaseModel, table=True): heating_by_room: HeatingByRoom = Relationship() renewables_id: uuid.UUID = Field(foreign_key="renewables.id") - renewables: Renewables = Relationship() \ No newline at end of file + renewables: Renewables = Relationship() + +class Occupant(BaseModel, table=True): + name: str + have_evidence_of_12_months_of_fuel_bill_data: bool + total_number_of_occupants: int + no_of_adult_occupants: int + no_of_child_occupants: int + no_of_occupant_of_a_pensionable_age: int + are_there_any_vulnerable_people: bool + is_there_anyone_with_a_disability: bool + status_of_occupant: str + landlord_wrote_that_the_tenent_agrees_assessment_been_supplied: bool \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 2dcdab1..8e24664 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -48,7 +48,8 @@ from etl.models.conditionReport import ( SecondaryHeating, HeatingByRoom, Renewables, - HeatingSystem + HeatingSystem, + Occupant ) from etl.models.topLevel import( @@ -84,7 +85,7 @@ class surveyedDataProcessor(): # [x] general_information = self.get_section_1() # [x] access_and_elevations = self.get_section_2() # [x] rooms = self.get_section_3() - # [] heating_system = self.get_section_4() + # [x] heating_system = self.get_section_4() # [] occupant_assessment = self.get_section_5() # [] site_name, reference_code, address, postcode = self.get_section_0() @@ -107,7 +108,31 @@ class surveyedDataProcessor(): access_and_elevations = self.load_access_and_elevations_from_condition_report(db_session) rooms = self.load_rooms_from_condition_report(db_session) heating_system = self.load_heating_system_from_condition_report(db_session) + occupant_assessment = self.load_occupant_info_from_condition_report(db_session) + def load_occupant_info_from_condition_report(self, db_session): + """ + occupant: Occupant + energy_use: EnergyUse + heating: HeatingFromConditionReport + shower_and_bath: ShowerAndBath + appliances: Optional[Appliances] + fridge_and_freezers: FridgeAndFreezers + cooker: Cooker + tumble_dryer: TumbleDryer + """ + self.load_occupant_information_from_condition_report(db_session) + + def load_occupant_information_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "occupant", + Occupant, + db_session, + additional_fields={ + "landlord_wrote_that_the_tenent_agrees_assessment_been_supplied":self.condition_report.master_obj.occupancy_assessment.occupant.landlord_has_written_confirmation_that_the_tenent_agrees_to_the_assessment_been_supplied + } + ) def load_heating_system_from_condition_report(self, db_session): general_condition = self.load_general_condition_of_heating_system_from_condition_report(db_session) @@ -117,15 +142,6 @@ class surveyedDataProcessor(): heating_by_room = self.load_heating_by_room_from_condition_report(db_session) renwables = self.load_renewables_from_condition_report(db_session) - print("hello junte") - print(general_condition.id) - print(main_heating_one.id) - print(main_heating_two.id) - print(secondary_heating.id) - print(heating_by_room.id) - print(renwables.id) - print("goodbye junte") - return self.get_attribute_and_load( self.condition_report.master_obj, "heating_system", From ebb2443a5c1d94c631473e4b241ea5a274291649 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 30 Jun 2025 15:46:46 +0000 Subject: [PATCH 18/22] save work --- etl/models/conditionReport.py | 79 +++++++++++- etl/surveyedData/surveryedData.py | 206 ++++++++++++++++++------------ 2 files changed, 199 insertions(+), 86 deletions(-) diff --git a/etl/models/conditionReport.py b/etl/models/conditionReport.py index 3eefb15..6113297 100644 --- a/etl/models/conditionReport.py +++ b/etl/models/conditionReport.py @@ -306,4 +306,81 @@ class Occupant(BaseModel, table=True): are_there_any_vulnerable_people: bool is_there_anyone_with_a_disability: bool status_of_occupant: str - landlord_wrote_that_the_tenent_agrees_assessment_been_supplied: bool \ No newline at end of file + landlord_wrote_that_the_tenent_agrees_assessment_been_supplied: bool + + +class EnergyUse(BaseModel, table=True): + property_tenure: str + who_is_the_electricity_payer: str + + +class HeatingFromConditionReport(BaseModel, table=True): + room_stat_in_temperature_in_celsius: Optional[str] = None + room_stat_location: Optional[str] = None + is_the_heating_pattern_known: Optional[str] = None + +class ShowerAndBath(BaseModel, table=True): + shower_type: str + do_you_know_the_no_of_showers_per_day_per_week: bool + please_input_no_of_showers_and_specify_a_day_or_a_week: str + do_you_know_the_number_of_baths_per_day_or_per_week: str + +class FridgeAndFreezers(BaseModel, table=True): + no_of_stand_alone_seperate_fridges: int + no_of_stand_alone_seperate_freezers: int + no_of_stand_alone_or_integrated_fridge_freezers: int + +class Cooker(BaseModel,table=True): + range_fuel: str + normal_large_range: str + cooker_type: str + +class TumbleDryer(BaseModel, table=True): + percentage_of_annual_use: int + space_for_outdoor_drying: bool + +class OccupantAssessment(BaseModel, table=True): + occupant_id: uuid.UUID = Field(foreign_key="occupant.id") + occupant: Occupant = Relationship() + + energy_use_id: uuid.UUID = Field(foreign_key="energyuse.id") + energy_use: EnergyUse = Relationship() + + heating_id: uuid.UUID = Field(foreign_key="heatingfromconditionreport.id") + heating: HeatingFromConditionReport = Relationship() + + shower_and_bath_id: uuid.UUID = Field(foreign_key="showerandbath.id") + shower_and_bath: ShowerAndBath = Relationship() + + # appliances: Optional[Appliances] + # appliances_id + + fridge_and_freezers_id: uuid.UUID = Field(foreign_key="fridgeandfreezers.id") + fridge_and_freezers: FridgeAndFreezers = Relationship() + + cooker_id: uuid.UUID = Field(foreign_key="cooker.id") + cooker: Cooker = Relationship() + + tumble_dryer_id: uuid.UUID = Field(foreign_key="tumbledryer.id") + tumble_dryer: TumbleDryer = Relationship() + +class ConditionReportModel(BaseModel, table=True): + project_site_name: str + property_reference_code: str + property_address: str + postcode: str + + general_information_id: uuid.UUID = Field(foreign_key="generalinformation.id") + general_information: GeneralInformation = Relationship() + + access_and_elevations_id: uuid.UUID = Field(foreign_key="accessandelevations.id") + access_and_elevations: AccessAndElevations = Relationship() + + rooms_id: uuid.UUID = Field(foreign_key="rooms.id") + rooms: Rooms = Relationship() + + heating_system_id: uuid.UUID = Field(foreign_key="heatingsystem.id") + heating_system: HeatingSystem = Relationship() + + occupancy_assessment_id: uuid.UUID = Field(foreign_key="occupantassessment.id") + occupancy_assessment: OccupantAssessment = Relationship() diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 8e24664..6e7a404 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -13,43 +13,17 @@ from etl.models.preSiteNoteTypes import ( from pprint import pprint from etl.models.conditionReport import ( - AssessorDetails, - InspectionAndProject, - TheProperty, ElevationInfo, - MainElevation, - Elevation, - GeneralInformation, - PropertyAccess, - ExternalElevation, - ExternalElevationFront, - ExternalElevationRear, - ExternalElevationGableOne, - ExternalElevationGableTwo, - ConservatoryOrOutbuilding, - AccessAndElevations, - Hallway, - VentilationInfo, - WindowsInfo, - RoomInfo, - LivingRoom, - DiningRoom, - Kitchen, - Utility, - WC, - Landing, - LoftSpace, - RoomInRoof, - Bedroom, - Bathroom, - Rooms, - GeneralConditionHeatingSystem, - MainHeatingOne, - MainHeatingTwo, - SecondaryHeating, - HeatingByRoom, - Renewables, - HeatingSystem, - Occupant + AssessorDetails, InspectionAndProject, TheProperty, ElevationInfo, + MainElevation, Elevation,GeneralInformation, + PropertyAccess, ExternalElevation, + ExternalElevationFront, ExternalElevationRear, ExternalElevationGableOne, + ExternalElevationGableTwo, ConservatoryOrOutbuilding, AccessAndElevations,Hallway, + VentilationInfo,WindowsInfo,RoomInfo,LivingRoom, DiningRoom, Kitchen, Utility, WC, + Landing,LoftSpace,RoomInRoof, Bedroom,Bathroom, Rooms, + GeneralConditionHeatingSystem, MainHeatingOne, MainHeatingTwo, + SecondaryHeating, HeatingByRoom, Renewables, HeatingSystem, Occupant, EnergyUse, + HeatingFromConditionReport, ShowerAndBath, FridgeAndFreezers, Cooker, TumbleDryer, + OccupantAssessment, ConditionReportModel ) from etl.models.topLevel import( @@ -81,47 +55,123 @@ class surveyedDataProcessor(): self.condition_report = pdf.get_reader() def load_condition_report(self, db_session): - # My task to complete load - # [x] general_information = self.get_section_1() - # [x] access_and_elevations = self.get_section_2() - # [x] rooms = self.get_section_3() - # [x] heating_system = self.get_section_4() - # [] occupant_assessment = self.get_section_5() - # [] site_name, reference_code, address, postcode = self.get_section_0() - - # return ConditionReportModel( - # project_site_name=site_name, - # property_reference_code=reference_code, - # property_address=address, - # postcode=postcode, - # general_information=general_information, - # access_and_elevations=access_and_elevations, - # rooms=rooms, - # heating_system=heating_system, - # occupancy_assessment=occupant_assessment, - # ) - # class HeatingSystem(BaseModel): - # heating_by_room: HeatingByRoom - # renewables: Renewables - general_information = self.load_general_information_from_condition_report(db_session) access_and_elevations = self.load_access_and_elevations_from_condition_report(db_session) rooms = self.load_rooms_from_condition_report(db_session) heating_system = self.load_heating_system_from_condition_report(db_session) occupant_assessment = self.load_occupant_info_from_condition_report(db_session) + condition_report = self.get_attribute_and_load( + self.condition_report, + "master_obj", + ConditionReportModel, + db_session, + additional_fields={ + "general_information_id": general_information.id, + "access_and_elevations_id": access_and_elevations.id, + "rooms_id": rooms.id, + "heating_system_id": heating_system.id, + "occupancy_assessment_id": occupant_assessment.id, + }, + ) + + # Create building table + data = { + "address": self.pre_site_note.survey_information.address, + "postcode": self.pre_site_note.survey_information.postcode, + "UPRN": self.pre_site_note.survey_information.uprn, + "landlord_id": landlord_id, + "domna_id": domna_id + } + building = self.upsert_record( + db_session=db_session, + model_class=Buildings, + data_dict=data, + lookup_field="UPRN", + ) + + return building + # Create document table + + def load_occupant_info_from_condition_report(self, db_session): - """ - occupant: Occupant - energy_use: EnergyUse - heating: HeatingFromConditionReport - shower_and_bath: ShowerAndBath - appliances: Optional[Appliances] - fridge_and_freezers: FridgeAndFreezers - cooker: Cooker - tumble_dryer: TumbleDryer - """ - self.load_occupant_information_from_condition_report(db_session) + occupant_info = self.load_occupant_information_from_condition_report(db_session) + energy_use = self.load_energy_use_from_condition_report(db_session) + heating = self.load_heating_from_condition_report(db_session) + showerAndBath = self.load_shower_and_bath_from_condition_report(db_session) + # The example I (Jun-te) had, had no appliances so skipped until I have one that needs this information + appliances = self.load_appliances_from_condition_report(db_session) + fridgeAndFreezers = self.load_fridge_and_freezer_from_condition_report(db_session) + cooker = self.load_cooker_from_condition_report(db_session) + tumble_dryer = self.load_tumble_dryer_from_condition_report(db_session) + + return self.get_attribute_and_load( + self.condition_report.master_obj, + "occupancy_assessment", + OccupantAssessment, + db_session, + additional_fields={ + "occupant_id": occupant_info.id, + "energy_use_id": energy_use.id, + "heating_id": heating.id, + "shower_and_bath_id": showerAndBath.id, + "fridge_and_freezers_id": fridgeAndFreezers.id, + "cooker_id": cooker.id, + "tumble_dryer_id": tumble_dryer.id + } + ) + + def load_tumble_dryer_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "tumble_dryer", + TumbleDryer, + db_session, + ) + + def load_cooker_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "cooker", + Cooker, + db_session, + ) + + def load_fridge_and_freezer_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "fridge_and_freezers", + FridgeAndFreezers, + db_session, + ) + + def load_appliances_from_condition_report(self, db_session): + print("Appliances was not done, please contact jun-te to complete this!") + return None + + def load_shower_and_bath_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "shower_and_bath", + ShowerAndBath, + db_session + ) + + def load_heating_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "heating", + HeatingFromConditionReport, + db_session + ) + + def load_energy_use_from_condition_report(self, db_session): + return self.get_attribute_and_load( + self.condition_report.master_obj.occupancy_assessment, + "energy_use", + EnergyUse, + db_session, + ) def load_occupant_information_from_condition_report(self, db_session): return self.get_attribute_and_load( @@ -634,20 +684,6 @@ class surveyedDataProcessor(): lookup_field="reference_number" ) - def create_building_table(self, db_session): - return self.upsert_record( - db_session=db_session, - model_class=Buildings, - data_dict={ - "address":"foo", - "potcode": "foobar", - "UPRN": self.pre_site_note.survey_information.uprn, - "landlord_id": "landlord_id", - "domna_id": "landlord_id", - }, - lookup_field="UPRN", - ) - def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session, lookup_field=None, additional_fields={}, exclude_list=True): found = getattr(obj, attr_string, None) if found: From 4d455cdda3c93c6472bae72bd5e601eaa1745473 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 1 Jul 2025 08:48:55 +0000 Subject: [PATCH 19/22] added new script for hubspot verification --- etl/hubspot_verification_to_db_load.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 etl/hubspot_verification_to_db_load.py diff --git a/etl/hubspot_verification_to_db_load.py b/etl/hubspot_verification_to_db_load.py new file mode 100644 index 0000000..bb42922 --- /dev/null +++ b/etl/hubspot_verification_to_db_load.py @@ -0,0 +1,18 @@ +import os +import pprint +from etl.hubSpotClient.hubspot import DealStage + +# Local development, comment this out to be production +os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" + +from etl.db.hubSpotLoad import HubspotTodb + + +hubspotClient = HubspotTodb() +deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) +pprint(deals) + + + + + From 58e27915cb8d05b6c9d824318806e51fd7d6a117 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 1 Jul 2025 23:36:31 +0000 Subject: [PATCH 20/22] save current progress --- etl/hubSpotClient/hubspot.py | 89 ++++++++++++++++++++++++-- etl/hubSpotClient/types.py | 31 +++++---- etl/hubspot_verification_to_db_load.py | 23 +++---- etl/scraper/scraper.py | 1 - etl/utils/utils.py | 55 ++++++++++------ 5 files changed, 149 insertions(+), 50 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index a05c309..c2d6af6 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -7,7 +7,10 @@ import time from pydantic import ValidationError from etl.utils.logger import Logger import logging - +import traceback +from hubspot.crm.objects.notes import SimplePublicObjectInput as NoteInput +from hubspot.crm.associations import BatchInputPublicAssociation, PublicAssociation +import time class DealStage(Enum): @@ -41,6 +44,7 @@ class HubSpotClient(): except Exception as e: return "Unknown Deal" # Fallback if the deal name is not found + def get_listings_from_deals_id(self, deals_id): from hubspot.crm.objects import PublicObjectSearchRequest found_notes = [] @@ -205,6 +209,9 @@ class HubSpotClient(): for deal in found_deals: domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id) try: + deal_name = deal.properties['dealname'] + self.logger.info(f"Validating <{deal_name}>") + input all_deals.append(SubmissionInfoFromDeal( deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], @@ -220,13 +227,46 @@ class HubSpotClient(): uprn = uprn, )) except Exception as e: + def format_error_note(e): + note_text = "⚠️ Error occurred while verifying deal data:

" + + if hasattr(e, "errors") and callable(e.errors): + note_text += "❌ Validation Errors:
" + for error in e.errors(): + loc = error.get('loc', 'N/A') + msg = error.get('msg', 'N/A') + error_type = error.get('type', 'N/A') + error_input = error.get('input', 'N/A') + + note_text += ( + f"• Field: {loc}
" + f"  - Message: {msg}
" + f"  - Type: {error_type}
" + f"  - Input: {error_input}

" + ) + else: + note_text += ( + "❗ Non-validation error:
" + f"
{str(e)}

" + ) + + note_text += ( + "🛠️ Please review this error and take necessary actions.
" + "Contact Jun-te if help is needed: +44 7519 530 549 or via Teams." + ) + + return note_text + deal_id = deal.properties['hs_object_id'] - self.logger.info(f"Deal <{deal_id}> not valid") + if hasattr(e, "errors"): + for error in e.errors(): + self.add_note_to_deal(deal_id, format_error_note(e)) + else: + self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True) + + + self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) - - - - return all_deals @@ -250,3 +290,40 @@ class HubSpotClient(): simple_public_object_input=deal_properties ) self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") + + + + def add_note_to_deal(self, deal_id, note_text): + try: + # Generate current time in milliseconds since epoch + hs_timestamp = int(time.time() * 1000) + + # Step 1: Create the note with hs_timestamp + note = NoteInput( + properties={ + "hs_note_body": note_text, + "hs_timestamp": hs_timestamp # Required field in your HubSpot setup + } + ) + created_note = self.client.crm.objects.notes.basic_api.create(note) + note_id = created_note.id + + # Step 2: Associate the note to the deal + association = PublicAssociation( + _from=note_id, + to=deal_id, + type="note_to_deal" + ) + + self.client.crm.associations.batch_api.create( + 'notes', + 'deals', + batch_input_public_association=BatchInputPublicAssociation( + inputs=[association] + ) + ) + + self.logger.info(f"📝 Note added to deal {deal_id}: {note_text}") + + except Exception as e: + self.logger.error(f"❌ Failed to add note to deal {deal_id}: {e}", exc_info=True) diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index 22ff6ff..e77acde 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -2,10 +2,14 @@ from sqlmodel import Field, SQLModel from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID import uuid -from pydantic import Field, field_validator, ValidationError, model_validator +from pydantic import Field, field_validator, model_validator from etl.utils.utils import get_sharepoint_path from etl.scraper.scraper import SharePointScraper, SharePointInstaller + + + + def string_to_installer(installer): if installer.upper() == "J & J CRUMP": return SharePointInstaller.JJC @@ -40,19 +44,24 @@ class SubmissionInfoFromDeal(BaseModel): @field_validator('post_sap_score', 'no_of_wet_rooms') @classmethod - def must_be_non_negative(cls, v): + def must_be_non_negative(cls, v, info): if v < 0: - raise ValidationError("Must be non-negative for Post Sap Score") + raise ValueError(f"{info.field_name} must be non-negative") return v @model_validator(mode="after") def check_submission_folder_path(self): - path = get_sharepoint_path(self.submission_folder_path) - installer = string_to_installer(self.installer) - sp = SharePointScraper(installer) - files = sp.get_folders_in_path(path) - if "value" in files: - if len(files["value"]) > 0: + errors = [] + + try: + path = get_sharepoint_path(self.submission_folder_path) + installer = string_to_installer(self.installer) + sp = SharePointScraper(installer) + files = sp.get_folders_in_path(path) + + if "value" in files and len(files["value"]) > 0: return self - - raise RuntimeError("Sharepoint URL invalid") + raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + + except Exception as e: + raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") \ No newline at end of file diff --git a/etl/hubspot_verification_to_db_load.py b/etl/hubspot_verification_to_db_load.py index bb42922..5553196 100644 --- a/etl/hubspot_verification_to_db_load.py +++ b/etl/hubspot_verification_to_db_load.py @@ -1,18 +1,15 @@ import os -import pprint -from etl.hubSpotClient.hubspot import DealStage +from pprint import pprint -# Local development, comment this out to be production +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.hubSpotClient.hubspot import DealStage, HubSpotClient +# Local development os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" -from etl.db.hubSpotLoad import HubspotTodb - - -hubspotClient = HubspotTodb() +hubspotClient = HubSpotClient() deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) -pprint(deals) - - - - - diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 6cbfd37..550e05d 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -13,7 +13,6 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() - print(f"Todays date is {today}") monday = today - timedelta(days=today.weekday()) # weekday() = 0 for Monday return f"W.C. {monday.strftime('%d.%m.%Y')}" diff --git a/etl/utils/utils.py b/etl/utils/utils.py index 286b645..584381a 100644 --- a/etl/utils/utils.py +++ b/etl/utils/utils.py @@ -1,23 +1,40 @@ from urllib.parse import unquote +class SharePointURLError(ValueError): + """Custom error for SharePoint URL parsing issues.""" + pass + + def get_sharepoint_path(url): - url_parts = url.split('/') - # Find the index of 'Forms' - forms_index = url_parts.index('Forms') - # Get the part after 'Forms' - after_forms = url_parts[forms_index + 1] - - # Find 'id=' and extract after it - if 'id=' in after_forms: - id_part = after_forms.split('id=')[1] - # Only keep the path before '&' (to ignore other parameters) - id_path = id_part.split('&')[0] - # Decode the path - decoded_path = unquote(id_path) - # Now, remove the leading '/sites/xxx/Shared Documents/' part - parts = decoded_path.split('Shared Documents') - if len(parts) > 1: - final_path = parts[1].strip('/') - return final_path + try: + url_parts = url.split('/') + + # Find the index of 'Forms' + forms_index = url_parts.index('Forms') + + # Get the part after 'Forms' + after_forms = url_parts[forms_index + 1] + + # Find 'id=' and extract after it + if 'id=' in after_forms: + id_part = after_forms.split('id=')[1] + + # Only keep the path before '&' (to ignore other parameters) + id_path = id_part.split('&')[0] + + # Decode the path + decoded_path = unquote(id_path) + + # Remove the leading '/sites/xxx/Shared Documents/' part + parts = decoded_path.split('Shared Documents') + + if len(parts) > 1: + final_path = parts[1].strip('/') + return final_path + else: + return decoded_path.strip('/') else: - return decoded_path.strip('/') \ No newline at end of file + raise SharePointURLError(f"The URL does not contain 'id=' parameter. URL: {url}") + + except (IndexError, ValueError) as e: + raise SharePointURLError(f"Error parsing SharePoint URL: {url}. Reason: {e}") \ No newline at end of file From 1e467cfd564bd5603f1efcccef7d71935db85df1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 9 Jul 2025 10:25:42 +0000 Subject: [PATCH 21/22] change pdf reader to file reader --- etl/{pdfReader => fileReader}/__init__.py | 0 .../pdfReaderToText.py | 19 +++++--- etl/{pdfReader => fileReader}/reportType.py | 4 +- etl/{pdfReader => fileReader}/sitenotes.py | 25 +++++++++-- etl/hubSpotClient/hubspot.py | 7 ++- etl/hubSpotClient/types.py | 44 ++++++++++++++----- etl/hubspot_verification_to_db_load.py | 5 +++ etl/scraper/scraper.py | 37 +++++++++++++++- etl/surveyedData/surveryedData.py | 31 ++++++++----- etl/utils/utils.py | 2 +- 10 files changed, 136 insertions(+), 38 deletions(-) rename etl/{pdfReader => fileReader}/__init__.py (100%) rename etl/{pdfReader => fileReader}/pdfReaderToText.py (69%) rename etl/{pdfReader => fileReader}/reportType.py (65%) rename etl/{pdfReader => fileReader}/sitenotes.py (99%) diff --git a/etl/pdfReader/__init__.py b/etl/fileReader/__init__.py similarity index 100% rename from etl/pdfReader/__init__.py rename to etl/fileReader/__init__.py diff --git a/etl/pdfReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py similarity index 69% rename from etl/pdfReader/pdfReaderToText.py rename to etl/fileReader/pdfReaderToText.py index 34e37ad..bc9643f 100644 --- a/etl/pdfReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -1,8 +1,8 @@ from etl.utils.logger import Logger import logging import pymupdf -from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor, CSR, ConditionReport -from etl.pdfReader.reportType import ReportType +from etl.fileReader.sitenotes import QuidosSiteNotesExtractor, CSR, WarmHomesConditionReport, ECOConditionReport, RDSAPEnergyReport +from etl.fileReader.reportType import ReportType class pdfReaderToText(): @@ -24,6 +24,7 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') + print(self.text_list) def get_list_of_text(self): return self.text_list @@ -41,7 +42,11 @@ class pdfReaderToText(): elif "Chartered Surveyor Report: Recommending Extraction of Defective Cavity Wall Insulation " in self.text_list: self.type = ReportType.CHARTED_SURVEYOR_REPORT elif "Osmosis ACD NEW PAS 2035 Condition Report".lower() in self.text_list[0].lower(): - self.type = ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT + self.type = ReportType.WARM_HOMES_CONDITION_REPORT + elif "Domna NEW PAS 2035 ECO Condition Report".lower() in self.text_list[0].lower(): + self.type = ReportType.ECO_CONDITION_REPORT + elif "ENERGY REPORT".lower() == self.text_list[0].lower(): + self.type = ReportType.RDSAP_ENERGY_REPORT else: pass return self.type @@ -53,6 +58,10 @@ class pdfReaderToText(): return QuidosSiteNotesExtractor(self.text_list) elif self.type == ReportType.CHARTED_SURVEYOR_REPORT: return CSR(self.text_list) - elif self.type == ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT: - return ConditionReport(self.text_list) + elif self.type == ReportType.WARM_HOMES_CONDITION_REPORT: + return WarmHomesConditionReport(self.text_list) + elif self.type == ReportType.ECO_CONDITION_REPORT: + return ECOConditionReport(self.text_list) + elif self.type == ReportType.RDSAP_ENERGY_REPORT: + return RDSAPEnergyReport(self.text_list) \ No newline at end of file diff --git a/etl/pdfReader/reportType.py b/etl/fileReader/reportType.py similarity index 65% rename from etl/pdfReader/reportType.py rename to etl/fileReader/reportType.py index a94847a..1db2efb 100644 --- a/etl/pdfReader/reportType.py +++ b/etl/fileReader/reportType.py @@ -7,4 +7,6 @@ class ReportType(Enum): ENERGY_PERFORMANCE_REPORT = "energy_performance_report" U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report" OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form" - OSMOSIS_CONDITION_PAS_2035_REPORT = "osmosis_condition_pas_2035_report" + ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report" + WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report" + RDSAP_ENERGY_REPORT = "rdsap_energy_report" diff --git a/etl/pdfReader/sitenotes.py b/etl/fileReader/sitenotes.py similarity index 99% rename from etl/pdfReader/sitenotes.py rename to etl/fileReader/sitenotes.py index 5bb3932..6026d9b 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -88,11 +88,30 @@ class CSR(SiteNotesExtractor): type=dict_.get('detailed_description_of_existing_cavity_wall_insulation_', "") ) if dict_ is not None else None - -class ConditionReport(SiteNotesExtractor): +class RDSAPEnergyReport(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) - self.type = ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT + self.type = ReportType.RDSAP_ENERGY_REPORT + self.master_obj = self.setup_energy_report() + + def setup_energy_report(self): + pass + +class ECOConditionReport(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.ECO_CONDITION_REPORT + self.master_obj = self.setup_condition_report() + + def setup_condition_report(self): + pass + + + +class WarmHomesConditionReport(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.WARM_HOMES_CONDITION_REPORT self.master_obj = self.setup_condition_report() def setup_condition_report(self): diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index c2d6af6..3b4913e 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -81,7 +81,7 @@ class HubSpotClient(): def get_domna_and_landlord_id(self, deals_id): data = self.get_listings_from_deals_id(deals_id) - return data.properties['domna_property_id'], data.properties['owner_property_id'], data.properties['national_uprn'] + return data.properties['domna_property_id'], data.properties['owner_property_id'], data.properties.get('national_uprn', '') or '' def get_notes_from_deals_id(self, deals_id): from hubspot.crm.objects import PublicObjectSearchRequest @@ -211,7 +211,7 @@ class HubSpotClient(): try: deal_name = deal.properties['dealname'] self.logger.info(f"Validating <{deal_name}>") - input + # input(f"Press enter to verfiy <{deal_name}>") all_deals.append(SubmissionInfoFromDeal( deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], @@ -228,7 +228,7 @@ class HubSpotClient(): )) except Exception as e: def format_error_note(e): - note_text = "⚠️ Error occurred while verifying deal data:

" + note_text = "⚠️ Automated Verification Failed:

" if hasattr(e, "errors") and callable(e.errors): note_text += "❌ Validation Errors:
" @@ -267,7 +267,6 @@ class HubSpotClient(): self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) - return all_deals def print_all_pipeline_ids(self): diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index e77acde..84ef0d5 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -5,6 +5,7 @@ import uuid from pydantic import Field, field_validator, model_validator from etl.utils.utils import get_sharepoint_path from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.surveyedData.surveryedData import surveyedDataProcessor @@ -16,7 +17,7 @@ def string_to_installer(installer): elif installer.upper() == "SCIS": return SharePointInstaller.SOUTH_COAST_INSULATION elif installer.upper() == "SGEC": - return SharePointInstaller.SGEC + return SharePointInstaller.JJC else: return None @@ -40,7 +41,7 @@ class SubmissionInfoFromDeal(BaseModel): submission_folder_path: str = Field(..., min_length=1) landlord_id: str = Field(..., min_length=1) domna_id: str = Field(..., min_length=1) - uprn: str = Field(..., min_length=1) + uprn: str @field_validator('post_sap_score', 'no_of_wet_rooms') @classmethod @@ -50,18 +51,37 @@ class SubmissionInfoFromDeal(BaseModel): return v @model_validator(mode="after") - def check_submission_folder_path(self): - errors = [] - + def check_sharepoint_link_and_contents(self): try: path = get_sharepoint_path(self.submission_folder_path) installer = string_to_installer(self.installer) sp = SharePointScraper(installer) - files = sp.get_folders_in_path(path) - - if "value" in files and len(files["value"]) > 0: - return self - raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") - except Exception as e: - raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") \ No newline at end of file + raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") + + try: + # Check if sharepoint link is reachable and has any contents + files = sp.get_folders_in_path(path) + if "value" in files and len(files["value"]) > 0: + pass + else: + raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + except Exception as e: + raise ValueError(str(e)) + + # download files in url and check files are there: + try: + files = sp.download_files_from_path(path) + print(files) + sdp = surveyedDataProcessor("fake address", files) + assert sdp.condition_report is not None, "Condition Report is missing" + assert sdp.energy_report is not None, "Energy Report pdf is missing" + except Exception as e: + raise ValueError(str(e)) + + return self + + + + + diff --git a/etl/hubspot_verification_to_db_load.py b/etl/hubspot_verification_to_db_load.py index 5553196..9b0f9e1 100644 --- a/etl/hubspot_verification_to_db_load.py +++ b/etl/hubspot_verification_to_db_load.py @@ -13,3 +13,8 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post hubspotClient = HubSpotClient() deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + + +# TODO sanity address check +# TODO load \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 550e05d..fb289cd 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -301,13 +301,46 @@ class SharePointScraper(): file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']}) each_file = [] for file_name, url in file_names_to_download.items(): - self.logger.info(f"Downloading {file_name} from {url}") + self.logger.debug(f"Downloading {file_name} from {url}") content = self.get_file_content(url) file_path = self.create_temp_file(content, f"{name}/{WEEK_COMMENCING}/{house_ass}/{address}/{file_name}") each_file.append(file_path) address_paths.update({address: each_file}) paths.append(address_paths) return paths + + def download_files_from_path(self, path): + """ + Download all non-media files from a list of root paths. + + Args: + root_paths (List[str]): List of full folder paths to start from. + + Returns: + List[Dict[str, List[str]]]: A list of dictionaries mapping address folder names to downloaded file paths. + """ + avoid = [".jpg", ".mov", ".JPG", ".heic", ".HEIC", ".png", ".PNG", ".jpeg", ".JPEG", ".mp4", ".MP4"] + + files_info = self.get_folders_in_path(path) + + if 'value' not in files_info: + raise RuntimeError(f"Failed to get files from {path}") + + file_names_to_download = { + file["name"]: file["@microsoft.graph.downloadUrl"] + for file in files_info['value'] + if 'file' in file and not any(file["name"].endswith(ext) for ext in avoid) + } + + downloaded_files = [] + for file_name, url in file_names_to_download.items(): + self.logger.info(f"Downloading {file_name} from {url}") + content = self.get_file_content(url) + file_path = self.create_temp_file(content, f"{path}/{file_name}") + downloaded_files.append(file_path) + + return downloaded_files + def create_temp_file(self, content, path): # Ensure the path is under /tmp/ @@ -320,6 +353,6 @@ class SharePointScraper(): with open(path, 'wb+') as temp_file: temp_file.write(content.getvalue()) - self.logger.info(f"Temporary file created at: {path}") + self.logger.debug(f"Temporary file created at: {path}") return path \ No newline at end of file diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 6e7a404..e4299ae 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,6 +1,7 @@ from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.pdfReader.reportType import ReportType import math +from xml.dom.minidom import parseString from etl.models.preSiteNoteTypes import ( AssessorInfo, CompanyInfo, PreSiteNotesSummaryInfo, @@ -38,21 +39,31 @@ class surveyedDataProcessor(): self.pre_site_note = None self.csr = None self.condition_report = None - self.identify_files() self.hubspot_deal_id = None + self.energy_report = None + self.identify_files() def identify_files(self): for file in self.files: - pdf = pdfReaderToText(file) - if pdf: - if pdf.type == ReportType.QUIDOS_PRESITE_NOTE: - self.pre_site_note = pdf.get_reader() - self.address = self.pre_site_note.survey_information.address - elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT: - self.csr = pdf.get_reader() - elif pdf.type == ReportType.OSMOSIS_CONDITION_PAS_2035_REPORT: - self.condition_report = pdf.get_reader() + + if file.lower().endswith('.pdf'): + pdf = pdfReaderToText(file) + if pdf: + if pdf.type == ReportType.QUIDOS_PRESITE_NOTE: + self.pre_site_note = pdf.get_reader() + self.address = self.pre_site_note.survey_information.address + elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT: + self.csr = pdf.get_reader() + elif pdf.type == ReportType.WARM_HOMES_CONDITION_REPORT: + self.condition_report = pdf.get_reader() + elif pdf.type == ReportType.ECO_CONDITION_REPORT: + self.condition_report = pdf.get_reader() + elif pdf.type == ReportType.RDSAP_ENERGY_REPORT: + self.energy_report = pdf.get_reader() + elif file.lower().endswith('.xml'): + print(f"identified an xml file {file.lower()}") + pass def load_condition_report(self, db_session): general_information = self.load_general_information_from_condition_report(db_session) diff --git a/etl/utils/utils.py b/etl/utils/utils.py index 584381a..ff77015 100644 --- a/etl/utils/utils.py +++ b/etl/utils/utils.py @@ -37,4 +37,4 @@ def get_sharepoint_path(url): raise SharePointURLError(f"The URL does not contain 'id=' parameter. URL: {url}") except (IndexError, ValueError) as e: - raise SharePointURLError(f"Error parsing SharePoint URL: {url}. Reason: {e}") \ No newline at end of file + raise SharePointURLError(f"Error with SharePoint URL, please check {url}. Reason: {e}") \ No newline at end of file From 4075fbaa3cc95379f29436dd0755c2ade14903fa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 10:08:58 +0000 Subject: [PATCH 22/22] survyed sign off --- ...ml => hubspot_surveyed_needs_sign_off.yml} | 6 +-- etl/daily_script.py | 2 +- etl/development.py | 2 +- etl/fileReader/reportType.py | 3 ++ etl/fileReader/sitenotes.py | 2 +- etl/fileReader/xmlReader.py | 43 +++++++++++++++++++ etl/hubSpotClient/types.py | 20 ++++++++- ....py => hubspot_surveyed_needs_sign_off.py} | 9 +++- etl/jjc_old_lewis_manual_way_.py | 2 +- etl/models/topLevel.py | 2 +- etl/scis_invoice.py | 2 +- etl/sgec_invoice.py | 2 +- etl/surveyedData/surveryedData.py | 20 +++++++-- etl/validator/validator.py | 4 +- 14 files changed, 99 insertions(+), 20 deletions(-) rename .github/workflows/{hubspot_deal_notes.yml => hubspot_surveyed_needs_sign_off.yml} (79%) create mode 100644 etl/fileReader/xmlReader.py rename etl/{hubspot_verification_to_db_load.py => hubspot_surveyed_needs_sign_off.py} (71%) diff --git a/.github/workflows/hubspot_deal_notes.yml b/.github/workflows/hubspot_surveyed_needs_sign_off.yml similarity index 79% rename from .github/workflows/hubspot_deal_notes.yml rename to .github/workflows/hubspot_surveyed_needs_sign_off.yml index ce80afe..2b036d6 100644 --- a/.github/workflows/hubspot_deal_notes.yml +++ b/.github/workflows/hubspot_surveyed_needs_sign_off.yml @@ -1,7 +1,7 @@ -name: Deal Notes From HubSpot Scraper +name: Daily Surved on: schedule: - - cron: '0 19 * * 0' + - cron: '0 17 * * 1-5' workflow_dispatch: jobs: @@ -24,6 +24,6 @@ jobs: run: | pwd ls -la - poetry run python etl/dimitra_hubspot_notes_gather.py + poetry run python etl/hubspot_surveyed_needs_sign_off.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/daily_script.py b/etl/daily_script.py index b2c76d9..1689167 100644 --- a/etl/daily_script.py +++ b/etl/daily_script.py @@ -1,5 +1,5 @@ import os -from pdfReader.pdfReaderToText import pdfReaderToText +from fileReader.pdfReaderToText import pdfReaderToText from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pprint, pformat import logging diff --git a/etl/development.py b/etl/development.py index f70c68d..9319e1c 100644 --- a/etl/development.py +++ b/etl/development.py @@ -1,6 +1,6 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller from pprint import pformat -from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd diff --git a/etl/fileReader/reportType.py b/etl/fileReader/reportType.py index 1db2efb..07ac12e 100644 --- a/etl/fileReader/reportType.py +++ b/etl/fileReader/reportType.py @@ -10,3 +10,6 @@ class ReportType(Enum): ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report" WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report" RDSAP_ENERGY_REPORT = "rdsap_energy_report" + LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark" + RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml" + FULLSAP_XML = "full_xml_needed_for_co_ordination" diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 6026d9b..f46726c 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -1,4 +1,4 @@ -from etl.pdfReader.reportType import ReportType +from etl.fileReader.reportType import ReportType from etl.transform.preSiteNoteTypes import ( CompanyInfo, PreSiteNotesSummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, diff --git a/etl/fileReader/xmlReader.py b/etl/fileReader/xmlReader.py new file mode 100644 index 0000000..d49b9d4 --- /dev/null +++ b/etl/fileReader/xmlReader.py @@ -0,0 +1,43 @@ +from etl.utils.logger import Logger +import logging +from xml.dom.minidom import parse +import os +from etl.fileReader.reportType import ReportType + +class xmlReader(): + def __init__(self, file_path): + self.source_path = file_path + self.logger = Logger(name='xmlReader', level=logging.INFO).get_logger() + self.xml_obj = None + self.type = None + self.get_xml_obj() + + + def get_xml_obj(self): + try: + if not os.path.exists(self.source_path): + self.logger.error(f"File not found: {self.source_path}") + return None + + with open(self.source_path, 'r', encoding='utf-8') as file: + self.xml_obj = parse(file) + self.get_type() + return self.xml_obj + + except Exception as e: + self.logger.error(f"Failed to parse XML file {self.source_path}: {e}") + self.xml_obj = None + return self.xml_obj + + def get_type(self): + xmlHeaderName = self.xml_obj.documentElement.tagName + xmlHeaderName = xmlHeaderName.lower() + if xmlHeaderName == 'RdSap-Report'.lower(): + self.type = ReportType.LIG_XML + elif xmlHeaderName == "SurveyRec".lower(): + self.type = ReportType.RDSAP_XML + elif xmlHeaderName == "ImportExportRecord".lower(): + self.type = ReportType.FULLSAP_XML + else: + pass + return self.type diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index 84ef0d5..dd97011 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -71,11 +71,27 @@ class SubmissionInfoFromDeal(BaseModel): # download files in url and check files are there: try: + files = sp.download_files_from_path(path) print(files) sdp = surveyedDataProcessor("fake address", files) - assert sdp.condition_report is not None, "Condition Report is missing" - assert sdp.energy_report is not None, "Energy Report pdf is missing" + missing_items = [] + + if sdp.condition_report is None: + missing_items.append("Condition Report") + + if sdp.energy_report is None: + missing_items.append("Energy Report PDF") + + if sdp.rd_sap_xml is None: + missing_items.append("RDSAP XML") + + if sdp.lig_sap_xml is None: + missing_items.append("LIG SAP XML") + + if missing_items: + raise ValueError(f"Missing required items: {', '.join(missing_items)}") + except Exception as e: raise ValueError(str(e)) diff --git a/etl/hubspot_verification_to_db_load.py b/etl/hubspot_surveyed_needs_sign_off.py similarity index 71% rename from etl/hubspot_verification_to_db_load.py rename to etl/hubspot_surveyed_needs_sign_off.py index 9b0f9e1..a8eeabc 100644 --- a/etl/hubspot_verification_to_db_load.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -1,3 +1,7 @@ +""" +This is the script that runs when we are at the 'surveyed-needs sign off' stage within hubspot +""" + import os from pprint import pprint @@ -15,6 +19,7 @@ hubspotClient = HubSpotClient() deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) +for deal in deals: + hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) -# TODO sanity address check -# TODO load \ No newline at end of file +# TODO load when we are at 'ready to co-ordination' - script! \ No newline at end of file diff --git a/etl/jjc_old_lewis_manual_way_.py b/etl/jjc_old_lewis_manual_way_.py index ff12e94..027b515 100644 --- a/etl/jjc_old_lewis_manual_way_.py +++ b/etl/jjc_old_lewis_manual_way_.py @@ -5,7 +5,7 @@ os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pformat -from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd import math diff --git a/etl/models/topLevel.py b/etl/models/topLevel.py index a2e2326..784d3e5 100644 --- a/etl/models/topLevel.py +++ b/etl/models/topLevel.py @@ -5,7 +5,7 @@ from datetime import datetime from pydantic import EmailStr from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID -from etl.pdfReader.reportType import ReportType +from etl.fileReader.reportType import ReportType class BaseModel(SQLModel): id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index fc9d481..9a479c6 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -1,6 +1,6 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pformat -from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd diff --git a/etl/sgec_invoice.py b/etl/sgec_invoice.py index fc9d481..9a479c6 100644 --- a/etl/sgec_invoice.py +++ b/etl/sgec_invoice.py @@ -1,6 +1,6 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pformat -from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index e4299ae..485d617 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,5 +1,6 @@ -from etl.pdfReader.pdfReaderToText import pdfReaderToText -from etl.pdfReader.reportType import ReportType +from etl.fileReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.xmlReader import xmlReader +from etl.fileReader.reportType import ReportType import math from xml.dom.minidom import parseString from etl.models.preSiteNoteTypes import ( @@ -41,6 +42,10 @@ class surveyedDataProcessor(): self.condition_report = None self.hubspot_deal_id = None self.energy_report = None + self.full_sap_xml = None + self.lig_sap_xml = None + self.rd_sap_xml = None + self.identify_files() @@ -62,8 +67,15 @@ class surveyedDataProcessor(): elif pdf.type == ReportType.RDSAP_ENERGY_REPORT: self.energy_report = pdf.get_reader() elif file.lower().endswith('.xml'): - print(f"identified an xml file {file.lower()}") - pass + xml = xmlReader(file) + if xml: + if xml.type is ReportType.FULLSAP_XML: + self.full_sap_xml = xml.xml_obj + + elif xml.type is ReportType.LIG_XML: + self.lig_sap_xml = xml.xml_obj + elif xml.type is ReportType.RDSAP_XML: + self.rd_sap_xml = xml.xml_obj def load_condition_report(self, db_session): general_information = self.load_general_information_from_condition_report(db_session) diff --git a/etl/validator/validator.py b/etl/validator/validator.py index 4a5ce3a..2dd88a5 100644 --- a/etl/validator/validator.py +++ b/etl/validator/validator.py @@ -2,8 +2,8 @@ import os import logging from etl.utils.logger import Logger import re -from etl.pdfReader.pdfReaderToText import pdfReaderToText -from etl.pdfReader.reportType import ReportType +from etl.fileReader.pdfReaderToText import pdfReaderToText +from etl.fileReader.reportType import ReportType class DomnaSharePointValidator():