From 4c9bcada1332d9a65bb0a754a5418b9ad5cd4bbe Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 12 May 2025 13:25:28 +0000 Subject: [PATCH] added migration scripts --- .../9f45742b4b2f_initial_migration.py | 328 ++++++++++++++++++ .../versions/e351a98aba54_init_some_tables.py | 117 ------- etl/load/preSiteNoteTypes.py | 43 ++- etl/surveyedData/surveryedData.py | 63 ++-- etl/transform/preSiteNoteTypes.py | 9 +- migration_db.sh | 3 +- 6 files changed, 397 insertions(+), 166 deletions(-) create mode 100644 alembic/versions/9f45742b4b2f_initial_migration.py delete mode 100644 alembic/versions/e351a98aba54_init_some_tables.py diff --git a/alembic/versions/9f45742b4b2f_initial_migration.py b/alembic/versions/9f45742b4b2f_initial_migration.py new file mode 100644 index 0000000..a59e894 --- /dev/null +++ b/alembic/versions/9f45742b4b2f_initial_migration.py @@ -0,0 +1,328 @@ +"""Initial migration + +Revision ID: 9f45742b4b2f +Revises: +Create Date: 2025-05-12 13:24:03.856980 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '9f45742b4b2f' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('companyinfo', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('trading_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('post_code', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('fax_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('related_party_disclosure', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('door', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('no_of_doors', sa.Integer(), nullable=False), + sa.Column('no_of_insulated_doors', sa.Integer(), nullable=False), + sa.Column('u_value_w_m2_k', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('floors', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('floor_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ground_floor_construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ground_floor_insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('floor_insulation_thickness_mm', sa.Float(), nullable=True), + sa.Column('u_value_known', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fluegasheatrecoverysystem', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('fghrs_present', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('heatingsystemcontrols', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('control_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('flue_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('fan_assisted_flue', sa.Boolean(), nullable=False), + sa.Column('heat_emitter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('mains_gas_available', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('heatingtype', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('hotwatercylinder', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('volume', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('thermostat', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('insulation', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('lighting', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('total_no_of_light_fittings', sa.Integer(), nullable=False), + sa.Column('total_no_of_lel_fittings', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('otherdetails', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('main_gas_avalible', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('photovoltaicpanel', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('pvs_are_connected_to_dwelling_electricity_meter', sa.Boolean(), nullable=False), + sa.Column('percentage_of_external_roof_area_with_pvs', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('presitenotessummaryinfo', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('reference_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('epc_language', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('region', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('town', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('county', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('property_tenure', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('transaction_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('inspection_date', sa.DateTime(), nullable=False), + sa.Column('current_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('potential_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('current_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('potential_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('current_annual_emissions', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('current_annual_emission_including_0925_multiplayer', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('current_annual_energy_costs', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('roofs', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('u_value_known', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('showerandbaths', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('no_of_rooms_with_baths_and_or_shower', sa.Integer(), nullable=False), + sa.Column('no_of_rooms_with_mixer_shower_and_no_baths', sa.Integer(), nullable=False), + sa.Column('no_of_rooms_with_mixer_shower_and_baths', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('solarwaterheating', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('solar_water_heating_details_known', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ventilationandcooling', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('no_of_open_fireplaces', sa.Integer(), nullable=False), + sa.Column('ventilation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('space_cooling_system_present', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('walls', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('insulation_thickness_mm', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('wall_thickness_measured', sa.Boolean(), nullable=False), + sa.Column('wall_thickness_mm', sa.Integer(), nullable=True), + sa.Column('u_value_known', sa.Boolean(), nullable=False), + sa.Column('u_value_w_m2_k', sa.Float(), nullable=True), + sa.Column('dry_lining', sa.Boolean(), nullable=False), + sa.Column('alternative_wall_present', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('waterheating', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('windturbine', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('wind_turbine', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('assessorinfo', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('accreditation_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('phone_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('email_address', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('company_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['company_id'], ['companyinfo.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('heating', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('heating_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('efficiency_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('heating_fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('brand_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('model_qualifer', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('sap_2009_table', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('percentage_of_heated_floor_area_served', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('controls_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['controls_id'], ['heatingsystemcontrols.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('propertydetail', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('age_band', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('wall_id', sa.Uuid(), nullable=True), + sa.Column('roof_id', sa.Uuid(), nullable=True), + sa.Column('floor_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['floor_id'], ['floors.id'], ), + sa.ForeignKeyConstraint(['roof_id'], ['roofs.id'], ), + sa.ForeignKeyConstraint(['wall_id'], ['walls.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('dimension', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('floor_area_m2', sa.Float(), nullable=False), + sa.Column('room_height_m', sa.Float(), nullable=False), + sa.Column('loss_perimeter_m', sa.Float(), nullable=False), + sa.Column('party_wall_length_m', sa.Float(), nullable=False), + sa.Column('property_detail_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('presitenote', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('summary_info_id', sa.Uuid(), nullable=False), + sa.Column('assessor_id', sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ), + sa.ForeignKeyConstraint(['summary_info_id'], ['presitenotessummaryinfo.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('propertydescription', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('built_form', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('detachment_or_position', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('no_of_main_property', sa.Integer(), nullable=False), + sa.Column('no_of_extension_1', sa.Integer(), nullable=True), + sa.Column('no_of_extension_2', sa.Integer(), nullable=True), + sa.Column('no_of_extension_3', sa.Integer(), nullable=True), + sa.Column('no_of_extension_4', sa.Integer(), nullable=True), + sa.Column('no_of_habitable_rooms', sa.Integer(), nullable=False), + sa.Column('no_of_heated_rooms', sa.Integer(), nullable=False), + sa.Column('heated_basement', sa.Boolean(), nullable=False), + sa.Column('conservatory_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('percentage_of_draught_proofed', sa.Integer(), nullable=False), + sa.Column('terrain_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('conservatory', sa.Boolean(), nullable=False), + sa.Column('main_property_id', sa.Uuid(), nullable=False), + sa.Column('ex1_property_id', sa.Uuid(), nullable=True), + sa.Column('ex2_property_id', sa.Uuid(), nullable=True), + sa.Column('ex3_property_id', sa.Uuid(), nullable=True), + sa.Column('ex4_property_id', sa.Uuid(), nullable=True), + sa.Column('door_id', sa.Uuid(), nullable=True), + sa.Column('ventilation_and_cooling_id', sa.Uuid(), nullable=True), + sa.Column('lighting_id', sa.Uuid(), nullable=True), + sa.Column('water_heating_id', sa.Uuid(), nullable=True), + sa.Column('hot_water_cylinder_id', sa.Uuid(), nullable=True), + sa.Column('solar_water_heating_id', sa.Uuid(), nullable=True), + sa.Column('shower_and_baths_id', sa.Uuid(), nullable=True), + sa.Column('flue_gas_heat_recovery_system_id', sa.Uuid(), nullable=True), + sa.Column('photovoltaic_panel_id', sa.Uuid(), nullable=True), + sa.Column('wind_turbine_id', sa.Uuid(), nullable=True), + sa.Column('other_details_id', sa.Uuid(), nullable=True), + sa.Column('main_heating_id', sa.Uuid(), nullable=True), + sa.Column('main_heating2_id', sa.Uuid(), nullable=True), + sa.Column('secondary_heating_type_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['door_id'], ['door.id'], ), + sa.ForeignKeyConstraint(['ex1_property_id'], ['propertydetail.id'], ), + sa.ForeignKeyConstraint(['ex2_property_id'], ['propertydetail.id'], ), + sa.ForeignKeyConstraint(['ex3_property_id'], ['propertydetail.id'], ), + sa.ForeignKeyConstraint(['ex4_property_id'], ['propertydetail.id'], ), + sa.ForeignKeyConstraint(['flue_gas_heat_recovery_system_id'], ['fluegasheatrecoverysystem.id'], ), + sa.ForeignKeyConstraint(['hot_water_cylinder_id'], ['hotwatercylinder.id'], ), + sa.ForeignKeyConstraint(['lighting_id'], ['lighting.id'], ), + sa.ForeignKeyConstraint(['main_heating2_id'], ['heating.id'], ), + sa.ForeignKeyConstraint(['main_heating_id'], ['heating.id'], ), + sa.ForeignKeyConstraint(['main_property_id'], ['propertydetail.id'], ), + sa.ForeignKeyConstraint(['other_details_id'], ['otherdetails.id'], ), + sa.ForeignKeyConstraint(['photovoltaic_panel_id'], ['photovoltaicpanel.id'], ), + sa.ForeignKeyConstraint(['secondary_heating_type_id'], ['heatingtype.id'], ), + sa.ForeignKeyConstraint(['shower_and_baths_id'], ['showerandbaths.id'], ), + sa.ForeignKeyConstraint(['solar_water_heating_id'], ['solarwaterheating.id'], ), + sa.ForeignKeyConstraint(['ventilation_and_cooling_id'], ['ventilationandcooling.id'], ), + sa.ForeignKeyConstraint(['water_heating_id'], ['waterheating.id'], ), + sa.ForeignKeyConstraint(['wind_turbine_id'], ['windturbine.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('windows', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('glazing_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('area_m2', sa.Float(), nullable=False), + sa.Column('roof_window', sa.Boolean(), nullable=False), + sa.Column('orientation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('u_value_w_m2_k', sa.Integer(), nullable=False), + sa.Column('g_value', sa.Integer(), nullable=False), + sa.Column('property_detail_id', sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('windows') + op.drop_table('propertydescription') + op.drop_table('presitenote') + op.drop_table('dimension') + op.drop_table('propertydetail') + op.drop_table('heating') + op.drop_table('assessorinfo') + op.drop_table('windturbine') + op.drop_table('waterheating') + op.drop_table('walls') + op.drop_table('ventilationandcooling') + op.drop_table('solarwaterheating') + op.drop_table('showerandbaths') + op.drop_table('roofs') + op.drop_table('presitenotessummaryinfo') + op.drop_table('photovoltaicpanel') + op.drop_table('otherdetails') + op.drop_table('lighting') + op.drop_table('insulation') + op.drop_table('hotwatercylinder') + op.drop_table('heatingtype') + op.drop_table('heatingsystemcontrols') + op.drop_table('fluegasheatrecoverysystem') + op.drop_table('floors') + op.drop_table('door') + op.drop_table('companyinfo') + # ### end Alembic commands ### diff --git a/alembic/versions/e351a98aba54_init_some_tables.py b/alembic/versions/e351a98aba54_init_some_tables.py deleted file mode 100644 index 43be0da..0000000 --- a/alembic/versions/e351a98aba54_init_some_tables.py +++ /dev/null @@ -1,117 +0,0 @@ -"""init some tables - -Revision ID: e351a98aba54 -Revises: -Create Date: 2025-05-08 06:52:41.139547 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -# revision identifiers, used by Alembic. -revision: str = 'e351a98aba54' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('companyinfo', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('trading_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('post_code', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('fax_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('related_party_disclosure', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('heatingsystemcontrols', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('control_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('flue_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('fan_assisted_flue', sa.Boolean(), nullable=False), - sa.Column('heat_emitter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('mains_gas_available', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('heatingtype', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('presitenotessummaryinfo', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('reference_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('epc_language', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('region', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('town', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('county', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('property_tenure', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('transaction_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('inspection_date', sa.DateTime(), nullable=False), - sa.Column('current_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('potential_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('current_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('potential_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('current_annual_emissions', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('current_annual_emission_including_0925_multiplayer', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('current_annual_energy_costs', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('assessorinfo', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('accreditation_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('phone_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('email_address', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('company_id', sa.Uuid(), nullable=False), - sa.ForeignKeyConstraint(['company_id'], ['companyinfo.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('heating', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('heating_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('efficiency_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('heating_fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('brand_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('model_qualifer', sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column('sap_2009_table', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('percentage_of_heated_floor_area_served', sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column('controls_id', sa.Uuid(), nullable=False), - sa.ForeignKeyConstraint(['controls_id'], ['heatingsystemcontrols.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('presitenote', - sa.Column('id', sa.Uuid(), nullable=False), - sa.Column('summary_info_id', sa.Uuid(), nullable=False), - sa.Column('assessor_id', sa.Uuid(), nullable=False), - sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ), - sa.ForeignKeyConstraint(['summary_info_id'], ['presitenotessummaryinfo.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - """Downgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('presitenote') - op.drop_table('heating') - op.drop_table('assessorinfo') - op.drop_table('presitenotessummaryinfo') - op.drop_table('heatingtype') - op.drop_table('heatingsystemcontrols') - op.drop_table('companyinfo') - # ### end Alembic commands ### diff --git a/etl/load/preSiteNoteTypes.py b/etl/load/preSiteNoteTypes.py index 5ce4314..9a85916 100644 --- a/etl/load/preSiteNoteTypes.py +++ b/etl/load/preSiteNoteTypes.py @@ -8,11 +8,27 @@ from sqlalchemy.dialects.postgresql import UUID class BaseModel(SQLModel): - id: uuid.UUID = Field( - default_factory=uuid.uuid4, - sa_column=Column(UUID(as_uuid=True), primary_key=True) + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + +# One class to rule them all +class PreSiteNote(BaseModel, table=True): + # Summary Info + summary_info_id: uuid.UUID = Field( + foreign_key="presitenotessummaryinfo.id", + nullable=False ) + summary_info: Optional["PreSiteNotesSummaryInfo"] = Relationship(back_populates="pre_site_notes") + + + # Assessor Info + assessor_id: uuid.UUID = Field( + foreign_key="assessorinfo.id", + nullable=False + ) + + assessor: Optional["AssessorInfo"] = Relationship(back_populates="pre_site_notes") + class Dimension(BaseModel, table=True): floor_area_m2: float @@ -20,6 +36,7 @@ class Dimension(BaseModel, table=True): loss_perimeter_m: float party_wall_length_m: float property_detail_id: Optional[uuid.UUID] = Field(default=None, foreign_key="propertydetail.id") + property_detail: Optional["PropertyDetail"] = Relationship(back_populates="dimensions") class Walls(BaseModel, table=True): @@ -57,6 +74,8 @@ class Windows(BaseModel, table=True): u_value_w_m2_k: int g_value: int property_detail_id: Optional[uuid.UUID] = Field(default=None, foreign_key="propertydetail.id") + property_detail: Optional["PropertyDetail"] = Relationship(back_populates="windows") + class PropertyDetail(BaseModel, table=True): @@ -69,8 +88,6 @@ class PropertyDetail(BaseModel, table=True): windows: List[Windows] = Relationship(back_populates="property_detail") -Dimension.property_detail = Relationship(back_populates="dimensions") -Windows.property_detail = Relationship(back_populates="windows") class Door(BaseModel, table=True): @@ -201,8 +218,14 @@ class AssessorInfo(BaseModel, table=True): phone_number: Optional[str] = None email_address: Optional[EmailStr] = None + company_id: Optional[uuid.UUID] = Field(default=None, foreign_key="companyinfo.id") + company: Optional["CompanyInfo"] = Relationship(back_populates="assessors") -class SurverySummaryInfo(BaseModel, table=True): + pre_site_notes: List[PreSiteNote] = Relationship(back_populates="assessor") + + + +class PreSiteNotesSummaryInfo(BaseModel, table=True): reference_number: str epc_language: str uprn: Optional[str] = "" @@ -222,6 +245,7 @@ class SurverySummaryInfo(BaseModel, table=True): current_annual_emission_including_0925_multiplayer: str current_annual_energy_costs: str + pre_site_notes: List[PreSiteNote] = Relationship(back_populates="summary_info") class CompanyInfo(BaseModel, table=True): trading_name: str @@ -229,6 +253,13 @@ class CompanyInfo(BaseModel, table=True): fax_number: Optional[str] = None related_party_disclosure: Optional[str] = None + assessors: List[AssessorInfo] = Relationship(back_populates="company") + + class Insulation(BaseModel, table=True): type: str + + + +PreSiteNote.update_forward_refs() diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 44eda12..3a54544 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,9 +1,10 @@ from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.pdfReader.reportType import ReportType import math -from etl.transform.preSiteNoteTypes import ( +from etl.load.preSiteNoteTypes import ( AssessorInfo, CompanyInfo, - PreSiteNotesSummaryInfo, PreSiteNote, + PreSiteNotesSummaryInfo, + PreSiteNote, PropertyDescription, Dimension, HeatingType, Heating, HeatingSystemControls, OtherDetails, WindTurbine, PhotovoltaicPanel, FlueGasHeatRecoverySystem, ShowerAndBaths, SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling, @@ -64,7 +65,12 @@ class surveyedDataProcessor(): property_des = self.pre_site_note.property_description.model_dump() # Seconday Heating - secondary_heating = self.get_attribute_and_load(self.pre_site_note.property_description, "secondaryHeatingType", HeatingType, db_session) + secondary_heating = self.get_attribute_and_load( + self.pre_site_note.property_description, + "secondaryHeatingType", + HeatingType, + db_session + ) # main heating 2 and main heating 2 controls mainheating2 = None @@ -198,7 +204,19 @@ class surveyedDataProcessor(): roof = self.get_attribute_and_load(obj, "roof", Roofs, db_session) floor = None if check_if_attribute_exists(obj, "floor"): - roof = self.get_attribute_and_load(obj, "floor", Floors, db_session) + floor = self.get_attribute_and_load(obj, "floor", Floors, db_session) + + property_detail = self.upsert_record( + db_session=db_session, + model_class=PropertyDetail, + data_dict={ + "age_band": obj.age_band, + "floor_id": floor.id if floor else None, + "roof_id": roof.id if roof else None, + "wall_id": wall.id if wall else None, + }, + lookup_field=None, + ) dimensions = [] if check_if_attribute_exists(obj, "dimensions"): @@ -210,6 +228,7 @@ class surveyedDataProcessor(): model_class=Dimension, data_dict=data, lookup_field=None, + additional_fields={"property_detail_id": property_detail.id}, ) dimensions.append(dimension.id) @@ -223,22 +242,9 @@ class surveyedDataProcessor(): model_class=Windows, data_dict=data, lookup_field=None, + additional_fields={"property_detail_id": property_detail.id}, ) - windows.append(windows.id) - - property_detail = self.upsert_record( - db_session=db_session, - model_class=PropertyDetail, - data = obj.model_dump(), - additional_fields={ - "windows_id": windows, - "dimensions_id": dimensions, - "floors_id": floor.id if floor else None, - "roof_id": roof.id if roof else None, - "wall_id": wall.id if wall else None, - - } - ) + windows.append(window.id) return property_detail @@ -284,20 +290,6 @@ class surveyedDataProcessor(): update_if_exists: bool = False, additional_fields: dict = None ): - """ - Upserts a SQLAlchemy model instance based on a lookup field. - - Args: - db_session: SQLAlchemy session. - model_class: SQLAlchemy model/table class. - data_dict: Dictionary of data for creating the model. - lookup_field: Unique identifier field name (str). - update_if_exists: Whether to update existing record or not. - additional_fields: Extra fields to inject (e.g., foreign keys). - - Returns: - SQLAlchemy model instance. - """ clean_data = data_dict # Merge additional fields if provided @@ -324,6 +316,7 @@ class surveyedDataProcessor(): db_session.commit() return new_record + def load_assessor_table(self, db_session): company = self.load_company_table(db_session) @@ -456,6 +449,4 @@ class surveyedDataProcessor(): def get_current_sap_score(self): score_list = self.pre_site_note.survey_information.current_sap.split(" ") score = int(score_list[1]) - return score - - + return score \ No newline at end of file diff --git a/etl/transform/preSiteNoteTypes.py b/etl/transform/preSiteNoteTypes.py index 7e25682..3b9df77 100644 --- a/etl/transform/preSiteNoteTypes.py +++ b/etl/transform/preSiteNoteTypes.py @@ -7,10 +7,7 @@ from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID class BaseModel(SQLModel): - id: uuid.UUID = Field( - default_factory=uuid.uuid4, - sa_column=Column(UUID(as_uuid=True), primary_key=True) - ) + pass class Dimension(BaseModel): floor_area_m2: float @@ -30,7 +27,7 @@ class CompanyInfo(BaseModel): return None return v -class SurverySummaryInfo(BaseModel): +class PreSiteNotesSummaryInfo(BaseModel): reference_number: str epc_language: str uprn: Optional[str] = "" @@ -79,7 +76,7 @@ class Door(BaseModel): no_of_insulated_doors: int u_value_w_m2_k: Optional[str] -class AssessorInfo(BaseModel, table=True): +class AssessorInfo(BaseModel): accreditation_number: str name: str phone_number: Optional[str] = None diff --git a/migration_db.sh b/migration_db.sh index d029b60..deabd64 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,2 +1,3 @@ -#poetry run alembic revision --autogenerate -m "init some tables" +#poetry run alembic revision --autogenerate -m "Initial migration" + poetry run alembic upgrade head