From 24bae95458522caaf8d068ec256002d9ee0a608c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 12 May 2025 16:28:42 +0000 Subject: [PATCH] save work --- .vscode/settings.json | 2 +- etl/db/hubSpotLoad.py | 8 +-- etl/load/preSiteNoteTypes.py | 81 +++++++++++++++++++++++++++++-- etl/surveyedData/surveryedData.py | 72 +++++++++++++++++++++++++-- migration_db.sh | 2 +- 5 files changed, 153 insertions(+), 12 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9158a79..ea20d57 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "bash": { "path": "/bin/bash" } - } + }, // Hot reload setting that needs to be in user settings // "jupyter.runStartupCommands": [ diff --git a/etl/db/hubSpotLoad.py b/etl/db/hubSpotLoad.py index ed21b8a..a8f0680 100644 --- a/etl/db/hubSpotLoad.py +++ b/etl/db/hubSpotLoad.py @@ -97,7 +97,9 @@ class HubspotTodb(): # Loads the pre site summary information summary_info = surveyedData.load_pre_site_notes_summary_table(db_session) - # Creates the a final pre site note table that links all information - presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info) + property_description = surveyedData.load_property_description(db_session) + + # Creates the a final pre site note table that links all information + presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description) + - surveyedData.load_property_description(db_session) \ No newline at end of file diff --git a/etl/load/preSiteNoteTypes.py b/etl/load/preSiteNoteTypes.py index 9a85916..87cffe5 100644 --- a/etl/load/preSiteNoteTypes.py +++ b/etl/load/preSiteNoteTypes.py @@ -10,9 +10,9 @@ from sqlalchemy.dialects.postgresql import UUID class BaseModel(SQLModel): id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) -# One class to rule them all + class PreSiteNote(BaseModel, table=True): - # Summary Info +# One class to rule them all summary_info_id: uuid.UUID = Field( foreign_key="presitenotessummaryinfo.id", nullable=False @@ -29,6 +29,13 @@ class PreSiteNote(BaseModel, table=True): assessor: Optional["AssessorInfo"] = Relationship(back_populates="pre_site_notes") + pre_site_note_description_id: uuid.UUID = Field( + foreign_key="propertydescription.id", + nullable=True + ) + + pre_site_note_description: Optional["PropertyDescription"] = Relationship(back_populates="pre_site_notes") + class Dimension(BaseModel, table=True): floor_area_m2: float @@ -84,6 +91,7 @@ class PropertyDetail(BaseModel, table=True): roof_id: Optional[uuid.UUID] = Field(default=None, foreign_key="roofs.id") floor_id: Optional[uuid.UUID] = Field(default=None, foreign_key="floors.id") + # Relationships dimensions: List[Dimension] = Relationship(back_populates="property_detail") windows: List[Windows] = Relationship(back_populates="property_detail") @@ -95,17 +103,24 @@ class Door(BaseModel, table=True): no_of_insulated_doors: int u_value_w_m2_k: Optional[str] + property_description: Optional["PropertyDescription"] = Relationship(back_populates="door") class VentilationAndCooling(BaseModel, table=True): no_of_open_fireplaces: int ventilation_type: str space_cooling_system_present: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="ventilation_and_cooling") + + class Lighting(BaseModel, table=True): total_no_of_light_fittings: int total_no_of_lel_fittings: int + property_description: Optional["PropertyDescription"] = Relationship(back_populates="lighting") + + class HeatingSystemControls(BaseModel, table=True): control_type: str @@ -128,16 +143,29 @@ class Heating(BaseModel, table=True): percentage_of_heated_floor_area_served: Optional[str] = "" controls_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingsystemcontrols.id") + property_description: Optional["PropertyDescription"] = Relationship( + back_populates="main_heating", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"} + ) + property_description2: Optional["PropertyDescription"] = Relationship( + back_populates="main_heating2", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"} + ) + + class HeatingType(BaseModel, table=True): heating_type: str fuel_type: str + property_description: Optional["PropertyDescription"] = Relationship(back_populates="secondary_heating_type") + class WaterHeating(BaseModel, table=True): heating_type: str fuel_type: str + property_description: Optional["PropertyDescription"] = Relationship(back_populates="water_heating") + + class HotWaterCylinder(BaseModel, table=True): volume: str @@ -145,34 +173,55 @@ class HotWaterCylinder(BaseModel, table=True): insulation_thickness: str thermostat: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="hot_water_cylinder") + + class SolarWaterHeating(BaseModel, table=True): solar_water_heating_details_known: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="solar_water_heating") + + class ShowerAndBaths(BaseModel, table=True): no_of_rooms_with_baths_and_or_shower: int no_of_rooms_with_mixer_shower_and_no_baths: int no_of_rooms_with_mixer_shower_and_baths: int + property_description: Optional["PropertyDescription"] = Relationship(back_populates="shower_and_baths") + + class FlueGasHeatRecoverySystem(BaseModel, table=True): fghrs_present: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="flue_gas_heat_recovery_system") + + class PhotovoltaicPanel(BaseModel, table=True): pvs_are_connected_to_dwelling_electricity_meter: bool percentage_of_external_roof_area_with_pvs: str + property_description: Optional["PropertyDescription"] = Relationship(back_populates="photovoltaic_panel") + + class WindTurbine(BaseModel, table=True): wind_turbine: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="wind_turbine") + + class OtherDetails(BaseModel, table=True): electricity_meter_type: str main_gas_avalible: bool + property_description: Optional["PropertyDescription"] = Relationship(back_populates="other_details") + + class PropertyDescription(BaseModel, table=True): built_form: str @@ -211,6 +260,30 @@ class PropertyDescription(BaseModel, table=True): main_heating2_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heating.id") secondary_heating_type_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingtype.id") + # Relationships + main_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_property_id]"}) + ex1_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex1_property_id]"}) + ex2_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex2_property_id]"}) + ex3_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex3_property_id]"}) + ex4_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex4_property_id]"}) + + # Related Models + door: Optional["Door"] = Relationship(back_populates="property_description") + ventilation_and_cooling: Optional["VentilationAndCooling"] = Relationship(back_populates="property_description") + lighting: Optional["Lighting"] = Relationship(back_populates="property_description") + water_heating: Optional["WaterHeating"] = Relationship(back_populates="property_description") + hot_water_cylinder: Optional["HotWaterCylinder"] = Relationship(back_populates="property_description") + solar_water_heating: Optional["SolarWaterHeating"] = Relationship(back_populates="property_description") + shower_and_baths: Optional["ShowerAndBaths"] = Relationship(back_populates="property_description") + flue_gas_heat_recovery_system: Optional["FlueGasHeatRecoverySystem"] = Relationship(back_populates="property_description") + photovoltaic_panel: Optional["PhotovoltaicPanel"] = Relationship(back_populates="property_description") + wind_turbine: Optional["WindTurbine"] = Relationship(back_populates="property_description") + other_details: Optional["OtherDetails"] = Relationship(back_populates="property_description") + main_heating: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"}) + main_heating2: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"}) + secondary_heating_type: Optional["HeatingType"] = Relationship(back_populates="property_description") + + pre_site_notes: Optional["PreSiteNote"] = Relationship(back_populates="pre_site_note_description") class AssessorInfo(BaseModel, table=True): accreditation_number: str @@ -221,7 +294,7 @@ class AssessorInfo(BaseModel, table=True): company_id: Optional[uuid.UUID] = Field(default=None, foreign_key="companyinfo.id") company: Optional["CompanyInfo"] = Relationship(back_populates="assessors") - pre_site_notes: List[PreSiteNote] = Relationship(back_populates="assessor") + pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="assessor") @@ -245,7 +318,7 @@ class PreSiteNotesSummaryInfo(BaseModel, table=True): current_annual_emission_including_0925_multiplayer: str current_annual_energy_costs: str - pre_site_notes: List[PreSiteNote] = Relationship(back_populates="summary_info") + pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="summary_info") class CompanyInfo(BaseModel, table=True): trading_name: str diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 3a54544..29998ff 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -154,7 +154,7 @@ class surveyedDataProcessor(): ) # hotwatercycling - hot_water_cyclinder = self.get_attribute_and_load( + hot_water_cylinder = self.get_attribute_and_load( self.pre_site_note.property_description, "hotWaterCylinder", HotWaterCylinder, @@ -250,7 +250,57 @@ class surveyedDataProcessor(): # main_property main_property = upload_property_detail("main_property") - print(main_property.id) + ex1_property = upload_property_detail("ex1_property") + ex2_property = upload_property_detail("ex2_property") + ex3_property = upload_property_detail("ex3_property") + ex4_property = upload_property_detail("ex4_property") + + data = self.pre_site_note.property_description.model_dump() + def remove_dicts_and_lists(data): + if isinstance(data, dict): + # Create a new dict with only primitive types (ignore dicts/lists) + return { + k: remove_dicts_and_lists(v) + for k, v in data.items() + if not isinstance(v, (dict, list)) + } + elif isinstance(data, list): + # Remove lists entirely + return None + else: + return data + data = remove_dicts_and_lists(data) + property_description = self.upsert_record( + db_session=db_session, + model_class=PropertyDescription, + data_dict=data, + lookup_field=None, + additional_fields={ + "main_heating_id": mainheating.id if mainheating else None, + "main_heating_controls_id": mainheatingcontrols.id if mainheatingcontrols else None, + "main_heating2_id": mainheating2.id if mainheating2 else None, + "main_heating2_controls_id": mainheating2controls.id if mainheating2controls else None, + "secondary_heating_type_id": secondary_heating.id if secondary_heating else None, + "other_details_id": otherDetails.id if otherDetails else None, + "wind_turbine_id": windTurbine.id if windTurbine else None, + "photovoltaic_panel_id": photo_volatic_panel.id if photo_volatic_panel else None, + "flue_gas_heat_recovery_system_id": flue_gas_heat_recovery_system.id if flue_gas_heat_recovery_system else None, + "shower_and_baths_id": shower_and_baths.id if shower_and_baths else None, + "solar_water_heating_id": solar_water_heating.id if solar_water_heating else None, + "hot_water_cylinder_id": hot_water_cylinder.id if hot_water_cylinder else None, + "water_heating_id": water_heating.id if water_heating else None, + "lighting_id": lighting.id if lighting else None, + "ventilation_and_cooling_id": ventilation_and_cooling.id if ventilation_and_cooling else None, + "door_id": door.id if door else None, + "main_property_id": main_property.id if main_property else None, + "ex1_property_id": ex1_property.id if ex1_property else None, + "ex2_property_id": ex2_property.id if ex2_property else None, + "ex3_property_id": ex3_property.id if ex3_property else None, + "ex4_property_id": ex4_property.id if ex4_property else None, + } + ) + print(f"property_description {property_description}") + return property_description @@ -268,11 +318,13 @@ class surveyedDataProcessor(): self, db_session, assessor, - summary_info + summary_info, + pre_site_note_description, ): preSiteNote = PreSiteNote( summary_info_id=summary_info.id, assessor_id=assessor.id, + pre_site_note_description_id=pre_site_note_description.id, ) db_session.add(preSiteNote) db_session.commit() @@ -301,20 +353,34 @@ class surveyedDataProcessor(): if not lookup_value: raise ValueError(f"Missing lookup field '{lookup_field}' in data.") + # Try to find existing record existing_record = db_session.query(model_class).filter( getattr(model_class, lookup_field) == lookup_value ).first() if existing_record: + # Update existing record if update_if_exists is True if update_if_exists: for key, value in clean_data.items(): setattr(existing_record, key, value) db_session.commit() return existing_record + + # Filter out invalid fields that don't exist in the model class + valid_fields = [field for field in clean_data if hasattr(model_class, field)] + clean_data = {field: clean_data[field] for field in valid_fields} + + print(f'clean data is {clean_data}') + + # Handle Pydantic models (with model_validate or parse_obj) new_record = model_class(**clean_data) + + # Add the new record to the session and commit db_session.add(new_record) db_session.commit() + return new_record + diff --git a/migration_db.sh b/migration_db.sh index deabd64..059baa9 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,3 +1,3 @@ -#poetry run alembic revision --autogenerate -m "Initial migration" +#poetry run alembic revision --autogenerate -m "mistake on foreign key" poetry run alembic upgrade head