mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
save work
This commit is contained in:
Jun-te Kim
parent
4c9bcada13
commit
24bae95458
5 changed files with 153 additions and 12 deletions
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
|
|
@ -7,7 +7,7 @@
|
|||
"bash": {
|
||||
"path": "/bin/bash"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Hot reload setting that needs to be in user settings
|
||||
// "jupyter.runStartupCommands": [
|
||||
|
|
|
|||
|
|
@ -97,7 +97,9 @@ class HubspotTodb():
|
|||
# Loads the pre site summary information
|
||||
summary_info = surveyedData.load_pre_site_notes_summary_table(db_session)
|
||||
|
||||
# Creates the a final pre site note table that links all information
|
||||
presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info)
|
||||
property_description = surveyedData.load_property_description(db_session)
|
||||
|
||||
# Creates the a final pre site note table that links all information
|
||||
presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description)
|
||||
|
||||
|
||||
surveyedData.load_property_description(db_session)
|
||||
|
|
@ -10,9 +10,9 @@ from sqlalchemy.dialects.postgresql import UUID
|
|||
class BaseModel(SQLModel):
|
||||
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
|
||||
|
||||
# One class to rule them all
|
||||
|
||||
class PreSiteNote(BaseModel, table=True):
|
||||
# Summary Info
|
||||
# One class to rule them all
|
||||
summary_info_id: uuid.UUID = Field(
|
||||
foreign_key="presitenotessummaryinfo.id",
|
||||
nullable=False
|
||||
|
|
@ -29,6 +29,13 @@ class PreSiteNote(BaseModel, table=True):
|
|||
|
||||
assessor: Optional["AssessorInfo"] = Relationship(back_populates="pre_site_notes")
|
||||
|
||||
pre_site_note_description_id: uuid.UUID = Field(
|
||||
foreign_key="propertydescription.id",
|
||||
nullable=True
|
||||
)
|
||||
|
||||
pre_site_note_description: Optional["PropertyDescription"] = Relationship(back_populates="pre_site_notes")
|
||||
|
||||
|
||||
class Dimension(BaseModel, table=True):
|
||||
floor_area_m2: float
|
||||
|
|
@ -84,6 +91,7 @@ class PropertyDetail(BaseModel, table=True):
|
|||
roof_id: Optional[uuid.UUID] = Field(default=None, foreign_key="roofs.id")
|
||||
floor_id: Optional[uuid.UUID] = Field(default=None, foreign_key="floors.id")
|
||||
|
||||
# Relationships
|
||||
dimensions: List[Dimension] = Relationship(back_populates="property_detail")
|
||||
windows: List[Windows] = Relationship(back_populates="property_detail")
|
||||
|
||||
|
|
@ -95,17 +103,24 @@ class Door(BaseModel, table=True):
|
|||
no_of_insulated_doors: int
|
||||
u_value_w_m2_k: Optional[str]
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="door")
|
||||
|
||||
class VentilationAndCooling(BaseModel, table=True):
|
||||
no_of_open_fireplaces: int
|
||||
ventilation_type: str
|
||||
space_cooling_system_present: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="ventilation_and_cooling")
|
||||
|
||||
|
||||
|
||||
class Lighting(BaseModel, table=True):
|
||||
total_no_of_light_fittings: int
|
||||
total_no_of_lel_fittings: int
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="lighting")
|
||||
|
||||
|
||||
|
||||
class HeatingSystemControls(BaseModel, table=True):
|
||||
control_type: str
|
||||
|
|
@ -128,16 +143,29 @@ class Heating(BaseModel, table=True):
|
|||
percentage_of_heated_floor_area_served: Optional[str] = ""
|
||||
controls_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingsystemcontrols.id")
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(
|
||||
back_populates="main_heating", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"}
|
||||
)
|
||||
property_description2: Optional["PropertyDescription"] = Relationship(
|
||||
back_populates="main_heating2", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"}
|
||||
)
|
||||
|
||||
|
||||
|
||||
class HeatingType(BaseModel, table=True):
|
||||
heating_type: str
|
||||
fuel_type: str
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="secondary_heating_type")
|
||||
|
||||
|
||||
class WaterHeating(BaseModel, table=True):
|
||||
heating_type: str
|
||||
fuel_type: str
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="water_heating")
|
||||
|
||||
|
||||
|
||||
class HotWaterCylinder(BaseModel, table=True):
|
||||
volume: str
|
||||
|
|
@ -145,34 +173,55 @@ class HotWaterCylinder(BaseModel, table=True):
|
|||
insulation_thickness: str
|
||||
thermostat: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="hot_water_cylinder")
|
||||
|
||||
|
||||
|
||||
class SolarWaterHeating(BaseModel, table=True):
|
||||
solar_water_heating_details_known: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="solar_water_heating")
|
||||
|
||||
|
||||
|
||||
class ShowerAndBaths(BaseModel, table=True):
|
||||
no_of_rooms_with_baths_and_or_shower: int
|
||||
no_of_rooms_with_mixer_shower_and_no_baths: int
|
||||
no_of_rooms_with_mixer_shower_and_baths: int
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="shower_and_baths")
|
||||
|
||||
|
||||
|
||||
class FlueGasHeatRecoverySystem(BaseModel, table=True):
|
||||
fghrs_present: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="flue_gas_heat_recovery_system")
|
||||
|
||||
|
||||
|
||||
class PhotovoltaicPanel(BaseModel, table=True):
|
||||
pvs_are_connected_to_dwelling_electricity_meter: bool
|
||||
percentage_of_external_roof_area_with_pvs: str
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="photovoltaic_panel")
|
||||
|
||||
|
||||
|
||||
class WindTurbine(BaseModel, table=True):
|
||||
wind_turbine: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="wind_turbine")
|
||||
|
||||
|
||||
|
||||
class OtherDetails(BaseModel, table=True):
|
||||
electricity_meter_type: str
|
||||
main_gas_avalible: bool
|
||||
|
||||
property_description: Optional["PropertyDescription"] = Relationship(back_populates="other_details")
|
||||
|
||||
|
||||
|
||||
class PropertyDescription(BaseModel, table=True):
|
||||
built_form: str
|
||||
|
|
@ -211,6 +260,30 @@ class PropertyDescription(BaseModel, table=True):
|
|||
main_heating2_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heating.id")
|
||||
secondary_heating_type_id: Optional[uuid.UUID] = Field(default=None, foreign_key="heatingtype.id")
|
||||
|
||||
# Relationships
|
||||
main_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_property_id]"})
|
||||
ex1_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex1_property_id]"})
|
||||
ex2_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex2_property_id]"})
|
||||
ex3_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex3_property_id]"})
|
||||
ex4_property: Optional["PropertyDetail"] = Relationship(sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.ex4_property_id]"})
|
||||
|
||||
# Related Models
|
||||
door: Optional["Door"] = Relationship(back_populates="property_description")
|
||||
ventilation_and_cooling: Optional["VentilationAndCooling"] = Relationship(back_populates="property_description")
|
||||
lighting: Optional["Lighting"] = Relationship(back_populates="property_description")
|
||||
water_heating: Optional["WaterHeating"] = Relationship(back_populates="property_description")
|
||||
hot_water_cylinder: Optional["HotWaterCylinder"] = Relationship(back_populates="property_description")
|
||||
solar_water_heating: Optional["SolarWaterHeating"] = Relationship(back_populates="property_description")
|
||||
shower_and_baths: Optional["ShowerAndBaths"] = Relationship(back_populates="property_description")
|
||||
flue_gas_heat_recovery_system: Optional["FlueGasHeatRecoverySystem"] = Relationship(back_populates="property_description")
|
||||
photovoltaic_panel: Optional["PhotovoltaicPanel"] = Relationship(back_populates="property_description")
|
||||
wind_turbine: Optional["WindTurbine"] = Relationship(back_populates="property_description")
|
||||
other_details: Optional["OtherDetails"] = Relationship(back_populates="property_description")
|
||||
main_heating: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating_id]"})
|
||||
main_heating2: Optional["Heating"] = Relationship(back_populates="property_description", sa_relationship_kwargs={"foreign_keys": "[PropertyDescription.main_heating2_id]"})
|
||||
secondary_heating_type: Optional["HeatingType"] = Relationship(back_populates="property_description")
|
||||
|
||||
pre_site_notes: Optional["PreSiteNote"] = Relationship(back_populates="pre_site_note_description")
|
||||
|
||||
class AssessorInfo(BaseModel, table=True):
|
||||
accreditation_number: str
|
||||
|
|
@ -221,7 +294,7 @@ class AssessorInfo(BaseModel, table=True):
|
|||
company_id: Optional[uuid.UUID] = Field(default=None, foreign_key="companyinfo.id")
|
||||
company: Optional["CompanyInfo"] = Relationship(back_populates="assessors")
|
||||
|
||||
pre_site_notes: List[PreSiteNote] = Relationship(back_populates="assessor")
|
||||
pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="assessor")
|
||||
|
||||
|
||||
|
||||
|
|
@ -245,7 +318,7 @@ class PreSiteNotesSummaryInfo(BaseModel, table=True):
|
|||
current_annual_emission_including_0925_multiplayer: str
|
||||
current_annual_energy_costs: str
|
||||
|
||||
pre_site_notes: List[PreSiteNote] = Relationship(back_populates="summary_info")
|
||||
pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="summary_info")
|
||||
|
||||
class CompanyInfo(BaseModel, table=True):
|
||||
trading_name: str
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ class surveyedDataProcessor():
|
|||
)
|
||||
|
||||
# hotwatercycling
|
||||
hot_water_cyclinder = self.get_attribute_and_load(
|
||||
hot_water_cylinder = self.get_attribute_and_load(
|
||||
self.pre_site_note.property_description,
|
||||
"hotWaterCylinder",
|
||||
HotWaterCylinder,
|
||||
|
|
@ -250,7 +250,57 @@ class surveyedDataProcessor():
|
|||
|
||||
# main_property
|
||||
main_property = upload_property_detail("main_property")
|
||||
print(main_property.id)
|
||||
ex1_property = upload_property_detail("ex1_property")
|
||||
ex2_property = upload_property_detail("ex2_property")
|
||||
ex3_property = upload_property_detail("ex3_property")
|
||||
ex4_property = upload_property_detail("ex4_property")
|
||||
|
||||
data = self.pre_site_note.property_description.model_dump()
|
||||
def remove_dicts_and_lists(data):
|
||||
if isinstance(data, dict):
|
||||
# Create a new dict with only primitive types (ignore dicts/lists)
|
||||
return {
|
||||
k: remove_dicts_and_lists(v)
|
||||
for k, v in data.items()
|
||||
if not isinstance(v, (dict, list))
|
||||
}
|
||||
elif isinstance(data, list):
|
||||
# Remove lists entirely
|
||||
return None
|
||||
else:
|
||||
return data
|
||||
data = remove_dicts_and_lists(data)
|
||||
property_description = self.upsert_record(
|
||||
db_session=db_session,
|
||||
model_class=PropertyDescription,
|
||||
data_dict=data,
|
||||
lookup_field=None,
|
||||
additional_fields={
|
||||
"main_heating_id": mainheating.id if mainheating else None,
|
||||
"main_heating_controls_id": mainheatingcontrols.id if mainheatingcontrols else None,
|
||||
"main_heating2_id": mainheating2.id if mainheating2 else None,
|
||||
"main_heating2_controls_id": mainheating2controls.id if mainheating2controls else None,
|
||||
"secondary_heating_type_id": secondary_heating.id if secondary_heating else None,
|
||||
"other_details_id": otherDetails.id if otherDetails else None,
|
||||
"wind_turbine_id": windTurbine.id if windTurbine else None,
|
||||
"photovoltaic_panel_id": photo_volatic_panel.id if photo_volatic_panel else None,
|
||||
"flue_gas_heat_recovery_system_id": flue_gas_heat_recovery_system.id if flue_gas_heat_recovery_system else None,
|
||||
"shower_and_baths_id": shower_and_baths.id if shower_and_baths else None,
|
||||
"solar_water_heating_id": solar_water_heating.id if solar_water_heating else None,
|
||||
"hot_water_cylinder_id": hot_water_cylinder.id if hot_water_cylinder else None,
|
||||
"water_heating_id": water_heating.id if water_heating else None,
|
||||
"lighting_id": lighting.id if lighting else None,
|
||||
"ventilation_and_cooling_id": ventilation_and_cooling.id if ventilation_and_cooling else None,
|
||||
"door_id": door.id if door else None,
|
||||
"main_property_id": main_property.id if main_property else None,
|
||||
"ex1_property_id": ex1_property.id if ex1_property else None,
|
||||
"ex2_property_id": ex2_property.id if ex2_property else None,
|
||||
"ex3_property_id": ex3_property.id if ex3_property else None,
|
||||
"ex4_property_id": ex4_property.id if ex4_property else None,
|
||||
}
|
||||
)
|
||||
print(f"property_description {property_description}")
|
||||
return property_description
|
||||
|
||||
|
||||
|
||||
|
|
@ -268,11 +318,13 @@ class surveyedDataProcessor():
|
|||
self,
|
||||
db_session,
|
||||
assessor,
|
||||
summary_info
|
||||
summary_info,
|
||||
pre_site_note_description,
|
||||
):
|
||||
preSiteNote = PreSiteNote(
|
||||
summary_info_id=summary_info.id,
|
||||
assessor_id=assessor.id,
|
||||
pre_site_note_description_id=pre_site_note_description.id,
|
||||
)
|
||||
db_session.add(preSiteNote)
|
||||
db_session.commit()
|
||||
|
|
@ -301,20 +353,34 @@ class surveyedDataProcessor():
|
|||
if not lookup_value:
|
||||
raise ValueError(f"Missing lookup field '{lookup_field}' in data.")
|
||||
|
||||
# Try to find existing record
|
||||
existing_record = db_session.query(model_class).filter(
|
||||
getattr(model_class, lookup_field) == lookup_value
|
||||
).first()
|
||||
|
||||
if existing_record:
|
||||
# Update existing record if update_if_exists is True
|
||||
if update_if_exists:
|
||||
for key, value in clean_data.items():
|
||||
setattr(existing_record, key, value)
|
||||
db_session.commit()
|
||||
return existing_record
|
||||
|
||||
# Filter out invalid fields that don't exist in the model class
|
||||
valid_fields = [field for field in clean_data if hasattr(model_class, field)]
|
||||
clean_data = {field: clean_data[field] for field in valid_fields}
|
||||
|
||||
print(f'clean data is {clean_data}')
|
||||
|
||||
# Handle Pydantic models (with model_validate or parse_obj)
|
||||
new_record = model_class(**clean_data)
|
||||
|
||||
# Add the new record to the session and commit
|
||||
db_session.add(new_record)
|
||||
db_session.commit()
|
||||
|
||||
return new_record
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
#poetry run alembic revision --autogenerate -m "Initial migration"
|
||||
#poetry run alembic revision --autogenerate -m "mistake on foreign key"
|
||||
|
||||
poetry run alembic upgrade head
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue