From 2612a1f1dcbcc0cc5d43ac4929565cc9d2f42e1c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 12 May 2025 10:21:18 +0000 Subject: [PATCH] save current pogress --- .vscode/settings.json | 8 +++- etl/osmosis_google_maps_.py | 7 +++- etl/surveyedData/surveryedData.py | 62 ++++++++++++++++++++++++++++++- etl/transform/preSiteNoteTypes.py | 42 ++++++++++++++++----- 4 files changed, 105 insertions(+), 14 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 49f0804..9158a79 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,13 @@ { "jupyter.interactiveWindow.textEditor.executeSelection": true, "python.REPL.sendToNativeREPL": true, - "notebook.output.scrolling": true + "notebook.output.scrolling": true, + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + } // Hot reload setting that needs to be in user settings // "jupyter.runStartupCommands": [ diff --git a/etl/osmosis_google_maps_.py b/etl/osmosis_google_maps_.py index ccc1d16..c8c2654 100644 --- a/etl/osmosis_google_maps_.py +++ b/etl/osmosis_google_maps_.py @@ -1,10 +1,13 @@ from monday import MondayClient import json import requests +import time +from tqdm import tqdm + + board_id = "8829428746" monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -import time def get_all_items(board_id, monday): # Parameters @@ -75,7 +78,7 @@ if not postcode_col_id or not location_col_id: raise Exception("Could not find 'postcode' or 'location' columns") items = get_all_items(board_id, monday) -for item in items: +for item in tqdm(items): item_name = item["name"] item_id = item["id"] diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index ea66566..44eda12 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -7,7 +7,7 @@ from etl.transform.preSiteNoteTypes import ( PropertyDescription, Dimension, HeatingType, Heating, HeatingSystemControls, OtherDetails, WindTurbine, PhotovoltaicPanel, FlueGasHeatRecoverySystem, ShowerAndBaths, SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling, - Door, + Door, Walls, Roofs, Floors, PropertyDetail, Windows ) import uuid @@ -187,6 +187,66 @@ class surveyedDataProcessor(): db_session, ) + def upload_property_detail(property_part="main_property"): + if check_if_attribute_exists(self.pre_site_note.property_description, property_part): + wall = None + obj = getattr(self.pre_site_note.property_description, property_part) + if check_if_attribute_exists(obj, "wall"): + wall = self.get_attribute_and_load(obj, "wall", Walls, db_session) + roof = None + if check_if_attribute_exists(obj, "roof"): + roof = self.get_attribute_and_load(obj, "roof", Roofs, db_session) + floor = None + if check_if_attribute_exists(obj, "floor"): + roof = self.get_attribute_and_load(obj, "floor", Floors, db_session) + + dimensions = [] + if check_if_attribute_exists(obj, "dimensions"): + dimension_obj = getattr(obj, "dimensions") + for eachDimension in dimension_obj: + data = eachDimension.model_dump() + dimension = self.upsert_record( + db_session=db_session, + model_class=Dimension, + data_dict=data, + lookup_field=None, + ) + dimensions.append(dimension.id) + + windows = [] + if check_if_attribute_exists(obj, "windows"): + windows_obj = getattr(obj, "windows") + for eachWindow in windows_obj: + data = eachWindow.model_dump() + window = self.upsert_record( + db_session=db_session, + model_class=Windows, + data_dict=data, + lookup_field=None, + ) + windows.append(windows.id) + + property_detail = self.upsert_record( + db_session=db_session, + model_class=PropertyDetail, + data = obj.model_dump(), + additional_fields={ + "windows_id": windows, + "dimensions_id": dimensions, + "floors_id": floor.id if floor else None, + "roof_id": roof.id if roof else None, + "wall_id": wall.id if wall else None, + + } + ) + + return property_detail + + # main_property + main_property = upload_property_detail("main_property") + print(main_property.id) + + def load_company_table(self, db_session): company_data = self.pre_site_note.company_information.model_dump() diff --git a/etl/transform/preSiteNoteTypes.py b/etl/transform/preSiteNoteTypes.py index 231971a..f77a2d7 100644 --- a/etl/transform/preSiteNoteTypes.py +++ b/etl/transform/preSiteNoteTypes.py @@ -13,7 +13,7 @@ class BaseModel(SQLModel): primary_key=True, ) -class Dimension(BaseModel): +class Dimension(BaseModel, table=True): floor_area_m2: float room_height_m: float loss_perimeter_m: float @@ -59,7 +59,7 @@ class PreSiteNotesSummaryInfo(BaseModel, table=True): ) -class Walls(BaseModel): +class Walls(BaseModel, table=True): construction: str insulation: str insulation_thickness_mm: str @@ -70,13 +70,13 @@ class Walls(BaseModel): dry_lining: bool alternative_wall_present: bool -class Roofs(BaseModel): +class Roofs(BaseModel, table=True): construction: str insulation_type: str insulation_thickness: str u_value_known: bool -class Floors(BaseModel): +class Floors(BaseModel, table=True): floor_type: str ground_floor_construction: str ground_floor_insulation_type: Optional[str] = "" @@ -172,7 +172,7 @@ class OtherDetails(BaseModel, table=True): electricity_meter_type: str main_gas_avalible: bool -class Windows(BaseModel): +class Windows(BaseModel, table=True): glazing_type: str area_m2: float roof_window: bool @@ -180,14 +180,35 @@ class Windows(BaseModel): u_value_w_m2_k: int g_value: int -class PropertyDetail(BaseModel): + property_db: Optional["PropertyDetail"] = Relationship(back_populates="windows_db") + + +class PropertyDetail(BaseModel, table=True): # change this name to build parts age_band: str - dimensions: List[Dimension] = [] - wall: Optional[Walls] = None - roof: Optional[Roofs] = None - floor: Optional[Floors] = None windows: Optional[List[Windows]] = [] + windows_db: List[Windows] = Relationship(back_populates="property_db") + + wall: Optional[Walls] = None + wall_id: Optional[uuid.UUID] = Field( + default=None, + foreign_key="walls.id" + ) + roof: Optional[Roofs] = None + roof_id: Optional[uuid.UUID] = Field( + default=None, + foreign_key="roofs.id" + ) + floor: Optional[Floors] = None + floor_id: Optional[uuid.UUID] = Field( + default=None, + foreign_key="floors.id" + ) + dimensions: List[Dimension] = [] + dimensions_id: Optional[List[uuid.UUID]] = Field( + default=None, + foreign_key="dimensions.id" + ) class PropertyDescription(BaseModel): @@ -212,6 +233,7 @@ class PropertyDescription(BaseModel): ex2_property: Optional[PropertyDetail] = None ex3_property: Optional[PropertyDetail] = None ex4_property: Optional[PropertyDetail] = None + # Do walls next, check if variable is real similar to main heating # Door door_id: Optional[uuid.UUID] = Field(