From 347008296bc04fbeb43d1c56662a240a1c42d09a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Aug 2023 16:04:52 +0100 Subject: [PATCH] Adding in updates to db --- backend/Property.py | 47 +++++++++++++++++-- .../app/db/functions/property_functions.py | 43 ++++++++++------- backend/app/db/models/portfolio.py | 13 +++-- backend/app/plan/router.py | 31 ++---------- model_data/app.py | 2 + 5 files changed, 84 insertions(+), 52 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index d836ad70..0dafdd83 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -38,6 +38,7 @@ class Property(BaseUtility): self.full_sap_epc = None self.in_conservation_area = None self.year_built = None + self.number_of_rooms = None self.energy = None self.ventilation = None @@ -184,7 +185,7 @@ class Property(BaseUtility): "wind_turbine": wind_turbine_count, } - def set_property_counts(self): + def set_count_variables(self): """ For EPC fields that are just counts, we'll set them here @@ -197,12 +198,18 @@ class Property(BaseUtility): "number_of_open_fireplaces": "number-open-fireplaces", "number_of_extensions": "extension-count", "number_of_storeys": "flat-storey-count", + "number_of_rooms": "number-habitable-rooms", } + null_attributes = ["number_of_storeys", "number_of_rooms"] + for attribute, epc_field in fields.items(): value = self.data["extension-count"] if value == "" or value in self.DATA_ANOMALY_MATCHES: - value = 0 + if attribute in null_attributes: + value = None + else: + value = 0 else: value = int(value) @@ -227,7 +234,7 @@ class Property(BaseUtility): self.set_solar_pv() self.set_solar_hot_water() self.set_wind_turbine() - self.set_property_counts() + self.set_count_variables() for description, attribute in cleaned.items(): @@ -274,3 +281,37 @@ class Property(BaseUtility): # We don't know when the property was built self.year_built = None + + def _clean_upload_data(self, to_update): + for k, v in to_update.items(): + if v in self.DATA_ANOMALY_MATCHES: + to_update[k] = None + return to_update + + def get_full_property_data(self): + """ + This method extracts the data which is pushed to the database, containing core information, from the EPC + about a property + :return: + """ + + property_data = { + "creation_status": "READY", + "uprn": int(self.data["uprn"]), + "building_reference_number": int(self.data["building-reference-number"]), + "has_pre_condition_report": True, + "has_recommendations": True, + "property_type": self.data["property-type"], + "built_form": self.data["built-form"], + "local_authority": self.data["local-authority-label"], + "constituency": self.data["constituency-label"], + "number_of_rooms": self.number_of_rooms, + "year_built": self.year_built, + "tenure": self.data["tenure"], + "current_epc_rating": self.data["current-energy-rating"], + "current_sap_points": self.data["current-energy-efficiency"] + } + + property_data = self._clean_upload_data(property_data) + + return property_data diff --git a/backend/app/db/functions/property_functions.py b/backend/app/db/functions/property_functions.py index 94c4734a..42dd3b0d 100644 --- a/backend/app/db/functions/property_functions.py +++ b/backend/app/db/functions/property_functions.py @@ -2,6 +2,7 @@ # This script contains methods for interacting with the property table in the database ### import datetime +import pytz from sqlalchemy.orm import sessionmaker from backend.app.db.models.portfolio import PropertyModel, PropertyCreationStatus, PortfolioStatus, PropertyTargetsModel from backend.app.db.connection import db_engine @@ -20,8 +21,6 @@ def create_property(portfolio_id: int, address: str, postcode: str) -> (int, boo Session = sessionmaker(bind=db_engine) with Session() as session: - now = datetime.datetime.now() - try: # Attempt to fetch the existing property existing_property = session.query(PropertyModel).filter_by( @@ -29,7 +28,7 @@ def create_property(portfolio_id: int, address: str, postcode: str) -> (int, boo ).one() # Update the 'updated_at' field - existing_property.updated_at = now + existing_property.updated_at = datetime.datetime.now(pytz.utc) # Merge the updated property back into the session session.merge(existing_property) @@ -43,8 +42,6 @@ def create_property(portfolio_id: int, address: str, postcode: str) -> (int, boo address=address, postcode=postcode, portfolio_id=portfolio_id, - created_at=now, - updated_at=now, creation_status=PropertyCreationStatus.LOADING, status=PortfolioStatus.ASSESSMENT.value, has_pre_condition_report=False, @@ -69,12 +66,10 @@ def create_property_targets(property_id: int, portfolio_id: int, epc_target=None :return: """ Session = sessionmaker(bind=db_engine) - now = datetime.datetime.now() with Session() as session: new_target = PropertyTargetsModel( property_id=property_id, portfolio_id=portfolio_id, - created_at=now, epc=epc_target, heat_demand=heat_demand_target ) @@ -84,16 +79,28 @@ def create_property_targets(property_id: int, portfolio_id: int, epc_target=None return True -def update_property_data(property_data): +def update_property_data(property_id: int, portfolio_id: int, property_data: dict): Session = sessionmaker(bind=db_engine) - now = datetime.datetime.now() + now = datetime.datetime.now(pytz.utc) with Session() as session: - new_target = PropertyTargetsModel( - property_id=property_id, - portfolio_id=portfolio_id, - created_at=now, - epc=epc_target, - heat_demand=heat_demand_target - ) - session.add(new_target) - session.commit() + try: + # Attempt to fetch the existing property + existing_property = session.query(PropertyModel).filter_by( + id=property_id, portfolio_id=portfolio_id + ).one() + + # Update the fields with the data in property_data + for key, value in property_data.items(): + setattr(existing_property, key, value) + + # Optionally, update the 'updated_at' field + existing_property.updated_at = now + + # Merge the updated property back into the session and commit + session.merge(existing_property) + session.commit() + + except NoResultFound: + raise Exception(f"Property with property_id {property_id} and portfolio_id {portfolio_id} not found") + + return True diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index c6a41cbc..7bce3688 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -1,4 +1,6 @@ import enum +import pytz +import datetime from sqlalchemy import Column, Integer, Text, Boolean, Float, DateTime, Enum, ForeignKey from sqlalchemy.ext.declarative import declarative_base @@ -40,8 +42,8 @@ class Portfolio(Base): property_valuation_increase = Column(Float) # Unit is always £ so we don't need to store the unit for the moment rental_yield_increase = Column(Float) # Unit is always £ so we don't need to store the unit for the moment total_work_hours = Column(Float) - created_at = Column(DateTime, nullable=False) - updated_at = Column(DateTime, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) + updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) class PropertyCreationStatus(enum.Enum): @@ -66,13 +68,14 @@ class PropertyModel(Base): portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) creation_status = Column(Enum(PropertyCreationStatus), nullable=False) uprn = Column(Integer) + building_reference_number = Column(Integer) status = Column(Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]), nullable=False) address = Column(Text) postcode = Column(Text) has_pre_condition_report = Column(Boolean) has_recommendations = Column(Boolean) - created_at = Column(DateTime, nullable=False) - updated_at = Column(DateTime, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) + updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) property_type = Column(Text) built_form = Column(Text) local_authority = Column(Text) @@ -148,6 +151,6 @@ class PropertyTargetsModel(Base): id = Column(Integer, primary_key=True, autoincrement=True) property_id = Column(Integer, ForeignKey('property.id'), nullable=False) portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) - created_at = Column(DateTime, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) epc = Column(Enum(Epc)) heat_demand = Column(Text) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 1d86cba8..a0b00a38 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -1,4 +1,3 @@ -import datetime from fastapi import APIRouter, Depends from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest @@ -13,7 +12,7 @@ from utils.uvalue_estimates import classify_decile_newvalues from model_data.EpcClean import EpcClean # database interaction functions -from backend.app.db.functions.property_functions import create_property, create_property_targets +from backend.app.db.functions.property_functions import create_property, create_property_targets, update_property_data # TODO: This is placeholder until data is stored in DB from backend.app.plan.uvalue_estimates_walls import uvalue_estimates_walls @@ -105,6 +104,7 @@ async def trigger_plan(body: PlanTriggerRequest): property_id=property_id, portfolio_id=body.portfolio_id, epc_target=body.goal_value, + heat_demand_target=None ) input_properties.append( @@ -212,30 +212,6 @@ async def trigger_plan(body: PlanTriggerRequest): # Upload property data for p in input_properties: - property_data = { - "creation_status": "COMPLETE", - "uprn": int(p.data["uprn"]), - "has_pre_condition_report": True, - "has_recommendations": True, - "property_type": p.data["property-type"], - "built_form": p.data["built-form"], - "local_authority": p.data["local-authority-label"], - "constituency": p.data["constituency-label"], - "number_of_rooms": p.data["number-habitable-rooms"], - "year_built": p.year_built, - "tenure": p.data["tenure"], - "current_epc_rating": p.data["current-energy-rating"], - "current_sap_points": p.data["current-energy-efficiency"] - } - - def clean_upload_data(to_update, to_clean_values): - for k, v in to_update.items(): - if v in to_clean_values: - to_update[k] = None - return to_update - - property_data = clean_upload_data(property_data, to_clean_values=p.DATA_ANOMALY_MATCHES) - def prepare_rating(field): rating_lookup = { "Very Good": 5, @@ -286,4 +262,7 @@ async def trigger_plan(body: PlanTriggerRequest): "co2_emissions": p.energy["co2_emissions"], } + property_data = p.get_full_property_data() + update_property_data(property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data) + return {"recommendations": recommendations} diff --git a/model_data/app.py b/model_data/app.py index ae4bc5b0..a3c62acb 100644 --- a/model_data/app.py +++ b/model_data/app.py @@ -76,6 +76,8 @@ def app(): cleaner = EpcClean(data) lighting_averages = cleaner.lighting_averages # TODO: WE need to store lighting_averages to a db + # We should also extend these averages so they're by more variables (property type, age band, constituency, + # etc) cleaner.clean() # TODO: cleaner.cleaned datasets to a db