From c71e6fe44f21e69bd37d7c83f35c5d69069b9fe9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 31 Jul 2023 15:02:12 +0100 Subject: [PATCH 1/3] added property and portfolio models' --- backend/app/db/models/portfolio.py | 50 ++++++++++++++ backend/app/db/models/property.py | 104 +++++++++++++++++++++++++++++ backend/app/plan/router.py | 2 + 3 files changed, 156 insertions(+) create mode 100644 backend/app/db/models/portfolio.py create mode 100644 backend/app/db/models/property.py diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py new file mode 100644 index 00000000..be67a7aa --- /dev/null +++ b/backend/app/db/models/portfolio.py @@ -0,0 +1,50 @@ +from sqlalchemy import Column, Integer, String, Text, Boolean, Float, DateTime, Enum +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class PortfolioStatus(Enum): + SCOPING = "scoping" + ASSESSMENT = "assessment" + TENDERING = "tendering" + PROJECT_UNDERWAY = "project underway" + COMPLETION_ON_TRACK = "completion; status: on track" + COMPLETION_DELAYED = "completion; status: delayed" + COMPLETION_AT_RISK = "completion; status: at risk" + COMPLETED = "completion; status: completed" + NEEDS_REVIEW = "needs review" + + +class PortfolioGoal(Enum): + VALUATION_IMPROVEMENT = "Valuation Improvement" + INCREASING_EPC = "Increasing EPC" + REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions" + ENERGY_SAVINGS = "Energy Savings" + NONE = "None" + + +class PortfolioRole(Enum): + CREATOR = "creator" + ADMIN = "admin" + READ = "read" + WRITE = "write" + + +class Portfolio(Base): + __tablename__ = 'portfolio' + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(Text, nullable=False) + budget = Column(Float) + status = Column(Enum(PortfolioStatus), nullable=False) + goal = Column(Enum(PortfolioGoal), nullable=False) + cost = Column(Float) + number_of_properties = Column(Integer) + co2_equivalent_savings = Column(Float) # Unit is always tonnes so we don't need to store the unit + energy_savings = Column(Float) # Unit is always kWh so we don't need to store the unit + energy_cost_savings = Column(Float) # Unit is always £ so we don't need to store the unit for the moment + property_valuation_increase = Column(Float) # Unit is always £ so we don't need to store the unit for the moment + rental_yield_increase = Column(Float) # Unit is always £ so we don't need to store the unit for the moment + total_work_hours = Column(Float) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) diff --git a/backend/app/db/models/property.py b/backend/app/db/models/property.py new file mode 100644 index 00000000..243967f7 --- /dev/null +++ b/backend/app/db/models/property.py @@ -0,0 +1,104 @@ +from sqlalchemy import Column, Integer, Text, Boolean, Float, DateTime, Enum, ForeignKey +from sqlalchemy.ext.declarative import declarative_base +from backend.app.db.models.portfolio import PortfolioStatus + +Base = declarative_base() + + +class PropertyCreationStatus(Enum): + LOADING = "LOADING" + READY = "READY" + ERROR = "ERROR" + + +class Epc(Enum): + A = "A" + B = "B" + C = "C" + D = "D" + E = "E" + F = "F" + G = "G" + + +class Property(Base): + __tablename__ = 'property' + id = Column(Integer, primary_key=True, autoincrement=True) + portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) + creation_status = Column(Enum(PropertyCreationStatus), nullable=False) + uprn = Column(Integer) + status = Column(Enum(PortfolioStatus)) + address = Column(Text) + postcode = Column(Text) + has_pre_condition_report = Column(Boolean) + has_recommendations = Column(Boolean) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + property_type = Column(Text) + built_form = Column(Text) + local_authority = Column(Text) + constituency = Column(Text) + number_of_rooms = Column(Integer) + year_built = Column(Text) + tenure = Column(Text) + current_epc_rating = Column(Enum(Epc)) + current_sap_points = Column(Float) + + +class FeatureRating(Enum): + VERY_GOOD = "Very good" + GOOD = "Good" + POOR = "Poor" + VERY_POOR = "Very poor" + NA = "N/A" + + +class PropertyDetailsEpc(Base): + __tablename__ = 'property_details_epc' + id = Column(Integer, primary_key=True, autoincrement=True) + property_id = Column(Integer, ForeignKey('property.id'), nullable=False) + portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) + full_address = Column(Text) + total_floor_area = Column(Float) + walls = Column(Text) + walls_rating = Column(Enum(FeatureRating)) + roof = Column(Text) + roof_rating = Column(Enum(FeatureRating)) + floor = Column(Text) + floor_rating = Column(Enum(FeatureRating)) + windows = Column(Text) + windows_rating = Column(Enum(FeatureRating)) + heating = Column(Text) + heating_rating = Column(Enum(FeatureRating)) + heating_contols = Column(Text) + heating_contols_rating = Column(Enum(FeatureRating)) + hot_water = Column(Text) + hot_water_rating = Column(Enum(FeatureRating)) + lighting = Column(Text) + lighting_rating = Column(Enum(FeatureRating)) + ventilation = Column(Text) + solar_pv = Column(Text) + solar_hot_water = Column(Text) + wind_turbine = Column(Text) + floor_height = Column(Float) + number_heated_rooms = Column(Integer) + heat_loss_corridor = Column(Boolean) + unheated_corridor_length = Column(Float) + number_of_open_fireplaces = Column(Integer) + number_of_extensions = Column(Integer) + number_of_storeys = Column(Integer) + mains_gas = Column(Boolean) + energy_tariff = Column(Text) + primary_energy_consumption = Column(Float) + co2_emissions = Column(Float) + + +class PropertyDetailsMeter(Base): + __tablename__ = 'property_details_meter' + id = Column(Integer, primary_key=True, autoincrement=True) + uprn = Column(Integer, nullable=False) + energy_supplier = Column(Text) + gas_supplier = Column(Text) + meter_reading_total = Column(Float) + meter_reading_electricity = Column(Float) + meter_reading_gas = Column(Float) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4934042e..4729b081 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -75,6 +75,8 @@ async def trigger_plan(body: PlanTriggerRequest): plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path) # TODO: Add validation to the file + # for config in plan_input: + # # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly logger.info("Getting EPC data") epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) From 1ceedc2ebb6ed5a7b9792e55f189f817ffd12a81 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 31 Jul 2023 16:26:54 +0100 Subject: [PATCH 2/3] Adding property table cration to router --- backend/Property.py | 3 +- .../app/db/functions/property_functions.py | 53 ++++++++ backend/app/db/models/portfolio.py | 127 ++++++++++++++++-- backend/app/db/models/property.py | 104 -------------- backend/app/plan/router.py | 33 +++-- 5 files changed, 195 insertions(+), 125 deletions(-) create mode 100644 backend/app/db/functions/property_functions.py delete mode 100644 backend/app/db/models/property.py diff --git a/backend/Property.py b/backend/Property.py index 5ed583de..25865a4f 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -29,7 +29,8 @@ class Property(BaseUtility): coordinates = None - def __init__(self, postcode, address1, epc_client=None, data=None): + def __init__(self, id, postcode, address1, epc_client=None, data=None): + self.id = id self.postcode = postcode self.address1 = address1 self.data = data diff --git a/backend/app/db/functions/property_functions.py b/backend/app/db/functions/property_functions.py new file mode 100644 index 00000000..a5031bd8 --- /dev/null +++ b/backend/app/db/functions/property_functions.py @@ -0,0 +1,53 @@ +### +# This script contains methods for interacting with the property table in the database +### +import datetime +from sqlalchemy.orm import sessionmaker +from backend.app.db.models.portfolio import PropertyModel, PropertyCreationStatus, PortfolioStatus +from backend.app.db.connection import db_engine +from sqlalchemy.orm.exc import NoResultFound + + +def create_property(portfolio_id: int, address: str, postcode: str): + """ + This function will create a record for the property in the database if it does not exist. + If it does exist, it will just update the updated_at field. + """ + Session = sessionmaker(bind=db_engine) + with Session() as session: + + now = datetime.datetime.now() + + try: + # Attempt to fetch the existing property + existing_property = session.query(PropertyModel).filter_by( + address=address, postcode=postcode, portfolio_id=portfolio_id + ).one() + + # Update the 'updated_at' field + existing_property.updated_at = now + + # Merge the updated property back into the session + session.merge(existing_property) + session.commit() + + return existing_property.id + + except NoResultFound: + # Property doesn't exist, create a new one + new_property = PropertyModel( + address=address, + postcode=postcode, + portfolio_id=portfolio_id, + created_at=now, + updated_at=now, + creation_status=PropertyCreationStatus.LOADING, + status=PortfolioStatus.ASSESSMENT.value + ) + + # Add the new property to the session + session.add(new_property) + + session.commit() + + return new_property.id diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index be67a7aa..88da3c2f 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -1,10 +1,11 @@ -from sqlalchemy import Column, Integer, String, Text, Boolean, Float, DateTime, Enum +import enum +from sqlalchemy import Column, Integer, Text, Boolean, Float, DateTime, Enum, ForeignKey from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() -class PortfolioStatus(Enum): +class PortfolioStatus(enum.Enum): SCOPING = "scoping" ASSESSMENT = "assessment" TENDERING = "tendering" @@ -16,7 +17,7 @@ class PortfolioStatus(Enum): NEEDS_REVIEW = "needs review" -class PortfolioGoal(Enum): +class PortfolioGoal(enum.Enum): VALUATION_IMPROVEMENT = "Valuation Improvement" INCREASING_EPC = "Increasing EPC" REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions" @@ -24,20 +25,13 @@ class PortfolioGoal(Enum): NONE = "None" -class PortfolioRole(Enum): - CREATOR = "creator" - ADMIN = "admin" - READ = "read" - WRITE = "write" - - class Portfolio(Base): __tablename__ = 'portfolio' id = Column(Integer, primary_key=True, autoincrement=True) name = Column(Text, nullable=False) budget = Column(Float) - status = Column(Enum(PortfolioStatus), nullable=False) - goal = Column(Enum(PortfolioGoal), nullable=False) + status = Column(Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]), nullable=False) + goal = Column(Enum(PortfolioGoal, values_callable=lambda x: [e.value for e in x]), nullable=False) cost = Column(Float) number_of_properties = Column(Integer) co2_equivalent_savings = Column(Float) # Unit is always tonnes so we don't need to store the unit @@ -48,3 +42,112 @@ class Portfolio(Base): total_work_hours = Column(Float) created_at = Column(DateTime, nullable=False) updated_at = Column(DateTime, nullable=False) + + +class PropertyCreationStatus(enum.Enum): + LOADING = "LOADING" + READY = "READY" + ERROR = "ERROR" + + +class Epc(enum.Enum): + A = "A" + B = "B" + C = "C" + D = "D" + E = "E" + F = "F" + G = "G" + + +class PropertyModel(Base): + __tablename__ = 'property' + id = Column(Integer, primary_key=True, autoincrement=True) + portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) + creation_status = Column(Enum(PropertyCreationStatus), nullable=False) + uprn = Column(Integer) + status = Column(Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]), nullable=False) + address = Column(Text) + postcode = Column(Text) + has_pre_condition_report = Column(Boolean) + has_recommendations = Column(Boolean) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + property_type = Column(Text) + built_form = Column(Text) + local_authority = Column(Text) + constituency = Column(Text) + number_of_rooms = Column(Integer) + year_built = Column(Text) + tenure = Column(Text) + current_epc_rating = Column(Enum(Epc)) + current_sap_points = Column(Float) + + +class FeatureRating(enum.Enum): + VERY_GOOD = "Very good" + GOOD = "Good" + POOR = "Poor" + VERY_POOR = "Very poor" + NA = "N/A" + + +class PropertyDetailsEpc(Base): + __tablename__ = 'property_details_epc' + id = Column(Integer, primary_key=True, autoincrement=True) + property_id = Column(Integer, ForeignKey('property.id'), nullable=False) + portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) + full_address = Column(Text) + total_floor_area = Column(Float) + walls = Column(Text) + walls_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + roof = Column(Text) + roof_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + floor = Column(Text) + floor_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + windows = Column(Text) + windows_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + heating = Column(Text) + heating_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + heating_contols = Column(Text) + heating_contols_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + hot_water = Column(Text) + hot_water_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + lighting = Column(Text) + lighting_rating = Column(Enum(FeatureRating, values_callable=lambda x: [e.value for e in x])) + ventilation = Column(Text) + solar_pv = Column(Text) + solar_hot_water = Column(Text) + wind_turbine = Column(Text) + floor_height = Column(Float) + number_heated_rooms = Column(Integer) + heat_loss_corridor = Column(Boolean) + unheated_corridor_length = Column(Float) + number_of_open_fireplaces = Column(Integer) + number_of_extensions = Column(Integer) + number_of_storeys = Column(Integer) + mains_gas = Column(Boolean) + energy_tariff = Column(Text) + primary_energy_consumption = Column(Float) + co2_emissions = Column(Float) + + +class PropertyDetailsMeter(Base): + __tablename__ = 'property_details_meter' + id = Column(Integer, primary_key=True, autoincrement=True) + uprn = Column(Integer, nullable=False) + energy_supplier = Column(Text) + gas_supplier = Column(Text) + meter_reading_total = Column(Float) + meter_reading_electricity = Column(Float) + meter_reading_gas = Column(Float) + + +class PropertyTargets(Base): + __tablename__ = 'property_targets' + id = Column(Integer, primary_key=True, autoincrement=True) + property_id = Column(Integer, ForeignKey('property.id'), nullable=False) + portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) + created_at = Column(DateTime, nullable=False) + epc = Column(Enum(Epc)) + heat_demand = Column(Text) diff --git a/backend/app/db/models/property.py b/backend/app/db/models/property.py deleted file mode 100644 index 243967f7..00000000 --- a/backend/app/db/models/property.py +++ /dev/null @@ -1,104 +0,0 @@ -from sqlalchemy import Column, Integer, Text, Boolean, Float, DateTime, Enum, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from backend.app.db.models.portfolio import PortfolioStatus - -Base = declarative_base() - - -class PropertyCreationStatus(Enum): - LOADING = "LOADING" - READY = "READY" - ERROR = "ERROR" - - -class Epc(Enum): - A = "A" - B = "B" - C = "C" - D = "D" - E = "E" - F = "F" - G = "G" - - -class Property(Base): - __tablename__ = 'property' - id = Column(Integer, primary_key=True, autoincrement=True) - portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) - creation_status = Column(Enum(PropertyCreationStatus), nullable=False) - uprn = Column(Integer) - status = Column(Enum(PortfolioStatus)) - address = Column(Text) - postcode = Column(Text) - has_pre_condition_report = Column(Boolean) - has_recommendations = Column(Boolean) - created_at = Column(DateTime, nullable=False) - updated_at = Column(DateTime, nullable=False) - property_type = Column(Text) - built_form = Column(Text) - local_authority = Column(Text) - constituency = Column(Text) - number_of_rooms = Column(Integer) - year_built = Column(Text) - tenure = Column(Text) - current_epc_rating = Column(Enum(Epc)) - current_sap_points = Column(Float) - - -class FeatureRating(Enum): - VERY_GOOD = "Very good" - GOOD = "Good" - POOR = "Poor" - VERY_POOR = "Very poor" - NA = "N/A" - - -class PropertyDetailsEpc(Base): - __tablename__ = 'property_details_epc' - id = Column(Integer, primary_key=True, autoincrement=True) - property_id = Column(Integer, ForeignKey('property.id'), nullable=False) - portfolio_id = Column(Integer, ForeignKey('portfolio.id'), nullable=False) - full_address = Column(Text) - total_floor_area = Column(Float) - walls = Column(Text) - walls_rating = Column(Enum(FeatureRating)) - roof = Column(Text) - roof_rating = Column(Enum(FeatureRating)) - floor = Column(Text) - floor_rating = Column(Enum(FeatureRating)) - windows = Column(Text) - windows_rating = Column(Enum(FeatureRating)) - heating = Column(Text) - heating_rating = Column(Enum(FeatureRating)) - heating_contols = Column(Text) - heating_contols_rating = Column(Enum(FeatureRating)) - hot_water = Column(Text) - hot_water_rating = Column(Enum(FeatureRating)) - lighting = Column(Text) - lighting_rating = Column(Enum(FeatureRating)) - ventilation = Column(Text) - solar_pv = Column(Text) - solar_hot_water = Column(Text) - wind_turbine = Column(Text) - floor_height = Column(Float) - number_heated_rooms = Column(Integer) - heat_loss_corridor = Column(Boolean) - unheated_corridor_length = Column(Float) - number_of_open_fireplaces = Column(Integer) - number_of_extensions = Column(Integer) - number_of_storeys = Column(Integer) - mains_gas = Column(Boolean) - energy_tariff = Column(Text) - primary_energy_consumption = Column(Float) - co2_emissions = Column(Float) - - -class PropertyDetailsMeter(Base): - __tablename__ = 'property_details_meter' - id = Column(Integer, primary_key=True, autoincrement=True) - uprn = Column(Integer, nullable=False) - energy_supplier = Column(Text) - gas_supplier = Column(Text) - meter_reading_total = Column(Float) - meter_reading_electricity = Column(Float) - meter_reading_gas = Column(Float) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4729b081..fff83e63 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -10,6 +10,9 @@ from recommendations.FloorRecommendations import FloorRecommendations from recommendations.WallRecommendations import WallRecommendations from utils.uvalue_estimates import classify_decile_newvalues +# database interaction functions +from backend.app.db.functions.property_functions import create_property + # TODO: This is placeholder until data is stored in DB from backend.app.plan.temp_cleaned_data import cleaned from backend.app.plan.uvalue_estimates_walls import uvalue_estimates_walls @@ -72,18 +75,32 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Getting the inputs") # Read in the trigger file from s3 bucket_name = get_settings().PLAN_TRIGGER_BUCKET + epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) + plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path) - # TODO: Add validation to the file - # for config in plan_input: - # # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly + input_properties = [] + for config in plan_input: + # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly + # TODO: implment validation + + # Create a record in db + property_id = create_property( + portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] + ) + return {"message": "success"} + + input_properties.append( + Property( + postcode=config['postcode'], + address1=config['address'], + epc_client=epc_client, + id=property_id + ) + ) + return {"message": "success"} logger.info("Getting EPC data") - epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) - input_properties = [ - Property(postcode=config['postcode'], address1=config['address'], epc_client=epc_client) - for config in plan_input - ] for p in input_properties: p.search_address_epc() From 2c4d06f746bb3324909bb629715e32d3d85aa500 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 31 Jul 2023 16:48:26 +0100 Subject: [PATCH 3/3] Added database post to plan trigger --- backend/app/db/functions/property_functions.py | 14 ++++++++++---- backend/app/plan/router.py | 9 +++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/app/db/functions/property_functions.py b/backend/app/db/functions/property_functions.py index a5031bd8..499b66b5 100644 --- a/backend/app/db/functions/property_functions.py +++ b/backend/app/db/functions/property_functions.py @@ -8,10 +8,14 @@ from backend.app.db.connection import db_engine from sqlalchemy.orm.exc import NoResultFound -def create_property(portfolio_id: int, address: str, postcode: str): +def create_property(portfolio_id: int, address: str, postcode: str) -> (int, bool): """ This function will create a record for the property in the database if it does not exist. If it does exist, it will just update the updated_at field. + :param portfolio_id: The ID of the portfolio the property belongs to + :param address: The address of the property + :param postcode: The postcode of the property + :return: The ID of the property and a boolean indicating whether it was created or not """ Session = sessionmaker(bind=db_engine) with Session() as session: @@ -31,7 +35,7 @@ def create_property(portfolio_id: int, address: str, postcode: str): session.merge(existing_property) session.commit() - return existing_property.id + return existing_property.id, False except NoResultFound: # Property doesn't exist, create a new one @@ -42,7 +46,9 @@ def create_property(portfolio_id: int, address: str, postcode: str): created_at=now, updated_at=now, creation_status=PropertyCreationStatus.LOADING, - status=PortfolioStatus.ASSESSMENT.value + status=PortfolioStatus.ASSESSMENT.value, + has_pre_condition_report=False, + has_recommendations=False ) # Add the new property to the session @@ -50,4 +56,4 @@ def create_property(portfolio_id: int, address: str, postcode: str): session.commit() - return new_property.id + return new_property.id, True diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fff83e63..a8dd36e6 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -85,10 +85,13 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: implment validation # Create a record in db - property_id = create_property( + property_id, is_new = create_property( portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] ) - return {"message": "success"} + + # if a new record was not created, we don't produduce recommendations + if not is_new: + continue input_properties.append( Property( @@ -98,10 +101,8 @@ async def trigger_plan(body: PlanTriggerRequest): id=property_id ) ) - return {"message": "success"} logger.info("Getting EPC data") - for p in input_properties: p.search_address_epc() p.set_year_built()