From 389e9f51da2af199ce3a79110ac29e9c39360c8d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Jul 2024 11:15:01 +0100 Subject: [PATCH] setting up storage to db --- backend/apis/GoogleSolarApi.py | 34 ++++++++++- backend/app/db/functions/solar_functions.py | 64 +++++++++++++++++++++ backend/app/db/models/solar.py | 21 +++++++ backend/app/plan/router.py | 18 +++++- recommendations/SolarPvRecommendations.py | 4 +- 5 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 backend/app/db/functions/solar_functions.py create mode 100644 backend/app/db/models/solar.py diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index f7b34d19..55041b74 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -5,6 +5,10 @@ from backend.ml_models.AnnualBillSavings import AnnualBillSavings import requests from functools import lru_cache import time +from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data +from utils.logger import setup_logger + +logger = setup_logger() class GoogleSolarApi: @@ -98,7 +102,10 @@ class GoogleSolarApi: raise @lru_cache(maxsize=128) - def get(self, longitude, latitude, energy_consumption, required_quality="MEDIUM", is_building=False): + def get( + self, longitude, latitude, energy_consumption, required_quality="MEDIUM", is_building=False, session=None, + uprn=None + ): """ Wrapper function that calls get_building_insights and extracts roof segments, with caching. @@ -107,10 +114,21 @@ class GoogleSolarApi: :param energy_consumption: The energy consumption of the building/unit associated to the longitude and latitude. :param required_quality: The required quality of the data (default is "MEDIUM"). :param is_building: Whether the energy consumption is for a building or a unit. + :param session: The database session to use for the query (default is None). + :param uprn: The unique property reference number (default is None). :return: The JSON response containing the building insights data. """ - self.insights_data = self.get_building_insights(longitude, latitude, required_quality) + is_outdated = False + if session is not None: + # Check if the data is already in the database + self.insights_data, _, is_outdated = get_solar_data( + session, longitude=longitude, latitude=latitude, uprn=uprn + ) + + # If we have no data in the db, or updated_at is more than 6 months + if self.insights_data is None or is_outdated: + self.insights_data = self.get_building_insights(longitude, latitude, required_quality) # Extract key data from the insights response self.roof_segments = self.insights_data["solarPotential"].get('roofSegmentStats', []) @@ -137,6 +155,18 @@ class GoogleSolarApi: # We now start finding the solar panel configurations self.optimise_solar_configuration(energy_consumption=energy_consumption, is_building=is_building) + def save_to_db(self, session, uprns_to_location): + if self.insights_data is None: + raise ValueError("No api data to store") + + logger.info("Storing to database") + + store_batch_data( + session=session, + api_data=self.insights_data, + uprns_to_location=uprns_to_location + ) + @staticmethod def lifetime_production_ac_kwh( row, diff --git a/backend/app/db/functions/solar_functions.py b/backend/app/db/functions/solar_functions.py new file mode 100644 index 00000000..e8bba137 --- /dev/null +++ b/backend/app/db/functions/solar_functions.py @@ -0,0 +1,64 @@ +import datetime +import pytz +from sqlalchemy.orm import Session +from sqlalchemy.orm.exc import NoResultFound +from backend.app.db.models.solar import Solar + + +def get_solar_data(session: Session, longitude: float = None, latitude: float = None, uprn: str = None): + """ + This function will fetch data from the solar table based on longitude and latitude or UPRN. + :param session: The database session + :param longitude: The longitude to search for + :param latitude: The latitude to search for + :param uprn: The UPRN to search for (overrides longitude and latitude if provided) + :return: The google_api_response and updated_at fields + """ + try: + if uprn: + # Search by UPRN + solar_data = session.query(Solar.google_api_response, Solar.updated_at).filter_by(uprn=uprn).one() + else: + # Search by longitude and latitude + solar_data = session.query(Solar.google_api_response, Solar.updated_at).filter( + Solar.longitude == longitude, + Solar.latitude == latitude + ).one() + + # Check if updated_at is more than 6 months old + six_months_ago = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=6 * 30) # Approximate 6 months + is_outdated = solar_data.updated_at < six_months_ago + + return solar_data.google_api_response, solar_data.updated_at, is_outdated + + except NoResultFound: + return None, None, False + + +def store_batch_data(session: Session, api_data: dict, uprns_to_location: list): + """ + This function will store the API data to the solar table against all of the UPRNs with longitude and latitude. + :param session: The database session + :param api_data: The API data to store + :param data_list: A list of dictionaries containing uprn, longitude, and latitude + """ + try: + # Convert the data_list to a list of dicts for bulk insert + records_to_update = [] + for data in uprns_to_location: + record = { + 'uprn': data['uprn'], + 'longitude': data['longitude'], + 'latitude': data['latitude'], + 'google_api_response': api_data, + 'updated_at': datetime.datetime.now(pytz.utc) + } + records_to_update.append(record) + + # Perform bulk insert or update + session.bulk_insert_mappings(Solar, records_to_update) + session.commit() + + except Exception as e: + session.rollback() + raise e diff --git a/backend/app/db/models/solar.py b/backend/app/db/models/solar.py new file mode 100644 index 00000000..9cc51e51 --- /dev/null +++ b/backend/app/db/models/solar.py @@ -0,0 +1,21 @@ +import datetime +import pytz +from sqlalchemy import Column, Integer, Float, DateTime, JSON +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class Solar(Base): + __tablename__ = 'solar' + id = Column(Integer, primary_key=True, autoincrement=True) + longitude = Column(Float, nullable=False) + latitude = Column(Float, nullable=False) + uprn = Column(Integer, nullable=False) + created_at = Column( + DateTime, nullable=False, default=datetime.datetime.now(pytz.utc) + ) + updated_at = Column( + DateTime, nullable=False, default=datetime.datetime.now(pytz.utc), onupdate=datetime.datetime.now(pytz.utc) + ) + google_api_response = Column(JSON, nullable=False) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 7d92f425..c85382e7 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -380,12 +380,14 @@ async def trigger_plan(body: PlanTriggerRequest): target_rating=body.goal_value, current_consumption=p.current_adjusted_energy ), - "property_id": p.id + "property_id": p.id, + "uprn": p.uprn } for p in input_properties if p.building_id is not None ] if building_ids: # Find the unique longitude and latitude pairs for each building id unique_coordinates = {} + building_uprns = {} for entry in building_ids: building_id = entry['building_id'] coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']} @@ -396,6 +398,16 @@ async def trigger_plan(body: PlanTriggerRequest): if coordinate_pair not in unique_coordinates[building_id]: unique_coordinates[building_id].append(coordinate_pair) + if building_id not in building_uprns: + building_uprns[building_id] = [] + + if entry['uprn'] not in building_uprns[building_id]: + building_uprns[building_id].append( + { + "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude'] + } + ) + solar_panel_configuration = {} for building_id, coordinates in unique_coordinates.items(): if len(coordinates) > 1: @@ -410,6 +422,7 @@ async def trigger_plan(body: PlanTriggerRequest): latitude=coordinates["latitude"], energy_consumption=energy_consumption, is_building=True, + session=session ) solar_panel_configuration[building_id] = { "insights_data": solar_api_client.insights_data, @@ -417,6 +430,9 @@ async def trigger_plan(body: PlanTriggerRequest): "n_units": len([entry for entry in building_ids if entry['building_id'] == building_id]) } + # Store the data in the database + solar_api_client.save_to_db(session=session, uprns_to_location=building_uprns[building_id]) + # Insert this into the properties that have this building id for p in input_properties: if p.building_id == building_id: diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 5219e323..af1e7f27 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -109,7 +109,9 @@ class SolarPvRecommendations: ) n_units = self.property.solar_panel_configuration["n_units"] - best_configurations = panel_performance.head(3).reset_index(drop=True) + # At a building level, we take a single configuration so that all properties a guaranteed to use + # the same configuration + best_configurations = panel_performance.head(1).reset_index(drop=True) for rank, recommendation_config in best_configurations.iterrows(): roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100)