From c693c6a633b09ce7a7464164f915c8f047cc6c9d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 12:19:28 +0100 Subject: [PATCH] updating write to db for solar api, wip --- backend/app/db/functions/solar_functions.py | 108 ++++++++++++-------- backend/app/plan/router.py | 1 - etl/sfr/midlands_portfolio_asset_list.py | 43 +++++++- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git a/backend/app/db/functions/solar_functions.py b/backend/app/db/functions/solar_functions.py index 59243f01..3ead1551 100644 --- a/backend/app/db/functions/solar_functions.py +++ b/backend/app/db/functions/solar_functions.py @@ -1,5 +1,6 @@ import datetime import pytz +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from backend.app.db.models.solar import Solar, SolarScenario @@ -38,57 +39,80 @@ def get_solar_data(session: Session, longitude: float = None, latitude: float = def store_batch_data(session: Session, api_data: dict, uprns_to_location: list, scenarios_data: list): """ This function will store the API data to the solar table against all of the UPRNs with longitude and latitude. + If a record already exists in the Solar table by UPRN, it will be updated instead of creating a new one. + Similarly, if a scenario exists in SolarScenario by number_panels, it will also be updated. + :param session: The database session :param api_data: The API data to store - :param uprns_to_location: A list of dictionaries containing uprn, longitude, and latitude + :param uprns_to_location: A list of dictionaries containing UPRN, longitude, and latitude :param scenarios_data: A list of dictionaries containing scenario data for each UPRN """ try: - - # Insert data into the Solar table and get the IDs - solar_records = [] + # Insert or update data into the Solar table for data in uprns_to_location: - solar_record = Solar( - uprn=data['uprn'], - longitude=data['longitude'], - latitude=data['latitude'], - google_api_response=api_data, - updated_at=datetime.datetime.now(pytz.utc) - ) - solar_records.append(solar_record) - session.add(solar_record) + existing_solar = session.execute(select(Solar).where(Solar.uprn == data['uprn'])).scalar_one_or_none() - session.flush() # Flush to get the IDs generated - - for record in solar_records: - session.refresh(record) # Refresh to populate the ID fields - - # Retrieve the IDs of the inserted records - inserted_ids = {record.uprn: record.id for record in solar_records} - - # Prepare the data for SolarScenario - scenario_records = [] - for data in uprns_to_location: - solar_id = inserted_ids.get(data['uprn']) - for scenario in scenarios_data: - scenario_record = SolarScenario( - solar_id=solar_id, - scenario_type=scenario['scenario_type'], - number_panels=scenario['number_panels'], - array_kwhp=scenario['array_kwhp'], - lifetime_dc_kwh=scenario['lifetime_dc_kwh'], - yearly_dc_kwh=scenario['yearly_dc_kwh'], - lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field - yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field - cost=scenario['cost'], - expected_payback_years=scenario.get('expected_payback_years'), # Optional field - panelled_roof_area=scenario['panelled_roof_area'], - is_default=scenario['is_default'] + if existing_solar: + # Update the existing record + existing_solar.longitude = data['longitude'] + existing_solar.latitude = data['latitude'] + existing_solar.google_api_response = api_data + existing_solar.updated_at = datetime.datetime.now(pytz.utc) + solar_id = existing_solar.id + else: + # Insert a new record + solar_record = Solar( + uprn=data['uprn'], + longitude=data['longitude'], + latitude=data['latitude'], + google_api_response=api_data, + updated_at=datetime.datetime.now(pytz.utc) ) - scenario_records.append(scenario_record) + session.add(solar_record) + session.flush() # Flush to get the IDs generated + session.refresh(solar_record) # Refresh to populate the ID field + solar_id = solar_record.id - # Insert data into the SolarScenario table - session.bulk_save_objects(scenario_records) + # Insert or update data in the SolarScenario table + for scenario in scenarios_data: + existing_scenario = session.execute( + select(SolarScenario).where( + SolarScenario.solar_id == solar_id, + SolarScenario.number_panels == scenario['number_panels'] + ) + ).scalar_one_or_none() + + if existing_scenario: + # Update the existing scenario record + existing_scenario.scenario_type = scenario['scenario_type'] + existing_scenario.array_kwhp = scenario['array_kwhp'] + existing_scenario.lifetime_dc_kwh = scenario['lifetime_dc_kwh'] + existing_scenario.yearly_dc_kwh = scenario['yearly_dc_kwh'] + existing_scenario.lifetime_ac_kwh = scenario.get('lifetime_ac_kwh') # Optional field + existing_scenario.yearly_ac_kwh = scenario.get('yearly_ac_kwh') # Optional field + existing_scenario.cost = scenario['cost'] + existing_scenario.expected_payback_years = scenario.get('expected_payback_years') # Optional field + existing_scenario.panelled_roof_area = scenario['panelled_roof_area'] + existing_scenario.is_default = scenario['is_default'] + else: + # Insert a new scenario record + scenario_record = SolarScenario( + solar_id=solar_id, + scenario_type=scenario['scenario_type'], + number_panels=scenario['number_panels'], + array_kwhp=scenario['array_kwhp'], + lifetime_dc_kwh=scenario['lifetime_dc_kwh'], + yearly_dc_kwh=scenario['yearly_dc_kwh'], + lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field + yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field + cost=scenario['cost'], + expected_payback_years=scenario.get('expected_payback_years'), # Optional field + panelled_roof_area=scenario['panelled_roof_area'], + is_default=scenario['is_default'] + ) + session.add(scenario_record) + + # Commit the changes after all operations session.commit() except Exception as e: diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e925fe00..b5cb96f6 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,7 +30,6 @@ from backend.app.plan.utils import get_cleaned from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc from backend.ml_models.api import ModelApi -from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.Property import Property from backend.apis.GoogleSolarApi import GoogleSolarApi diff --git a/etl/sfr/midlands_portfolio_asset_list.py b/etl/sfr/midlands_portfolio_asset_list.py index 0434b45a..01a01907 100644 --- a/etl/sfr/midlands_portfolio_asset_list.py +++ b/etl/sfr/midlands_portfolio_asset_list.py @@ -1,4 +1,5 @@ import pandas as pd +from utils.s3 import save_csv_to_s3 def app(): @@ -7,5 +8,45 @@ def app(): :return: """ + portfolio_id = 108 + # Read in the portfolio EPC data - epc_data = pd.read_excel() + epc_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" + ) + + asset_list = epc_data[ + [ + "ADDRESS1", "POSTCODE", "UPRN" + ] + ].copy().rename( + columns={ + "ADDRESS1": "address", + "POSTCODE": "postcode", + "UPRN": "uprn" + } + ) + + # Store data and prepare payload + + filename = f"{8}/{portfolio_id}/asset_list.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(portfolio_id), + "housing_type": "Private", + "goal": "Increasing EPC", + "goal_value": "C", + "trigger_file_path": filename, + "already_installed_file_path": "", + "patches_file_path": "", + "non_invasive_recommendations_file_path": "", + "budget": None, + "scenario_name": "EPC C Package", + "multi_plan": True, + } + print(body)