putting together data upload to db

This commit is contained in:
Khalim Conn-Kowlessar 2024-07-25 18:43:14 +01:00
parent 6702eb65b0
commit 4192ee7d69

View file

@ -1,5 +1,5 @@
import pandas as pd
from sqlalchemy.orm import sessionmaker
from backend.app.db.connection import db_engine
from utils.s3 import read_from_s3, list_files_and_subfolders_in_s3_folder, list_xmls_in_s3_folder
from utils.logger import setup_logger
from etl.xml_survey_extraction.XmlParser import XmlParser
@ -43,6 +43,7 @@ def main():
logger.info(f"Exatracted XMLS for the energy assessments")
# For each property, we download the xmls and extract the data
database_data = []
for uprn, xmls in assessments_map.items():
extracted_data = {}
for xml in xmls:
@ -59,6 +60,16 @@ def main():
extracted_epc = xml_parser.epc
extracted_additional_data = xml_parser.additional_data
data_to_update = {
**extracted_epc, **extracted_additional_data
}
extracted_data.update(data_to_update)
database_data.append(extracted_data)
logger.info("Uploading data to the database")
session = sessionmaker(bind=db_engine)()
# TODO: Set a portfolio ID, Target and Automatically upload the asset list and create the event for the portfolio
# TODO: In order to get the full data associated to the heating system, we need to download and parse the pcdb which