From 4192ee7d690b1a74a5e1a3e361abe5c08a48bc43 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 25 Jul 2024 18:43:14 +0100 Subject: [PATCH] putting together data upload to db --- etl/xml_survey_extraction/app.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/etl/xml_survey_extraction/app.py b/etl/xml_survey_extraction/app.py index 3e41b5fb..c6e16e3b 100644 --- a/etl/xml_survey_extraction/app.py +++ b/etl/xml_survey_extraction/app.py @@ -1,5 +1,5 @@ -import pandas as pd - +from sqlalchemy.orm import sessionmaker +from backend.app.db.connection import db_engine from utils.s3 import read_from_s3, list_files_and_subfolders_in_s3_folder, list_xmls_in_s3_folder from utils.logger import setup_logger from etl.xml_survey_extraction.XmlParser import XmlParser @@ -43,6 +43,7 @@ def main(): logger.info(f"Exatracted XMLS for the energy assessments") # For each property, we download the xmls and extract the data + database_data = [] for uprn, xmls in assessments_map.items(): extracted_data = {} for xml in xmls: @@ -59,6 +60,16 @@ def main(): extracted_epc = xml_parser.epc extracted_additional_data = xml_parser.additional_data + data_to_update = { + **extracted_epc, **extracted_additional_data + } + extracted_data.update(data_to_update) + + database_data.append(extracted_data) + + logger.info("Uploading data to the database") + session = sessionmaker(bind=db_engine)() + # TODO: Set a portfolio ID, Target and Automatically upload the asset list and create the event for the portfolio # TODO: In order to get the full data associated to the heating system, we need to download and parse the pcdb which