junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-12 13:29:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

putting together data upload to db

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-07-25 18:43:14 +01:00
parent 6702eb65b0
commit 4192ee7d69
1 changed files with 13 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
etl/xml_survey_extraction/app.py
View file

@ -1,5 +1,5 @@
import pandas as pd from sqlalchemy.orm import sessionmaker
from backend.app.db.connection import db_engine
from utils.s3 import read_from_s3, list_files_and_subfolders_in_s3_folder, list_xmls_in_s3_folder from utils.s3 import read_from_s3, list_files_and_subfolders_in_s3_folder, list_xmls_in_s3_folder
from utils.logger import setup_logger from utils.logger import setup_logger
from etl.xml_survey_extraction.XmlParser import XmlParser from etl.xml_survey_extraction.XmlParser import XmlParser
@ -43,6 +43,7 @@ def main():
logger.info(f"Exatracted XMLS for the energy assessments") logger.info(f"Exatracted XMLS for the energy assessments")
# For each property, we download the xmls and extract the data # For each property, we download the xmls and extract the data
database_data = []
for uprn, xmls in assessments_map.items(): for uprn, xmls in assessments_map.items():
extracted_data = {} extracted_data = {}
for xml in xmls: for xml in xmls:
@ -59,6 +60,16 @@ def main():
extracted_epc = xml_parser.epc extracted_epc = xml_parser.epc
extracted_additional_data = xml_parser.additional_data extracted_additional_data = xml_parser.additional_data
data_to_update = {
**extracted_epc, **extracted_additional_data
}
extracted_data.update(data_to_update)
database_data.append(extracted_data)
logger.info("Uploading data to the database")
session = sessionmaker(bind=db_engine)()
# TODO: Set a portfolio ID, Target and Automatically upload the asset list and create the event for the portfolio # TODO: Set a portfolio ID, Target and Automatically upload the asset list and create the event for the portfolio
# TODO: In order to get the full data associated to the heating system, we need to download and parse the pcdb which # TODO: In order to get the full data associated to the heating system, we need to download and parse the pcdb which