From 34d6a075289b0c2d31d75a1bad8ea5c969f12fca Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 17:07:42 +0100 Subject: [PATCH] Pushed non-intrusive survey results to bd --- .../app/db/functions/non_intrusive_surveys.py | 50 ++++++++++ .../app/db/models/non_intrusive_surveys.py | 8 +- etl/customers/immo/pilot/non_invasive.py | 99 +++++++++++-------- 3 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 backend/app/db/functions/non_intrusive_surveys.py diff --git a/backend/app/db/functions/non_intrusive_surveys.py b/backend/app/db/functions/non_intrusive_surveys.py new file mode 100644 index 00000000..93348121 --- /dev/null +++ b/backend/app/db/functions/non_intrusive_surveys.py @@ -0,0 +1,50 @@ +from sqlalchemy.orm import Session +from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes + + +def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500): + """ + Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents + one survey and its associated notes. + + :param session: SQLAlchemy Session object through which all database transactions are handled. + :param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn', + 'survey_date', 'surveyor', and other notes as key-value pairs. + :param batch_size: The size of each batch to be processed (default is 500). + :return: None + """ + + # Helper function to process each batch + def process_batch(batch): + surveys = [] + notes = [] + + for note in batch: + survey = NonIntrusiveSurvey( + uprn=note['uprn'], + survey_date=note['survey_date'], + surveyor=note['surveyor'] + ) + surveys.append(survey) + + session.add_all(surveys) + session.flush() # Get IDs for surveys + + for note, survey in zip(batch, surveys): + for key, value in note.items(): + if key not in ['uprn', 'survey_date', 'surveyor']: + notes.append(NonIntrusiveSurveyNotes( + survey_id=survey.id, + title=key, + note=value + )) + + session.bulk_save_objects(notes) + session.commit() + + # Split the data into batches and process each batch + total = len(non_invasive_notes) + for start in range(0, total, batch_size): + end = min(start + batch_size, total) + batch = non_invasive_notes[start:end] + process_batch(batch) diff --git a/backend/app/db/models/non_intrusive_surveys.py b/backend/app/db/models/non_intrusive_surveys.py index c5f3734a..bc2d8adc 100644 --- a/backend/app/db/models/non_intrusive_surveys.py +++ b/backend/app/db/models/non_intrusive_surveys.py @@ -1,9 +1,5 @@ -from sqlalchemy import Column, BigInteger, String, Float, Boolean, TIMESTAMP, ForeignKey, Enum, Integer +from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer from sqlalchemy.orm import declarative_base -from sqlalchemy.sql import func -from backend.app.db.models.portfolio import Portfolio, PropertyModel -from backend.app.db.models.materials import Material -from datatypes.enums import QuantityUnits Base = declarative_base() @@ -18,6 +14,8 @@ class NonIntrusiveSurvey(Base): class NonIntrusiveSurveyNotes(Base): + __tablename__ = 'non_intrusive_survey_notes' + id = Column(BigInteger, primary_key=True, autoincrement=True) survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False) title = Column(String, nullable=False) diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py index 0a376388..6dc22c62 100644 --- a/etl/customers/immo/pilot/non_invasive.py +++ b/etl/customers/immo/pilot/non_invasive.py @@ -1,4 +1,8 @@ -import extract_msg +# import extract_msg +from datetime import datetime +from sqlalchemy.orm import sessionmaker +from backend.app.db.connection import db_engine +from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes def parse_msg_body(text): @@ -55,24 +59,25 @@ def app(): # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " # "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" # ) - filepath = ( - "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " - "- 27 Milton Road Coseley Bilston WV14 8HZ.msg" - ) - - with extract_msg.Message(filepath) as msg: - body = msg.body - - from pprint import pprint - pprint(parse_msg_body(body)) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " + # "- 27 Milton Road Coseley Bilston WV14 8HZ.msg" + # ) + # + # with extract_msg.Message(filepath) as msg: + # body = msg.body + # + # from pprint import pprint + # pprint(parse_msg_body(body)) # We manually create the non-invasive notes for the pilot non_invasive_notes = [ { 'uprn': 90028499, - 'address': '5 Oaklands', - 'postcode': 'B62 0JA', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '5 Oaklands', + # 'postcode': 'B62 0JA', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation. ' 'There is a shared alleyway with the neighbour, that is a solid brick wall.', @@ -83,9 +88,10 @@ def app(): }, { 'uprn': 90055152, - 'address': '6 Beech Road', - 'postcode': 'DY1 4BP', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '6 Beech Road', + # 'postcode': 'DY1 4BP', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, ' 'retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', @@ -96,9 +102,10 @@ def app(): }, { 'uprn': 90070461, - 'address': '8 Corporation Road', - 'postcode': 'DY2 7PX', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '8 Corporation Road', + # 'postcode': 'DY2 7PX', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': "External wall insulation", 'Wall Render': "Render finish throughout", 'Existing solar PV': 'No existing solar', @@ -107,9 +114,10 @@ def app(): }, { 'uprn': 90022227, - 'address': '21 Wells Road', - 'postcode': 'DY5 3TB', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '21 Wells Road', + # 'postcode': 'DY5 3TB', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': None, @@ -119,9 +127,10 @@ def app(): }, { 'uprn': 90077535, - 'address': '47 Fairfield Road', - 'postcode': 'DY8 5UJ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '47 Fairfield Road', + # 'postcode': 'DY8 5UJ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': None, @@ -131,9 +140,10 @@ def app(): }, { 'uprn': 90060989, - 'address': '53 Bromley', - 'postcode': 'DY5 4PJ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '53 Bromley', + # 'postcode': 'DY5 4PJ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - " "recommends a cavity wall fill", "Roof": "Hipped roof", @@ -143,9 +153,10 @@ def app(): }, { 'uprn': 90048026, - 'address': '91 Osprey Drive', - 'postcode': 'DY1 2JS', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '91 Osprey Drive', + # 'postcode': 'DY1 2JS', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': 'Tile hung front and rear of property', @@ -155,9 +166,10 @@ def app(): }, { 'uprn': 90093693, - 'address': '150 Huntingtree Road', - 'postcode': 'B63 4HP', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '150 Huntingtree Road', + # 'postcode': 'B63 4HP', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Heating': 'Electric (storage heaters)', 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', @@ -168,9 +180,10 @@ def app(): }, { 'uprn': 90051858, - 'address': '195 Ashenhurst Road', - 'postcode': 'DY1 2JB', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '195 Ashenhurst Road', + # 'postcode': 'DY1 2JB', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': "Solid render front and rear of property", @@ -180,9 +193,10 @@ def app(): }, { 'uprn': 90106884, - 'address': '27 Milton Road', - 'postcode': 'WV14 8HZ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '27 Milton Road', + # 'postcode': 'WV14 8HZ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': "Solid render front and rear of property", @@ -192,4 +206,5 @@ def app(): }, ] - # TODO: Push the non-invasive results straight to the database from here + session = sessionmaker(bind=db_engine)() + upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500)