mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
Pushed non-intrusive survey results to bd
This commit is contained in:
Khalim Conn-Kowlessar
parent
527291b439
commit
34d6a07528
3 changed files with 110 additions and 47 deletions
50
backend/app/db/functions/non_intrusive_surveys.py
Normal file
50
backend/app/db/functions/non_intrusive_surveys.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
from sqlalchemy.orm import Session
|
||||
from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes
|
||||
|
||||
|
||||
def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500):
|
||||
"""
|
||||
Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents
|
||||
one survey and its associated notes.
|
||||
|
||||
:param session: SQLAlchemy Session object through which all database transactions are handled.
|
||||
:param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn',
|
||||
'survey_date', 'surveyor', and other notes as key-value pairs.
|
||||
:param batch_size: The size of each batch to be processed (default is 500).
|
||||
:return: None
|
||||
"""
|
||||
|
||||
# Helper function to process each batch
|
||||
def process_batch(batch):
|
||||
surveys = []
|
||||
notes = []
|
||||
|
||||
for note in batch:
|
||||
survey = NonIntrusiveSurvey(
|
||||
uprn=note['uprn'],
|
||||
survey_date=note['survey_date'],
|
||||
surveyor=note['surveyor']
|
||||
)
|
||||
surveys.append(survey)
|
||||
|
||||
session.add_all(surveys)
|
||||
session.flush() # Get IDs for surveys
|
||||
|
||||
for note, survey in zip(batch, surveys):
|
||||
for key, value in note.items():
|
||||
if key not in ['uprn', 'survey_date', 'surveyor']:
|
||||
notes.append(NonIntrusiveSurveyNotes(
|
||||
survey_id=survey.id,
|
||||
title=key,
|
||||
note=value
|
||||
))
|
||||
|
||||
session.bulk_save_objects(notes)
|
||||
session.commit()
|
||||
|
||||
# Split the data into batches and process each batch
|
||||
total = len(non_invasive_notes)
|
||||
for start in range(0, total, batch_size):
|
||||
end = min(start + batch_size, total)
|
||||
batch = non_invasive_notes[start:end]
|
||||
process_batch(batch)
|
||||
|
|
@ -1,9 +1,5 @@
|
|||
from sqlalchemy import Column, BigInteger, String, Float, Boolean, TIMESTAMP, ForeignKey, Enum, Integer
|
||||
from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.sql import func
|
||||
from backend.app.db.models.portfolio import Portfolio, PropertyModel
|
||||
from backend.app.db.models.materials import Material
|
||||
from datatypes.enums import QuantityUnits
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
|
@ -18,6 +14,8 @@ class NonIntrusiveSurvey(Base):
|
|||
|
||||
|
||||
class NonIntrusiveSurveyNotes(Base):
|
||||
__tablename__ = 'non_intrusive_survey_notes'
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False)
|
||||
title = Column(String, nullable=False)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import extract_msg
|
||||
# import extract_msg
|
||||
from datetime import datetime
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from backend.app.db.connection import db_engine
|
||||
from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes
|
||||
|
||||
|
||||
def parse_msg_body(text):
|
||||
|
|
@ -55,24 +59,25 @@ def app():
|
|||
# "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 "
|
||||
# "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg"
|
||||
# )
|
||||
filepath = (
|
||||
"/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO "
|
||||
"- 27 Milton Road Coseley Bilston WV14 8HZ.msg"
|
||||
)
|
||||
|
||||
with extract_msg.Message(filepath) as msg:
|
||||
body = msg.body
|
||||
|
||||
from pprint import pprint
|
||||
pprint(parse_msg_body(body))
|
||||
# filepath = (
|
||||
# "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO "
|
||||
# "- 27 Milton Road Coseley Bilston WV14 8HZ.msg"
|
||||
# )
|
||||
#
|
||||
# with extract_msg.Message(filepath) as msg:
|
||||
# body = msg.body
|
||||
#
|
||||
# from pprint import pprint
|
||||
# pprint(parse_msg_body(body))
|
||||
|
||||
# We manually create the non-invasive notes for the pilot
|
||||
non_invasive_notes = [
|
||||
{
|
||||
'uprn': 90028499,
|
||||
'address': '5 Oaklands',
|
||||
'postcode': 'B62 0JA',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '5 Oaklands',
|
||||
# 'postcode': 'B62 0JA',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation. '
|
||||
'There is a shared alleyway with the neighbour, that is a solid brick wall.',
|
||||
|
|
@ -83,9 +88,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90055152,
|
||||
'address': '6 Beech Road',
|
||||
'postcode': 'DY1 4BP',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '6 Beech Road',
|
||||
# 'postcode': 'DY1 4BP',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, '
|
||||
'retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
|
|
@ -96,9 +102,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90070461,
|
||||
'address': '8 Corporation Road',
|
||||
'postcode': 'DY2 7PX',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '8 Corporation Road',
|
||||
# 'postcode': 'DY2 7PX',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': "External wall insulation",
|
||||
'Wall Render': "Render finish throughout",
|
||||
'Existing solar PV': 'No existing solar',
|
||||
|
|
@ -107,9 +114,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90022227,
|
||||
'address': '21 Wells Road',
|
||||
'postcode': 'DY5 3TB',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '21 Wells Road',
|
||||
# 'postcode': 'DY5 3TB',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
'Wall Render': None,
|
||||
|
|
@ -119,9 +127,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90077535,
|
||||
'address': '47 Fairfield Road',
|
||||
'postcode': 'DY8 5UJ',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '47 Fairfield Road',
|
||||
# 'postcode': 'DY8 5UJ',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
'Wall Render': None,
|
||||
|
|
@ -131,9 +140,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90060989,
|
||||
'address': '53 Bromley',
|
||||
'postcode': 'DY5 4PJ',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '53 Bromley',
|
||||
# 'postcode': 'DY5 4PJ',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - "
|
||||
"recommends a cavity wall fill",
|
||||
"Roof": "Hipped roof",
|
||||
|
|
@ -143,9 +153,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90048026,
|
||||
'address': '91 Osprey Drive',
|
||||
'postcode': 'DY1 2JS',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '91 Osprey Drive',
|
||||
# 'postcode': 'DY1 2JS',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
'Wall Render': 'Tile hung front and rear of property',
|
||||
|
|
@ -155,9 +166,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90093693,
|
||||
'address': '150 Huntingtree Road',
|
||||
'postcode': 'B63 4HP',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '150 Huntingtree Road',
|
||||
# 'postcode': 'B63 4HP',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Heating': 'Electric (storage heaters)',
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
|
|
@ -168,9 +180,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90051858,
|
||||
'address': '195 Ashenhurst Road',
|
||||
'postcode': 'DY1 2JB',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '195 Ashenhurst Road',
|
||||
# 'postcode': 'DY1 2JB',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
'Wall Render': "Solid render front and rear of property",
|
||||
|
|
@ -180,9 +193,10 @@ def app():
|
|||
},
|
||||
{
|
||||
'uprn': 90106884,
|
||||
'address': '27 Milton Road',
|
||||
'postcode': 'WV14 8HZ',
|
||||
'Surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
# 'address': '27 Milton Road',
|
||||
# 'postcode': 'WV14 8HZ',
|
||||
'surveyor': 'Carl Fitzgerald - The Warmfront Team',
|
||||
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
|
||||
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
|
||||
'CIGA check and extracting the cavity, replacing with bead insulation.',
|
||||
'Wall Render': "Solid render front and rear of property",
|
||||
|
|
@ -192,4 +206,5 @@ def app():
|
|||
},
|
||||
]
|
||||
|
||||
# TODO: Push the non-invasive results straight to the database from here
|
||||
session = sessionmaker(bind=db_engine)()
|
||||
upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue