junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-12 13:29:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Pushed non-intrusive survey results to bd

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-04-13 17:07:42 +01:00
parent 527291b439
commit 34d6a07528
3 changed files with 110 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
backend/app/db/functions/non_intrusive_surveys.py Normal file
View file

@ -0,0 +1,50 @@
from sqlalchemy.orm import Session
from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes
def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500):
"""
Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents
one survey and its associated notes.
:param session: SQLAlchemy Session object through which all database transactions are handled.
:param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn',
'survey_date', 'surveyor', and other notes as key-value pairs.
:param batch_size: The size of each batch to be processed (default is 500).
:return: None
"""
# Helper function to process each batch
def process_batch(batch):
surveys = []
notes = []
for note in batch:
survey = NonIntrusiveSurvey(
uprn=note['uprn'],
survey_date=note['survey_date'],
surveyor=note['surveyor']
)
surveys.append(survey)
session.add_all(surveys)
session.flush() # Get IDs for surveys
for note, survey in zip(batch, surveys):
for key, value in note.items():
if key not in ['uprn', 'survey_date', 'surveyor']:
notes.append(NonIntrusiveSurveyNotes(
survey_id=survey.id,
title=key,
note=value
))
session.bulk_save_objects(notes)
session.commit()
# Split the data into batches and process each batch
total = len(non_invasive_notes)
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
batch = non_invasive_notes[start:end]
process_batch(batch)

8
backend/app/db/models/non_intrusive_surveys.py
View file

@ -1,9 +1,5 @@
from sqlalchemy import Column, BigInteger, String, Float, Boolean, TIMESTAMP, ForeignKey, Enum, Integer from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer
from sqlalchemy.orm import declarative_base from sqlalchemy.orm import declarative_base
from sqlalchemy.sql import func
from backend.app.db.models.portfolio import Portfolio, PropertyModel
from backend.app.db.models.materials import Material
from datatypes.enums import QuantityUnits
Base = declarative_base() Base = declarative_base()
@ -18,6 +14,8 @@ class NonIntrusiveSurvey(Base):
class NonIntrusiveSurveyNotes(Base): class NonIntrusiveSurveyNotes(Base):
__tablename__ = 'non_intrusive_survey_notes'
id = Column(BigInteger, primary_key=True, autoincrement=True) id = Column(BigInteger, primary_key=True, autoincrement=True)
survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False) survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False)
title = Column(String, nullable=False) title = Column(String, nullable=False)

99
etl/customers/immo/pilot/non_invasive.py
View file

@ -1,4 +1,8 @@
import extract_msg # import extract_msg
from datetime import datetime
from sqlalchemy.orm import sessionmaker
from backend.app.db.connection import db_engine
from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes
def parse_msg_body(text): def parse_msg_body(text):
@ -55,24 +59,25 @@ def app():
# "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 "
# "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" # "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg"
# ) # )
filepath = ( # filepath = (
"/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO "
"- 27 Milton Road Coseley Bilston WV14 8HZ.msg" # "- 27 Milton Road Coseley Bilston WV14 8HZ.msg"
) # )
#
with extract_msg.Message(filepath) as msg: # with extract_msg.Message(filepath) as msg:
body = msg.body # body = msg.body
#
from pprint import pprint # from pprint import pprint
pprint(parse_msg_body(body)) # pprint(parse_msg_body(body))
# We manually create the non-invasive notes for the pilot # We manually create the non-invasive notes for the pilot
non_invasive_notes = [ non_invasive_notes = [
{ {
'uprn': 90028499, 'uprn': 90028499,
'address': '5 Oaklands', # 'address': '5 Oaklands',
'postcode': 'B62 0JA', # 'postcode': 'B62 0JA',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation. ' 'CIGA check and extracting the cavity, replacing with bead insulation. '
'There is a shared alleyway with the neighbour, that is a solid brick wall.', 'There is a shared alleyway with the neighbour, that is a solid brick wall.',
@ -83,9 +88,10 @@ def app():
}, },
{ {
'uprn': 90055152, 'uprn': 90055152,
'address': '6 Beech Road', # 'address': '6 Beech Road',
'postcode': 'DY1 4BP', # 'postcode': 'DY1 4BP',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, ' 'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, '
'retro drilled, containing loose fibre insulation. Consider getting a ' 'retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
@ -96,9 +102,10 @@ def app():
}, },
{ {
'uprn': 90070461, 'uprn': 90070461,
'address': '8 Corporation Road', # 'address': '8 Corporation Road',
'postcode': 'DY2 7PX', # 'postcode': 'DY2 7PX',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': "External wall insulation", 'Wall Insulation': "External wall insulation",
'Wall Render': "Render finish throughout", 'Wall Render': "Render finish throughout",
'Existing solar PV': 'No existing solar', 'Existing solar PV': 'No existing solar',
@ -107,9 +114,10 @@ def app():
}, },
{ {
'uprn': 90022227, 'uprn': 90022227,
'address': '21 Wells Road', # 'address': '21 Wells Road',
'postcode': 'DY5 3TB', # 'postcode': 'DY5 3TB',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
'Wall Render': None, 'Wall Render': None,
@ -119,9 +127,10 @@ def app():
}, },
{ {
'uprn': 90077535, 'uprn': 90077535,
'address': '47 Fairfield Road', # 'address': '47 Fairfield Road',
'postcode': 'DY8 5UJ', # 'postcode': 'DY8 5UJ',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
'Wall Render': None, 'Wall Render': None,
@ -131,9 +140,10 @@ def app():
}, },
{ {
'uprn': 90060989, 'uprn': 90060989,
'address': '53 Bromley', # 'address': '53 Bromley',
'postcode': 'DY5 4PJ', # 'postcode': 'DY5 4PJ',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - " 'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - "
"recommends a cavity wall fill", "recommends a cavity wall fill",
"Roof": "Hipped roof", "Roof": "Hipped roof",
@ -143,9 +153,10 @@ def app():
}, },
{ {
'uprn': 90048026, 'uprn': 90048026,
'address': '91 Osprey Drive', # 'address': '91 Osprey Drive',
'postcode': 'DY1 2JS', # 'postcode': 'DY1 2JS',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
'Wall Render': 'Tile hung front and rear of property', 'Wall Render': 'Tile hung front and rear of property',
@ -155,9 +166,10 @@ def app():
}, },
{ {
'uprn': 90093693, 'uprn': 90093693,
'address': '150 Huntingtree Road', # 'address': '150 Huntingtree Road',
'postcode': 'B63 4HP', # 'postcode': 'B63 4HP',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Heating': 'Electric (storage heaters)', 'Heating': 'Electric (storage heaters)',
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
@ -168,9 +180,10 @@ def app():
}, },
{ {
'uprn': 90051858, 'uprn': 90051858,
'address': '195 Ashenhurst Road', # 'address': '195 Ashenhurst Road',
'postcode': 'DY1 2JB', # 'postcode': 'DY1 2JB',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
'Wall Render': "Solid render front and rear of property", 'Wall Render': "Solid render front and rear of property",
@ -180,9 +193,10 @@ def app():
}, },
{ {
'uprn': 90106884, 'uprn': 90106884,
'address': '27 Milton Road', # 'address': '27 Milton Road',
'postcode': 'WV14 8HZ', # 'postcode': 'WV14 8HZ',
'Surveyor': 'Carl Fitzgerald - The Warmfront Team', 'surveyor': 'Carl Fitzgerald - The Warmfront Team',
'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
'CIGA check and extracting the cavity, replacing with bead insulation.', 'CIGA check and extracting the cavity, replacing with bead insulation.',
'Wall Render': "Solid render front and rear of property", 'Wall Render': "Solid render front and rear of property",
@ -192,4 +206,5 @@ def app():
}, },
] ]
# TODO: Push the non-invasive results straight to the database from here session = sessionmaker(bind=db_engine)()
upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500)