From d8caacae97006638aed112e7c8682a0a23372690 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 17:46:06 +0100 Subject: [PATCH] creating non-invasive survey results WIP --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- etl/customers/immo/pilot/non_invasive.py | 131 ++++++++++++++++++++++ etl/customers/immo/pilot/requirements.txt | 1 + 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 etl/customers/immo/pilot/non_invasive.py create mode 100644 etl/customers/immo/pilot/requirements.txt diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py new file mode 100644 index 00000000..cb978059 --- /dev/null +++ b/etl/customers/immo/pilot/non_invasive.py @@ -0,0 +1,131 @@ +import extract_msg + + +def parse_msg_body(text): + # Split the text into lines + lines = text.split('\r\n') + + # Dictionary to hold the parsed data + data = {} + + # Process each line + for line in lines: + # Remove all asterisks and extra whitespace + clean_line = line.replace('*', '').strip() + + if clean_line: # Ensure the line is not empty after cleaning + # Attempt to split clean '=' if present + if '=' in clean_line: + clean_line = clean_line.replace(' = ', ': ') + + # Use line content as a key with a default value indicating presence + # Generate a unique key for lines without '=' + data[f"Info{len(data) + 1}"] = clean_line + + return data + + +def app(): + """ + This code retrieves the results of the non-invasive surveys, to be stored in S3 + :return: + """ + + # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/5 Oaklands B62 " + # "0JA/Immo - 5 Oaklands Halesowen B62 0JA.msg") + # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/6 Beech Rd DY1 " + # "4BP/IMMO - 6 Beech Road Dudley DY1 4BP.msg") + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/8 Corporation Rd DY2 " + # "7PX/IMMO - 8 Corporation Road Dudley DY2 7PX.msg" + # ) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/21 Wells Rd DY5 3TB/" + # "IMMO - 21 Wells Road Brierley Hill DY5 3TB.msg" + # ) + filepath = ( + "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/47 Fairfield Rd DY8 " + "5UJ/IMMO - 47 Fairfield Road Wordsley Stourbridge DY8 5UJ.msg" + ) + + with extract_msg.Message(filepath) as msg: + sender = msg.sender + recipients = msg.to + subject = msg.subject + body = msg.body + # If the msg has attachments, they can be extracted as well + attachments = msg.attachments + + from pprint import pprint + pprint(parse_msg_body(body)) + + # We manually create the non-invasive notes for the pilot + non_invasive_notes = [ + { + 'address': '5 Oaklands', + 'postcode': 'B62 0JA', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation. ' + 'There is a shared alleyway with the neighbour, that is a solid brick wall.', + 'Wall Render': 'Partial render between top of ground floor window and bottom of 1st floor window', + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: North East, Back house direction: South West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '6 Beech Road', + 'postcode': 'DY1 4BP', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, ' + 'retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Side house direction: North East', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '8 Corporation Road', + 'postcode': 'DY2 7PX', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': "External wall insulation", + 'Wall Render': "Render finish throughout", + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: North East, Back house direction: South West', + 'Access to mains?': None, + }, + { + + 'address': '21 Wells Road', + 'postcode': 'DY5 3TB', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: East, Back house direction: West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '47 Fairfield Road', + 'postcode': 'DY8 5UJ', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: East, Back house direction: West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': None, + 'postcode': None, + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': None, + 'Wall Render': None, + 'Existing solar PV': None, + 'Orientation': None, + 'Access to mains?': None, + }, + ] diff --git a/etl/customers/immo/pilot/requirements.txt b/etl/customers/immo/pilot/requirements.txt new file mode 100644 index 00000000..4673ab35 --- /dev/null +++ b/etl/customers/immo/pilot/requirements.txt @@ -0,0 +1 @@ +extract-msg