from utils.s3 import read_excel_from_s3 from utils.logger import setup_logger from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes logger = setup_logger() class UploadNonIntrusives: """ This class handles the upload of findings from the non-intrusive surveys, to the database """ COLUMN_PREFIXES: dict = { 'Surveyor First Name': 'Surveyor', 'Surveyor Last Name': 'Surveyor', 'House Number': 'Property Details', 'Address Line 1': 'Property Details', 'Address Line 2': 'Property Details', 'Postcode': 'Property Details', 'Property Year Built': 'Property Details', 'Wall Construction': 'Walls', 'Wall Construction Notes': 'Walls', 'Existing insulation?': 'Walls', 'Retro Drilled?': 'Walls', 'Condition (cracks & damp)': 'Walls', 'Condition Notes': 'Walls', 'Alternative walls': 'Walls', 'Alternative walls percentage': 'Walls', 'Adequate Ventilation?': 'Walls', 'Ventilation notes': 'Walls', 'Party wall': 'Walls', 'Floor Type': 'Floor', 'Wall render': 'Wall Render', 'Wall Render Condition': 'Wall Render', 'Roof Type': 'Roof', 'Roof insulation ': 'Roof', 'Roof Condition': 'Roof', 'Obvious Roof Shading': 'Roof', 'Roof orientation - Primary': 'Roof', 'Roof orientation - Secondary': 'Roof', 'Obstructions on the roof': 'Roof', 'Flue type': 'Heating', 'Is there an extension?': 'Access', 'Are there any out-buildings?': 'Access', 'Is there a conservatory?': 'Access', 'Is the property straight onto a footpath?': 'Access', 'Is there a requirement for planning consent for works?': 'Access', 'Is there space for an external unit?': 'Air Source Heat Pump', 'Could a cylinder fit in the loft?': 'Air Source Heat Pump', 'Are there obvious areas of heat loss from the walls?': 'Thermography', 'Are there obvious areas of heat loss from the roof?': 'Thermography', 'Does the existing insulation exhibit signs of inconsistent performance or underperformance?': 'Thermography', 'Is there excessive levels of heat loss from windows?': 'Thermography', 'Is there excessive levels of heat loss from doors?': 'Thermography', 'Material inside the walls': 'Borescope Test', 'Cavity depth (mm)': 'Borescope Test', 'Is there rubble in the cavity?': 'Borescope Test', 'Wall tie type': 'Borescope Test', 'Wall tie integrity': 'Borescope Test', 'Inner block work': 'Borescope Test', 'Current glazing': 'Windows', 'Windows Age (pre/post 2002)': 'Windows', 'Glazing gap': 'Windows', 'Are there obvious trickle vents in the windows?': 'Windows', 'Is there sufficient space in the garden?': 'Ground Source Heat Pump', 'Does the property need a CIGA check?': 'Funding', 'Is the property eligible for GBIS?': 'Funding', 'Is the property eligible for ECO4?': 'Funding', 'Is the property eligible for the Local Authority Flex Scheme?': 'Funding', 'Is the property eligible for HUG?': 'Funding', 'Is the property eligible for LAD?': 'Funding', 'Other funding recommendations': 'Funding' } def __init__(self, s3_template_location, s3_bucket, uprn_lookup, survey_date): self.s3_template_location = s3_template_location self.s3_bucket = s3_bucket self.template = self.read_template() self.uprn_lookup = uprn_lookup self.survey_date = survey_date def read_template(self): """ This method reads the template from S3 """ return read_excel_from_s3(file_key=self.s3_template_location, bucket_name=self.s3_bucket, header_row=2) def upload(self): """ This method uploads the non-intrusive survey data to the database """ if self.uprn_lookup is None: raise Exception("Implement call to ordnance survey to get uprn lookup data") logger.info("Preparing non-intrusive notes") non_intrusives = self.template.to_dict(orient="records") non_invasive_notes = [] for survey in non_intrusives: # Remove any NAN entries survey_clean = {self.COLUMN_PREFIXES[k] + ": " + k: v for k, v in survey.items() if v == v} uprn_data = [ x for x in self.uprn_lookup if ( str(x['House Number']).strip() == str(survey_clean['Property Details: House Number']).strip() and x['Address Line 1'] == survey_clean['Property Details: Address Line 1'].strip() and x['Address Line 2'] == survey_clean['Property Details: Address Line 2'].strip() and x['Postcode'] == survey_clean['Property Details: Postcode'].strip() ) ] if len(uprn_data) != 1: address = ( str(survey_clean['Property Details: House Number']) + ' ' + survey_clean['Property Details: Address Line 1'] + ' ' + survey_clean['Property Details: Address Line 2'] + ' ' + survey_clean['Property Details: Postcode'] ) raise Exception(f"Failed to find UPRN data for {address}") surveyor = ( survey_clean.pop("Surveyor: Surveyor First Name") + " " + survey_clean.pop("Surveyor: Surveyor Last Name") ) # Include all of the information apart from data that includes the Property details prefix and the # surveyor - we do however include Property Details: Property Year Built notes_to_upload = { k: v for k, v in survey_clean.items() if k == "Property Details: Property Year Built" or ( not k.startswith("Property Details") and not k.startswith("Surveyor") ) } non_invasive_notes.append({ "uprn": uprn_data[0]['uprn'], "surveyor": surveyor, "survey_date": self.survey_date, **notes_to_upload }) # Implement call to upload notes_to_upload to the database logger.info("Uploading non-intrusive notes to the database") session = sessionmaker(bind=db_engine)() upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500)