diff --git a/etl/xml_survey_extraction/XmlParser.py b/etl/xml_survey_extraction/XmlParser.py index 4ca1eb50..7f317f29 100644 --- a/etl/xml_survey_extraction/XmlParser.py +++ b/etl/xml_survey_extraction/XmlParser.py @@ -1,5 +1,6 @@ import re import usaddress +from datetime import datetime from xml.dom.minidom import parseString from backend.app.utils import sap_to_epc from etl.xml_survey_extraction.pcdb import heating_data @@ -119,6 +120,8 @@ class XmlParser: floor_height = None insulation_wall_area = None + floor_dimensions = None + rrn = None database_data = None @@ -230,7 +233,9 @@ class XmlParser: self.epc = { "uprn": self.uprn, + "uprn-source": "Address Matched", "property-type": property_type, + "building-reference-number": "", **self.get_sap(), **self.get_property_address(), "low-energy-fixed-light-count": self.get_node_value('Low-Energy-Fixed-Lighting-Outlets-Count'), @@ -252,7 +257,7 @@ class XmlParser: "mainheatcont-description": self.get_property_summary_value('Main-Heating-Controls', 'Description'), "sheating-energy-eff": self.RATINGS_MAP[ - self.get_property_summary_value('Secondary-Heating', 'Energy-Efficiency-Rating'), + self.get_property_summary_value('Secondary-Heating', 'Energy-Efficiency-Rating') ], "local-authority": "", # Not included in the xml "local-authority-label": "", @@ -304,7 +309,7 @@ class XmlParser: "sheating-env-eff": self.RATINGS_MAP[ self.get_property_summary_value('Secondary-Heating', 'Environmental-Efficiency-Rating') ], - "lighting_description": self.get_property_summary_value('Lighting', 'Description'), + "lighting-description": self.get_property_summary_value('Lighting', 'Description'), "roof-env-eff": self.RATINGS_MAP[ self.get_property_summary_value('Roof', 'Environmental-Efficiency-Rating') ], @@ -329,10 +334,11 @@ class XmlParser: self.get_property_summary_value('Main-Heating-Controls', 'Environmental-Efficiency-Rating') ], "lmk-key": "", # Doesn't exist for non-EPC xmls - "wind-turbines-count": self.get_node_value('Wind-Turbines-Count'), + "wind-turbine-count": self.get_node_value('Wind-Turbines-Count'), "tenure": self.TENURE_MAP[self.get_node_value('Tenure')], "floor-level": floor_level, "potential-energy-efficiency": self.get_energy_assessment_value('Energy-Rating-Potential'), + "potentual-energy-rating": sap_to_epc(float(self.get_energy_assessment_value('Energy-Rating-Potential'))), "hot-water-energy-eff": self.RATINGS_MAP[ self.get_property_summary_value('Hot-Water', 'Energy-Efficiency-Rating') ], @@ -341,10 +347,15 @@ class XmlParser: "hotwater-description": self.get_property_summary_value('Hot-Water', 'Description'), "co2-emissions-current": self.get_node_value('CO2-Emissions-Current'), "heating-cost-current": self.get_node_value('Heating-Cost-Current'), + "heating-cost-potential": self.get_energy_assessment_value('Heating-Cost-Potential'), "hot-water-cost-current": self.get_node_value('Hot-Water-Cost-Current'), + "hot-water-cost-potential": self.get_energy_assessment_value('Hot-Water-Cost-Potential'), "lighting-cost-current": self.get_node_value('Lighting-Cost-Current'), "energy-consumption-current": self.get_node_value('Energy-Consumption-Current'), "lodgement-date": self.get_node_value('Inspection-Date'), + "lodgement-datetime": + datetime.strptime(self.get_node_value('Inspection-Date'), "%Y-%m-%d").isoformat(), + "mainheat-description": self.get_property_summary_value('Main-Heating', 'Description'), } @@ -511,8 +522,15 @@ class XmlParser: posttown = self.get_node(property_tag.getElementsByTagName("Post-Town")[0]) postcode = self.get_node(property_tag.getElementsByTagName("Postcode")[0]) address = ", ".join( - [x for x in [self.address1, self.address2, self.address3, self.posttown, self.postcode] if x is not None] + [x for x in [address1, address2, address3] if x is not None] ) + county = property_tag.getElementsByTagName("County") + if county: + county = county[0].firstChild.nodeValue + + # Seems to be unavailable in the xml + constituency = None + constituency_label = None return { "address1": address1, @@ -520,7 +538,10 @@ class XmlParser: "address3": address3, "posttown": posttown, "postcode": postcode, - "address": address + "address": address, + "county": county, + "constituency": constituency, + "constituency-label": constituency_label } def get_property_dimensions(self): @@ -572,3 +593,41 @@ class XmlParser: self.insulation_wall_area = self.heat_loss_perimeter * self.floor_height * self.INSULATION_WALL_AREA_FACTOR self.perimeter = self.heat_loss_perimeter + self.party_wall_length + + def get_floor_dimensions(self): + + """ + Extracts physical measurements of the property such as the floor area, room height, etc. + across the main dwelling and any extensions. + :return: + """ + + def get_part_value(node, tag_name): + element = node.getElementsByTagName(tag_name) + if element and element[0].firstChild: + return element[0].firstChild.nodeValue + return None + + # Each part will correspond to the main + sap_building_parts = self.xml.getElementsByTagName("SAP-Building-Part") + + floor_dimensions = [] + for building_part in sap_building_parts: + building_part_identifier = building_part.getElementsByTagName("Identifier")[0].firstChild.nodeValue + sap_floor_dimensions = building_part.getElementsByTagName("SAP-Floor-Dimension") + + data = [ + { + 'building_part_identifier': building_part_identifier, + 'floor': get_part_value(floor_dimension, 'Floor'), + 'floor_construction': get_part_value(floor_dimension, 'Floor-Construction'), + 'floor_insulation': get_part_value(floor_dimension, 'Floor-Insulation'), + 'heat_loss-perimeter': get_part_value(floor_dimension, 'Heat-Loss-Perimeter'), + 'party_wall-length': get_part_value(floor_dimension, 'Party-Wall-Length'), + 'total_floor-area': get_part_value(floor_dimension, 'Total-Floor-Area'), + 'room_height': get_part_value(floor_dimension, 'Room-Height') + } for floor_dimension in sap_floor_dimensions + ] + floor_dimensions.extend(data) + + self.floor_dimensions = floor_dimensions diff --git a/etl/xml_survey_extraction/app.py b/etl/xml_survey_extraction/app.py index 9bcbb168..c70097d4 100644 --- a/etl/xml_survey_extraction/app.py +++ b/etl/xml_survey_extraction/app.py @@ -9,6 +9,7 @@ logger = setup_logger() SURVEYORS = "JAFFERSONS ENERGY CONSULTANTS" PROJECT_CODE = "VDE001" BUCKET = "retrofit-energy-assessments-dev" +PORTFOLIO_ID = None def main(): @@ -48,3 +49,5 @@ def main(): xml_parser = XmlParser(file=xml_data_io, filekey=xml, uprn=uprn) xml_parser.run() logger.info(f"Extracted data from {xml}") + + # TODO: Set a portfolio ID, Target and Automatically upload the asset list and create the event for the portfolio