diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index c2784eb1..06ec5907 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -598,7 +598,35 @@ class AssetList: self.standardised_asset_list[self.landlord_year_built].dt.year ) else: - raise NotImplementedError("Year built column must be a datetime - implement me") + # We attempt to convert the year built to a datetime, by detecting the format and converting + + def extract_year(date_str): + """ + Extracts the year from a date string in the format '01-Jul-YYYY'. + Returns the extracted year as an integer or None if the format is incorrect. + """ + known_errors = ["#MULTIVALUE"] + + if pd.isnull(date_str) or date_str in known_errors: + return None + + if isinstance(date_str, str): + match = re.match(r"\d{1,2}-[A-Za-z]{3}-(\d{4})", date_str) + if match: + return int(match.group(1)) # Extract the year and convert to integer + + if isinstance(date_str, datetime): + return date_str.year + + # Check if date_str is a year itself + if str(date_str).isdigit() & (len(str(date_str)) == 4): + return int(date_str) + + raise NotImplementedError("Unhandled format for year built - implement me") + + self.standardised_asset_list[self.landlord_year_built] = self.standardised_asset_list[ + self.landlord_year_built + ].apply(extract_year) # We now create standard lookups to_remap = { @@ -619,6 +647,8 @@ class AssetList: "standard_map": existing_pv_mappings.EXISTING_PV_MAPPINGS } } + # Keep just entries where the key is not None + to_remap = {k: v for k, v in to_remap.items() if k is not None} for variable, config in to_remap.items(): logger.info("Standardising variable: %s", variable) diff --git a/asset_list/app.py b/asset_list/app.py index 89b15c06..1cb7808e 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -255,7 +255,7 @@ def app(): landlord_wall_construction = "Wallinsul" landlord_heating_system = "HeatSorc" landlord_existing_pv = None - landlord_property_id = None + landlord_property_id = "Property Reference" # Maps addresses to uprn in problematic cases MANUAL_UPRN_MAP = {} diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index 89bfe0c4..b58f13f2 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -1,3 +1,5 @@ +import numpy as np + STANDARD_HEATING_SYSTEMS = { "gas combi boiler", "electric storage heaters", @@ -35,12 +37,31 @@ HEATING_MAPPINGS = { "Eco Electric Radiators": "electric radiators", "Gas fire": "other", "Backboiler - Solid fuel": "other", - 'combi - gas': 'gas combi boiler', 'e7 storage heaters': 'electric storage heaters', - 'district heating system': 'district heating', 'condensing boiler - gas': 'gas condensing boiler', - 'boiler oil/other': 'oil boiler', 'condensing combi - gas': 'gas condensing combi', - 'air source source heat pump': 'air source heat pump', 'biomass boiler': 'boiler - other fuel', - 'ground source heat pump': 'ground source heat pump', 'electric oil filled radiators': 'electric radiators', - 'solid fuel': 'other', 'lpg boiler': 'boiler - other fuel', 'electric boiler': 'electric boiler', + 'combi - gas': 'gas combi boiler', + 'e7 storage heaters': 'electric storage heaters', + 'district heating system': 'district heating', + 'condensing boiler - gas': 'gas condensing boiler', + 'boiler oil/other': 'oil boiler', + 'condensing combi - gas': 'gas condensing combi', + 'air source source heat pump': 'air source heat pump', + 'biomass boiler': 'boiler - other fuel', + 'ground source heat pump': 'ground source heat pump', + 'electric oil filled radiators': 'electric radiators', + 'solid fuel': 'other', + 'lpg boiler': 'boiler - other fuel', + 'electric boiler': 'electric boiler', 'no data': 'unknown', 'boiler communal/commercial - gas': 'communal gas boiler', - 'eco electric radiators': 'electric radiators', 'gas fire': 'other', 'backboiler - solid fuel': 'other', + 'eco electric radiators': 'electric radiators', + 'gas fire': 'other', 'backboiler - solid fuel': 'other', + 'ASHP': 'air source heat pump', + 'COMMHEAT': 'communal gas boiler', + 'GBB': 'gas combi boiler', + 'GFS': 'gas condensing boiler', + 'GWA': 'gas condensing boiler', + 'GWM': 'gas condensing combi', + 'HDU': 'district heating', + 'OILBLR': 'oil boiler', + 'SOLIDFUEL': 'boiler - other fuel', + 'STORHTR': 'high heat retention storage heaters', + np.nan: 'unknown', } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index ec569123..2612f058 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -1,7 +1,7 @@ # These are the standard categories for property types STANDARD_PROPERTY_TYPES = { "house", "flat", "maisonette", "bungalow", "park home", "block house", "bedsit", "coach house", - "unknown", "other" + "unknown", "other", "block of flats" } # This is a basic mapping that we use to map values that we've seen commonly to standard values @@ -15,4 +15,11 @@ PROPERTY_MAPPING = { "BEDSIT": "bedsit", "COACHSE": "coach house", "coachse": "coach house", + 'Admin Unit Type': 'unknown', + 'Block': 'block of flats', + 'Bungalow': 'bungalow', + 'Flat': 'flat', + 'House': 'house', + 'Maisonette': 'maisonette', + 'Stairwell': 'other' } diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index 1fc52fcb..82b31d01 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -1,7 +1,8 @@ STANDARD_WALL_CONSTRUCTIONS = { "uninsulated cavity", "filled cavity", "partial insulated cavity", "cavity unknown insulation", - "timber frame", "uninsulated solid brick", - "insulated solid brick", "system built", "granite or whinstone", "other", "unknown", "sandstone or limestone", + "uninsulated solid brick", "insulated solid brick", "solid brick unknown insulation", + "timber frame", + "system built", "granite or whinstone", "other", "unknown", "sandstone or limestone", "cob", "new build - average thermal transmittance", } @@ -70,8 +71,7 @@ WALL_CONSTRUCTION_MAPPINGS = { 'average thermal transmittance 0.28 w/m?k': 'unknown', 'Cavity wall, filled cavity': 'filled cavity', 'Cavity wall, filled cavity and external insulation': 'filled cavity', - 'Granite or whinstone, as built, no insulation (assumed)': 'granite or ' - 'whinstone', + 'Granite or whinstone, as built, no insulation (assumed)': 'granite or whinstone', 'Solid brick, as built, insulated (assumed)': 'insulated solid brick', 'Solid brick, as built, no insulation (assumed)': 'uninsulated solid brick', 'Solid brick, with external insulation': 'insulated solid brick', @@ -84,4 +84,9 @@ WALL_CONSTRUCTION_MAPPINGS = { 'Timber frame, as built, no insulation (assumed)': 'timber frame', 'Timber frame, as built, partial insulation (assumed)': 'timber frame', 'Timber frame, with additional insulation': 'timber frame', + 'CAVITY': 'partial unknown cavity', + 'COMB': 'unknown', + 'NONE': 'unknown', + 'NOTKNOWN': 'unknown', + 'SOLID': 'solid brick unknown insulation', }