junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tweaked inputs for standardised asset list

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2026-02-20 12:12:43 +00:00
parent 61e9fb28be
commit d51af41125
9 changed files with 298 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="AssetList" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>

288
asset_list/AssetList.py
View file

@ -36,14 +36,13 @@ from dotenv import load_dotenv
logger = setup_logger()
load_dotenv(dotenv_path="../backend/.env")
# OpenAI API Key (set this in your environment variables for security)
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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")
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY",
"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")
class DataRemapper:
def __init__(self, standard_values, standard_map=None, max_tokens=1000):
print(f"{OPENAI_API_KEY}")
"""
Initialize the remapper with standard values and a predefined mapping.
@ -1298,8 +1297,8 @@ class AssetList:
self.standardised_asset_list[
self.ATTRIBUTE_HAS_SOLAR
] = self.standardised_asset_list[
self.FIND_EPC_DATA_NAMES["Solar photovoltaics"]
] | ~self.standardised_asset_list[
self.FIND_EPC_DATA_NAMES["Solar photovoltaics"]
] | ~self.standardised_asset_list[
self.EPC_API_DATA_NAMES["photo-supply"]
].isin(
["0.0", 0, None, "", np.nan]
@ -1317,7 +1316,7 @@ class AssetList:
property_type=(
str(x[self.STANDARD_PROPERTY_TYPE]).title()
if str(x[self.STANDARD_PROPERTY_TYPE]).title()
in accepted_epc_property_types
in accepted_epc_property_types
else (
x[self.EPC_API_DATA_NAMES["property-type"]]
if not pd.isnull(
@ -1375,9 +1374,9 @@ class AssetList:
self.standardised_asset_list.apply(
lambda x: estimate_perimeter(
floor_area=x[self.EPC_API_DATA_NAMES["total-floor-area"]]
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
num_rooms=x[self.EPC_API_DATA_NAMES["number-habitable-rooms"]]
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
),
axis=1,
)
@ -1462,7 +1461,7 @@ class AssetList:
year_lower_bound = (
2007
if x[self.EPC_API_DATA_NAMES["construction-age-band"]]
== "England and Wales: 2007 onwards"
== "England and Wales: 2007 onwards"
else 2012
)
@ -1517,7 +1516,7 @@ class AssetList:
age_band_matches = (
"EPC Age Band Matches Year Built"
if x[self.STANDARD_YEAR_BUILT]
== int(x[self.EPC_API_DATA_NAMES["construction-age-band"]])
== int(x[self.EPC_API_DATA_NAMES["construction-age-band"]])
else "EPC Age Band is different from Year Built"
)
@ -1547,7 +1546,7 @@ class AssetList:
age_band_matches = (
"EPC Age Band Matches Year Built"
if (x[self.STANDARD_YEAR_BUILT] >= float(lower_date))
and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date))
and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date))
else (
"EPC Age Band is older than Year Built"
if x[self.STANDARD_YEAR_BUILT] > float(upper_date)
@ -1719,22 +1718,22 @@ class AssetList:
if self.non_intrusives_present:
if self.new_format_non_insturives_present_v2:
non_intrusives_wall_filter = (
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
["EMPTY", "PARTIAL", "EMPTY CAVITY"]
)
else:
non_intrusives_wall_filter = (
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
["EMPTY", "PARTIAL"]
)
elif self.old_format_non_intrusives_present:
non_intrusives_wall_filter = self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
[
"empty cavity",
"partial fill",
@ -1744,18 +1743,18 @@ class AssetList:
"empty cav",
]
) | (
(
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
.str.strip()
.str.contains("empty cavity|partial fill")
& ~self.standardised_asset_list["non-intrusives: WFT Findings"]
.astype(str)
.str.lower()
.str.strip()
.str.contains("major access issues")
)
)
(
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
.str.strip()
.str.contains("empty cavity|partial fill")
& ~self.standardised_asset_list["non-intrusives: WFT Findings"]
.astype(str)
.str.lower()
.str.strip()
.str.contains("major access issues")
)
)
else:
# We set the filter to False, as we have no non-intrusives
non_intrusives_wall_filter = False
@ -1767,12 +1766,12 @@ class AssetList:
)
else:
year_built_filter = (
self.standardised_asset_list[self.STANDARD_YEAR_BUILT]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
) | (
self.standardised_asset_list["epc_year_upper_bound"]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
)
self.standardised_asset_list[self.STANDARD_YEAR_BUILT]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
) | (
self.standardised_asset_list["epc_year_upper_bound"]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
)
# Criteria:
# The property isn't a bedsit
@ -1813,8 +1812,8 @@ class AssetList:
] = (
~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity_has_solar"
]
"non_intrusive_indicates_empty_cavity_has_solar"
]
& (
~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(
["bedsit"]
@ -1890,8 +1889,8 @@ class AssetList:
.str.lower()
.isin(self.EPC_NO_WALL_INSULATION_DESCRIPTIONS)
| self.standardised_asset_list[self.STANDARD_WALL_CONSTRUCTION].isin(
["uninsulated cavity"]
)
["uninsulated cavity"]
)
)
######################################################
@ -1928,8 +1927,8 @@ class AssetList:
extraction_wall_filter = (
extraction_wall_filter
& ~self.standardised_asset_list[
"non-intrusives: Eligibility (Red/Yellow/Green)"
].isin(["RED"])
"non-intrusives: Eligibility (Red/Yellow/Green)"
].isin(["RED"])
)
self.standardised_asset_list[
@ -2025,26 +2024,26 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_data_indicates_correct_heating_system"
] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains(
"air source heat pump|ground source heat pump|boiler and radiators, electric"
)
) | (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
self.EPC_API_DATA_NAMES["mainheat-description"]
]
== "Controls for high heat retention storage heaters"
.str.lower()
.str.contains(
"air source heat pump|ground source heat pump|boiler and radiators, electric"
)
) | (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
]
== "Controls for high heat retention storage heaters"
)
)
)
# If the landlord has given us the heating system, we default to that on heating upgrades. Because of the
# poor heating in place, if the EPC indicates that this property had a low efficiency heating system but the
@ -2052,25 +2051,25 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_data_indicates_requires_heating_upgrade"
] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters|room heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
self.EPC_API_DATA_NAMES["mainheat-description"]
]
!= "Controls for high heat retention storage heaters"
.str.lower()
.str.contains("electric storage heaters|room heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
]
!= "Controls for high heat retention storage heaters"
)
) & (
~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
["district heating", "communal heating", "communal gas boiler"]
)
& ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
.astype(str)
.str.contains("gas ")
)
) & (
~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
["district heating", "communal heating", "communal gas boiler"]
)
& ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
.astype(str)
.str.contains("gas ")
)
# Basic check - both of the previous two shouldn't be true simultaneously
if (
@ -2150,8 +2149,8 @@ class AssetList:
self.standardised_asset_list[
"solar_non_intrusives_walls_insulated"
] = self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
[
"retro drilled",
"retro filled",
@ -2160,8 +2159,8 @@ class AssetList:
"retro drilled and filled",
]
) | self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().str.contains(
"non-intrusives: WFT Findings"
].str.lower().str.strip().str.contains(
"retro drilled"
)
else:
@ -2178,14 +2177,19 @@ class AssetList:
)
self.standardised_asset_list["solar_epc_walls_insulated"] = (
self.standardised_asset_list[self.EPC_API_DATA_NAMES["walls-description"]]
.str.lower()
.str.contains("|".join(self.EPC_INSULATED_WALLS_SUBSTRINGS))
) | (
self.standardised_asset_list["walls_u_value"].apply(
lambda x: x <= 0.7 if not pd.isnull(x) else False
)
)
self.standardised_asset_list[
self.EPC_API_DATA_NAMES[
"walls-description"]]
.str.lower()
.str.contains("|".join(
self.EPC_INSULATED_WALLS_SUBSTRINGS))
) | (
self.standardised_asset_list[
"walls_u_value"].apply(
lambda x: x <= 0.7 if not pd.isnull(
x) else False
)
)
roof_data = []
for desc in self.standardised_asset_list[
@ -2227,20 +2231,20 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_loft_needs_topup"
] = self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].apply(
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].apply(
lambda x: int(x) < 200 if str(x).isdigit() else False
) | (
(
self.standardised_asset_list["is_loft"]
| self.standardised_asset_list["is_pitched"]
(
self.standardised_asset_list["is_loft"]
| self.standardised_asset_list["is_pitched"]
)
& (
self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].isin(["below average", "none"])
)
)
& (
self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].isin(["below average", "none"])
)
)
self.standardised_asset_list["epc_has_floor_recommendation"] = (
self.standardised_asset_list["epc_has_floor_recommendation"].fillna(False)
@ -2249,15 +2253,16 @@ class AssetList:
# Check if the boiler is electric
# We check if it contains both the terms boiler & electric
self.standardised_asset_list["has_electric_boiler"] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.isin(["boiler and radiators, electric"])
) | (
self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
== "electric boiler"
)
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.isin(["boiler and radiators, electric"])
) | (
self.standardised_asset_list[
self.STANDARD_HEATING_SYSTEM]
== "electric boiler"
)
####################################
# Check solar eligibility
@ -2395,11 +2400,11 @@ class AssetList:
empty_cavity_map = {
"non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE
+ ": ",
+ ": ",
"non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property "
"already has solar: ",
"already has solar: ",
"non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, "
f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
}
for variable, description in empty_cavity_map.items():
self.standardised_asset_list["cavity_reason"] = np.where(
@ -2415,8 +2420,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
@ -2441,8 +2446,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& self.standardised_asset_list[
"non_intrusive_indicates_cavity_extraction"
]
@ -2457,8 +2462,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: Insulated"]
== "RETRO DRILLED"
@ -2474,8 +2479,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: Insulated"]
== "FILLED AT BUILD"
@ -2491,8 +2496,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& pd.isnull(self.standardised_asset_list["cavity_reason"])
),
f"{self.EPC_EMPTY}: " + self.standardised_asset_list["SAP Category"],
@ -2636,7 +2641,7 @@ class AssetList:
identified_work = self.standardised_asset_list[
~pd.isnull(self.standardised_asset_list["cavity_reason"])
| ~pd.isnull(self.standardised_asset_list["solar_reason"])
][self.DOMNA_PROPERTY_ID].values
][self.DOMNA_PROPERTY_ID].values
if self.DOMNA_PROPERTY_ID in self.outcomes.columns:
self.outcomes_for_output = self.outcomes[
@ -2671,12 +2676,12 @@ class AssetList:
blocks_of_flats = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
== "block of flats"
]
]
non_blocks_of_flats = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
!= "block of flats"
]
]
# Produce some aggregate figures
self.work_type_figures = {
@ -2719,7 +2724,7 @@ class AssetList:
blocks = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
== "block of flats"
].copy()
].copy()
if blocks.empty:
return
@ -2856,7 +2861,7 @@ class AssetList:
self.standardised_asset_list = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
!= "block of flats"
]
]
self.standardised_asset_list = pd.concat(
[self.standardised_asset_list, expanded_blocks], ignore_index=True
@ -2936,7 +2941,7 @@ class AssetList:
# find any block refs with more than 50% emptires
viable_empty_blocks = self.block_analysis_df[
self.block_analysis_df["Percentage of Empties"] >= 0.50
]
]
if not viable_empty_blocks.empty:
project_code_lookup = viable_empty_blocks[["Block Reference"]].copy()
@ -3175,7 +3180,7 @@ class AssetList:
contact_details = pd.read_excel(local_filepath, sheet_name=sheet_name)[
[self.contact_detail_fields["landlord_property_id"]] + details_colnames
]
]
contact_details = contact_details[
~pd.isnull(
contact_details[self.contact_detail_fields["landlord_property_id"]]
@ -3568,10 +3573,13 @@ class AssetList:
"Non-Intrusives: Date Checked <LISTING non_intrusives__date_checked>": date_of_inspections,
"Non-Intrusives: Wall Type <LISTING non_intrusives__wall_type>": non_intrusives_construction,
"Non-intrusives: Insulation <LISTING non_intrusives__insulation>": non_intrusives_insulated,
"Non-intrusives: Insulation Material <LISTING non_intrusives__insulation_material>": non_intrusives_insulation_material,
"Non-Intrusives: CIGA Check Required <LISTING non_intrusives__ciga_check_required>": non_intrusives_ciga_check_required,
"Non-intrusives: Insulation Material <LISTING non_intrusives__insulation_material>":
non_intrusives_insulation_material,
"Non-Intrusives: CIGA Check Required <LISTING non_intrusives__ciga_check_required>":
non_intrusives_ciga_check_required,
"Non-Intrusives: PV Access Issues <LISTING non_intrusives__access_issues>": non_intrusives_pv_access,
"Non-Intrusives: Roof Orientation <LISTING non_intrusives__roof_orientation>": non_intrusives_roof_orientation,
"Non-Intrusives: Roof Orientation <LISTING non_intrusives__roof_orientation>":
non_intrusives_roof_orientation,
"Non-Intrusives: Surveyor Notes <LISTING non_intrusives__surveyor_notes>": non_intrusives_surveyor_notes,
"Non-Intrusives: Surveyor Name <LISTING non_intrusives__surveyor_name>": non_intrusives_surveyor_name,
"CIGA: Date Requested <LISTING ciga__date_requested>": None, # TODO: Don't have this for the moment
@ -3748,8 +3756,8 @@ class AssetList:
# We compare address line 1 to full address
if any(
df[self.STANDARD_FULL_ADDRESS]
.str.lower()
.str.contains(row["Address Line 1"].lower(), na=False)
.str.lower()
.str.contains(row["Address Line 1"].lower(), na=False)
):
df = df[
df[self.STANDARD_FULL_ADDRESS]
@ -3989,7 +3997,7 @@ class AssetList:
matched = matched[
matched["houseno"].astype(str) == house_no_to_match
]
]
if matched.shape[0] == 1:
lookup_i.append(
{
@ -4014,7 +4022,7 @@ class AssetList:
)[0]
matched = matched[
matched[self.STANDARD_FULL_ADDRESS] == best_match
]
]
lookup_i.append(
{
"row_id": x["row_id"],
@ -4325,7 +4333,7 @@ class AssetList:
df = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_LANDLORD_PROPERTY_ID]
== row[master_id_colnames[idx]]
]
]
if df.shape[0] == 1:
matched.append(
{
@ -4431,7 +4439,7 @@ class AssetList:
)[1]
)
> 90
]
]
if df.shape[0] == 0:
unmatched.append(row["row_id"])
@ -4439,8 +4447,8 @@ class AssetList:
if any(
df[self.STANDARD_FULL_ADDRESS]
.str.lower()
.str.contains(
.str.lower()
.str.contains(
" ".join(
[row[house_no_col], row["Street / Block Name"]]
).lower()
@ -4467,7 +4475,7 @@ class AssetList:
row[property_type_col].split(" ")[-1].lower()
)
& (df[self.STANDARD_PROPERTY_TYPE] != "block of flats")
]
]
if df.shape[0] != 1:
# We have multiple matches - it's likely because the landlord has a duplicate

59
asset_list/app.py
View file

@ -13,12 +13,11 @@ from asset_list.utils import get_data
from dotenv import load_dotenv
from backend.SearchEpc import SearchEpc
load_dotenv(dotenv_path="../backend/.env")
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv(
"EPC_AUTH_TOKEN",
)
OPENAI_API_KEY = os.getenv(
"OPENAI_API_KEY",
)
@ -74,24 +73,25 @@ def app():
Property UPRN
"""
data_folder = "/workspaces/model/asset_list"
data_filename = "assests.xlsx"
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lifespace Rentals/Missed"
# data_filename = "For Modelling - Final - reviewed.xlsx"
data_filename = "Missed Properties - with address.xlsx"
sheet_name = "Sheet1"
postcode_column = "Postcode"
address1_column = "Address"
address1_method = "house_number_extraction"
fulladdress_column = None
address_cols_to_concat = ["Address"]
address1_column = "address1"
address1_method = None
fulladdress_column = "address1"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = "UPRN"
landlord_property_type = "Archetype"
landlord_built_form = "Bedroom Count"
landlord_wall_construction = "Wall Insulation Type"
landlord_roof_construction = "Roof Type"
landlord_heating_system = "Boiler Type"
landlord_property_type = "Type"
landlord_built_form = None
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "Tab"
landlord_property_id = "Reference"
landlord_sap = None
outcomes_filename = None
outcomes_sheetname = None
@ -243,7 +243,7 @@ def app():
if skip is not None and not force_retrieve_data:
if i <= skip:
continue
chunk = asset_list.standardised_asset_list[i : i + chunk_size]
chunk = asset_list.standardised_asset_list[i: i + chunk_size]
epc_data_chunk, errors_chunk, no_epc_chunk = get_data(
df=chunk,
row_id_name=asset_list.DOMNA_PROPERTY_ID,
@ -386,7 +386,7 @@ def app():
# Retrieve just the data we need
epc_df = epc_df[
[asset_list.DOMNA_PROPERTY_ID] + list(asset_list.EPC_API_DATA_NAMES.keys())
].rename(columns=asset_list.EPC_API_DATA_NAMES)
].rename(columns=asset_list.EPC_API_DATA_NAMES)
# Look for columns not in the find my EPC data, which will have happened if we didn't
# retrieve it in the first place
@ -403,16 +403,12 @@ def app():
find_my_epc_data[
[asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"]
+ list(asset_list.FIND_EPC_DATA_NAMES.keys())
].rename(columns=asset_list.FIND_EPC_DATA_NAMES),
].rename(columns=asset_list.FIND_EPC_DATA_NAMES),
how="left",
on=asset_list.DOMNA_PROPERTY_ID,
)
asset_list.merge_data(epc_df)
# asset_list.standardised_asset_list = asset_list.standardised_asset_list[
# asset_list.standardised_asset_list["domna_full_address"]
# != "120 Airdrie Crescent, Burnley, Lancashire"
# ]
asset_list.extract_attributes()
asset_list.identify_worktypes()
@ -426,27 +422,6 @@ def app():
os.path.join(data_folder, ".".join(data_filename.split(".")[:-1]))
+ " - Standardised.xlsx"
)
# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data
# Determine inspections priority
# solar_jobs = asset_list.standardised_asset_list[~pd.isnull(asset_list.standardised_asset_list["solar_reason"])][
# "domna_postcode"].unique()
# asset_list.standardised_asset_list["in_solar_area"] = asset_list.standardised_asset_list["domna_postcode"].isin(
# solar_jobs
# )
# # Same for cav
# cavity_jobs = asset_list.standardised_asset_list[
# ~pd.isnull(asset_list.standardised_asset_list["cavity_reason"])
# ]["domna_postcode"].unique()
# asset_list.standardised_asset_list["in_cavity_area"] = asset_list.standardised_asset_list["domna_postcode"].isin(
# cavity_jobs
# )
# # We prioritise properties that are in solar areas and cavity areas
# import numpy as np
# asset_list.standardised_asset_list["inspection_priority"] = np.where(
# asset_list.standardised_asset_list["in_solar_area"] | asset_list.standardised_asset_list["in_cavity_area"],
# 1, 2
# )
with pd.ExcelWriter(filename) as writer:
asset_list.standardised_asset_list.to_excel(

103
asset_list/mappings/built_form.py
View file

@ -528,6 +528,107 @@ BUILT_FORM_MAPPINGS = {
'House: Semi Detached: Top Floor': 'semi-detached',
'House: End Terrace: Ground Floor': 'end-terrace',
'Maisonette: Enclosed End Terrace: Mid Floor': 'enclosed end-terrace',
'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace'
'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace',
'2 BED MID TERRACED HOUSE': 'mid-terrace',
'4 BED SEMI DETACHED-PARLOURED': 'semi-detached',
'2 BED END TERRACED HOUSE': 'end-terrace',
'3 BED MID TERRACED HOUSE': 'mid-terrace',
'3 BED SEMI DETACHED HOUSE': 'semi-detached',
'3 BED MID TERRACE - PARLOURED': 'mid-terrace',
'3 BED END TERRACE - PARLOURED': 'end-terrace',
'4 BED+ END TERRACED HOUSE': 'end-terrace',
'3 BED END TERRACED HOUSE': 'end-terrace',
'3 BED SEMI DETACHED-PARLOURED': 'semi-detached',
'4 BED+ END TERRACE - PARLOURED': 'end-terrace',
'2 BED SEMI DETACHED HOUSE': 'semi-detached',
'3 BED DETACHED HOUSE': 'detached',
'2 BED GRD FLR COTT FLT-CNT STR': 'ground floor',
'2 BED 1ST FLOOR WALKUP FLAT': 'mid-floor',
'1 BED GRD FL COTT FLAT-OWN ENT': 'ground floor',
'1 BED 1ST FL WALK UP DECK ACC': 'mid-floor',
'2 BED MAISONETTE UPPER COM ENT': 'mid-floor',
'2 BED GRD FLR COTT FLT OWN ENT': 'ground floor',
'1 BED BUNGALOW': 'unknown',
'2 BED GRD FL COTT FLT-OWN ENTR': 'ground floor',
'1 BED 1ST FL COTT FLT-CNT STR': 'mid-floor',
'1 BED GRD FL WALK UP OWN ENT': 'ground floor',
'1 BED GRD FLOOR WALKUP FLAT': 'ground floor',
'2 BED GRD FLOOR WALKUP FLAT': 'ground floor',
'2 BED 1ST FLR FLT-SHELTERED': 'mid-floor',
'2 BED BUNGALOW': 'unknown',
'2 BED GRD FLR COTT FLT(P)-1950': 'ground floor',
'Ground Floor Front Left': 'ground floor',
'End-Terrace House': 'end-terrace',
'Ground floor': 'ground floor',
'Ground Floor Front Right': 'ground floor',
'End Terrace (GII List)': 'end-terrace',
'Semi Detached House': 'semi-detached',
'Ground Floor Right': 'ground floor',
'PB Ground Floor Flat': 'ground floor',
'Basement and Ground Floor': 'ground floor',
'Semi-detached bungalow': 'detached',
'Detached Cottage': 'detached',
'Lower & Ground Floor': 'ground floor',
'Ground FLoor Flat': 'ground floor',
'ground floor': 'ground floor',
'Ground Floor Left': 'ground floor',
'Semi-detached House': 'detached',
'Basement & Lower Ground': 'basement',
'Semi-Detached House': 'detached',
'Ground floor flat -': 'ground floor',
'Basement Flat': 'basement',
'semi-detached bungalow': 'semi-detached',
'Lower Ground Floor Flat': 'ground floor',
'Ground floor Flat': 'ground floor',
'Ground Floor flat': 'ground floor',
'Ground': 'ground floor',
'Semi detached Bungalow': 'semi-detached',
'ground floor flat': 'ground floor',
'Mid terrace House': 'mid-terrace',
'Raised Ground Floor': 'ground floor',
'Basement Floor': 'basement',
'Second floor flat': 'mid-floor',
'Fourth Floor Flat': 'mid-floor',
'First/Second Maisonette': 'mid-floor',
'Ground/First': 'ground floor',
'First and Second Floor': 'mid-floor',
'Terrace House': 'mid-terrace',
'1st/2nd Floor Maisonette': 'mid-floor',
'Semi-det House': 'semi-detached',
'First': 'mid-floor',
'Ground & First Floor': 'ground floor',
'End of Terrace House': 'end-terrace',
'2nd Floor Purpose Built': 'mid-floor',
'First/Second Floor Maison': 'mid-floor',
'GFF purpose built': 'ground floor',
'Second': 'mid-floor',
'Semi-det House (GII List)': 'semi-detached',
'3rd and 4th Floor': 'mid-floor',
'First Floor flat': 'mid-floor',
'Mid-Terrace House': 'mid-terrace',
'1st & 2nd Floors': 'mid-floor',
'Ground/first floor': 'ground floor',
'FFF purpose built': 'mid-floor',
'Second floor': 'mid-floor',
'Second/Third floor': 'mid-floor',
'First floor Flat': 'mid-floor',
'First floor': 'mid-floor',
'Lower Ground Flat': 'basement',
'First Floor Rear Flat': 'mid-floor',
'First & Second Floor': 'mid-floor',
'Ground & Lower Ground': 'basement',
'First Floor Rear': 'mid-floor',
'First & Second': 'mid-floor',
'First Floor Front': 'mid-floor',
'First & Second Floors': 'mid-floor',
'First/Second Floor': 'mid-floor',
'Sem-detach house': 'semi-detached',
'Second Floor Flat (Top)': 'top-floor',
'3 FloorTerrace House': 'mid-terrace',
'First floor flat': 'mid-floor',
'First & Second Floor Flat': 'mid-floor',
'First Floor Purpose Built': 'mid-floor',
'Purpose built First Floor': 'mid-floor',
}

19
asset_list/mappings/heating_systems.py
View file

@ -498,6 +498,23 @@ HEATING_MAPPINGS = {
'Boiler: A rated Combi, System 2: Boiler: A rated Combi': 'gas combi boiler',
'System 2: Boiler: A rated Regular Boiler, Boiler: A rated Regular Boiler': 'gas boiler, radiators',
'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler'
'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler',
'IDEAL ISAR HE30': 'gas combi boiler',
'WORCESTER GREENSTAR 25 SI': 'gas combi boiler',
'POTTERTON PROMAX COMBI 28 HE PLUS': 'gas combi boiler',
'WORCESTER GREENSTAR 28I JUNIOR': 'gas combi boiler',
'BAXI ASSURE 25 COMBI': 'gas combi boiler',
'POTTERTON PROMAX COMBI 28 HE PLUS A': 'gas combi boiler',
'WORCESTER GREENSTAR 30 SI': 'gas combi boiler',
'POTTERTON SUPRIMA 40L': 'gas boiler, radiators',
'POTTERTON ASSURE 30 COMBI': 'gas combi boiler',
'POTTERTON PROMAX 28 COMBI ERP': 'gas combi boiler',
'BAXI ASSURE 30 COMBI': 'gas combi boiler',
'POTTERTON PROMAX 18 SYSTEM ERP': 'gas boiler, radiators',
'POTTERTON PROMAX COMBI 33 HE PLUS A': 'gas combi boiler',
'POTTERTON SUPRIMA 40 HE': 'gas boiler, radiators',
'FERROLI MODENA 102': 'gas boiler, radiators',
'POTTERTON PROMAX COMBI 24 HE PLUS A': 'gas combi boiler'
}

5
asset_list/mappings/property_type.py
View file

@ -444,6 +444,9 @@ PROPERTY_MAPPING = {
'Warden Bungalow': 'bungalow',
'Warden Flat': 'flat',
'Upper Floor Flat': 'flat',
'Extracare Scheme': 'other'
'Extracare Scheme': 'other',
'SHELTERED': 'unknown',
'PARLOUR': 'unknown',
}

4
asset_list/mappings/roof.py
View file

@ -320,6 +320,8 @@ ROOF_CONSTRUCTION_MAPPINGS = {
'Pitched (slates or tiles) access to loft, 100mm': 'pitched insulated',
'Pitched (slates or tiles) no loft access, 200mm': 'pitched insulated',
'Pitched (slates or tiles) access to loft, 200mm': 'pitched insulated',
'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation'
'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation',
'Pitched roofs': 'pitched unknown insulation',
}

5
asset_list/mappings/walls.py
View file

@ -369,6 +369,9 @@ WALL_CONSTRUCTION_MAPPINGS = {
'Solid Brick, As built': 'solid brick unknown insulation',
'System built, As built': 'system built unknown insulation',
'Timber frame, As built': 'timber frame unknown insulation',
'Cavity, As built': 'cavity unknown insulation'
'Cavity, As built': 'cavity unknown insulation',
'FILLED CAVITY': 'filled cavity',
'EXTERNAL': 'insulated solid brick',
'AS BUILT': 'other'
}