From 731d16bb01d7fe38c720a4314990fc1937b1293d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 14:27:27 +0100 Subject: [PATCH 1/9] finshing medway and thurrock --- asset_list/AssetList.py | 29 +++++-- asset_list/app.py | 2 +- etl/customers/medway/flag_reviewed.py | 104 ++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 etl/customers/medway/flag_reviewed.py diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index b7dd8d70..529aef3d 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -1235,11 +1235,11 @@ class AssetList: elif self.old_format_non_intrusives_present: non_intrusives_wall_filter = ( self.standardised_asset_list['non-intrusives: WFT Findings'].str.lower().str.strip().isin( - ["empty cavity", "partial fill", "empty", "EMPTY CAVITY 70MM", "partial"] + ["empty cavity", "partial fill", "empty", "EMPTY CAVITY 70MM", "partial", "empty cav"] ) | ( ( self.standardised_asset_list['non-intrusives: WFT Findings'] - .str.lower().str.strip().str.contains("empty cavity|partial fill|empty|partial") & + .str.lower().str.strip().str.contains("empty cavity|partial fill") & ~self.standardised_asset_list['non-intrusives: WFT Findings'] .astype(str).str.lower().str.strip().str.contains("major access issues") ) @@ -1368,8 +1368,16 @@ class AssetList: print("Review these categories!!!!") extraction_wall_filter = ( self.standardised_asset_list['non-intrusives: WFT Findings'].str.lower().str.strip().isin( - ["retro drilled", "retro filled", "fibre from build", "polybead", "retro drilled and filled", - "retro drilled & filled", "blown in white wool", "blown in yellow wool"] + [ + 'blown in yellow wool', 'retro drilled & filled', 'white fibre from build', + 'foam filled from build', 'retro drilled gas in block', 'block in rock wool', 'rdf / tilehung', + 'fibre from build', 'blown in rock wool', 'rdf / tile hung', 'retro drilled', + 'rock wool from build', 'part rendered retro drilled', 'white fibtr from build.', + 'retro drilled and filled', 'blown in white wool', 'blown in yellow fibre from build', 'rdf', + 'polybead', 'foam filled', 'blown in white bead from build', 'blown in yellow fibre', + 'retro drilled det', 'blown in rockwool', 'retro drilled det empty cav', 'retro drilled end', + 'retro filled extension', 'retro filled', 'foam' + ] ) ) @@ -1433,7 +1441,18 @@ class AssetList: self.standardised_asset_list["solar_epc_data_indicates_correct_heating_system"] & self.standardised_asset_list["solar_epc_data_indicates_requires_heating_upgrade"] ).sum(): - raise ValueError("Both heating system checks are true - this should not be possible") + logger.info("We have an example of both heating system checks being true - checking known cases") + known_edge_cases = ['Ground source heat pump, radiators, electric, Electric storage heaters'] + error_cases = self.standardised_asset_list[ + ( + self.standardised_asset_list["solar_epc_data_indicates_correct_heating_system"] & + self.standardised_asset_list["solar_epc_data_indicates_requires_heating_upgrade"] + ) + ] + if all(error_cases[self.EPC_API_DATA_NAMES["mainheat-description"]].isin(known_edge_cases)): + logger.info("Within known edge cases") + else: + raise ValueError("Both heating system checks are true - this should not be possible") # Check 3: Does the property meet the fabric condition # Solar PV installs are subject to the minimum insulation requirements which means: diff --git a/asset_list/app.py b/asset_list/app.py index d5ce7226..e3c612a7 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -64,7 +64,7 @@ def app(): # Thurrock data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thurrock" - data_filename = "THURROCK COUNCIL.xlsx" + data_filename = "THURROCK COUNCIL - For analysis.xlsx" sheet_name = "Assets" postcode_column = 'Postcode' fulladdress_column = "Full Address" diff --git a/etl/customers/medway/flag_reviewed.py b/etl/customers/medway/flag_reviewed.py new file mode 100644 index 00000000..e2b27670 --- /dev/null +++ b/etl/customers/medway/flag_reviewed.py @@ -0,0 +1,104 @@ +""" +This script marks which properties have been reviewed by the Medway. +""" +import pandas as pd +import numpy as np +from tqdm import tqdm +import os + +asset_list = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/MEDWAY Asset List - Standardised.xlsx", + sheet_name="Standardised Asset List", +) +flat_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/MEDWAY Asset List - Standardised.xlsx", + sheet_name="Flat Data", +) + +reviewed_assets = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/Programme Final Check.xlsx", +) +exclude_from_programme = reviewed_assets.copy() # [reviewed_assets["Khalim - include in programme"] == "No"].copy() +exclude_from_programme["exclusion_reason"] = None +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme["UPRN"] == "SOLD"), + "Sold", + exclude_from_programme["exclusion_reason"], +) +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme["Include in SHDF Bid?"] == "Definite"), + "Included in SHDF Bid", + exclude_from_programme["exclusion_reason"], +) + +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme['Move Forward'] == "EPC C"), + "Excluded from Programme", + exclude_from_programme["exclusion_reason"], +) + +# exclude_from_programme = exclude_from_programme[~pd.isnull(exclude_from_programme["exclusion_reason"])] +exclude_from_programme = exclude_from_programme.reset_index(drop=True) +exclude_from_programme["row_id"] = exclude_from_programme.index + +# Match to asset list +matched = [] +for _, x in tqdm(exclude_from_programme.iterrows(), total=len(exclude_from_programme)): + + if x["No."] == 218 and x["Postcode"] == "ME8 6QB": + pc = "ME8 6QP" + elif x["No."] == 198 and x["Postcode"] == "ME8 6HL": + pc = "ME8 6LU" + elif x["No."] == "39a" and x["Postcode"] == "ME7 2BU": + pc = "ME7 2BU" + else: + pc = x["Postcode"] + + hn = x["No."] + + m = asset_list[ + (asset_list["domna_address_1"] == str(hn)) & + (asset_list["domna_postcode"] == str(pc)) + ] + + if m.empty: + m = asset_list[ + (asset_list["domna_full_address"].str.replace(",", "").str.lower().str.contains( + x["Address"].lower().strip())) + ] + + if m.shape[0] == 1: + matched.append( + { + "full_address": m["domna_full_address"].values[0], + "postcode": m["domna_postcode"].values[0], + "review_no": x["No."], + "review_address": x["Address"], + "review_postcode": x["Postcode"], + "exclusion_reason": x["exclusion_reason"], + "landlord_property_id": m["landlord_property_id"].values[0], + "ciga_guarantee": x["Unnamed: 21"] + } + ) + continue + + raise NotImplementedError("FIX ME") + +matched = pd.DataFrame(matched) + +matched = matched.rename( + columns={"review_address": "ciga_check_address"} +) + +asset_list = asset_list.merge( + matched[["landlord_property_id", "exclusion_reason", "ciga_guarantee", "ciga_check_address"]], + how="left", on="landlord_property_id" +) + +# Store as an excel +filename = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/Reviewed Standardised Programme.xlsx" +# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data + +with pd.ExcelWriter(filename) as writer: + asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False) + flat_data.to_excel(writer, sheet_name="Flat Data", index=False) From 9f46b23b72623afb217ff30db5ff0d33bf38b8d5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 15:34:12 +0100 Subject: [PATCH 2/9] added testing automation with tox and added new test to handle error case when fetching address from epc registry --- .github/workflows/unit_tests.yml | 5 --- Makefile | 5 +++ README.md | 3 +- asset_list/AssetList.py | 2 +- asset_list/requirements.txt | 2 +- backend/SearchEpc.py | 2 +- backend/engine/requirements.txt | 4 +- backend/tests/test_search_epc.py | 8 +++- etl/spatial/tests/test_borehole_client.py | 40 ------------------- pytest.ini | 4 +- .../dev.txt => test.requirements.txt | 1 + tox.ini | 11 +++++ 12 files changed, 32 insertions(+), 55 deletions(-) create mode 100644 Makefile delete mode 100644 etl/spatial/tests/test_borehole_client.py rename backend/app/requirements/dev.txt => test.requirements.txt (85%) create mode 100644 tox.ini diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 39d285f2..22b13ab2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,8 +36,3 @@ jobs: run: | pip install -r model_data/requirements/dev.txt pytest -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v2 -# with: -# token: ${{ secrets.CODECOV_TOKEN }} -# fail_ci_if_error: true diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e26ab667 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +setup: + pip install tox + +test: + tox \ No newline at end of file diff --git a/README.md b/README.md index df36cfe7..9268ba25 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,5 @@ To run tests in a specific service, e.g. inside of model_data, simply run pytest --cov-config=model_data/.coveragerc --cov=model_data ``` -This will produce the test results and coverage reports \ No newline at end of file +This will produce the test results and coverage reports + diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 529aef3d..4b7a11ec 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -10,7 +10,7 @@ from openai import OpenAI import numpy as np import pandas as pd from tqdm import tqdm -from fuzzywuzzy import process +from thefuzz import process from utils.logger import setup_logger from backend.SearchEpc import SearchEpc from BaseUtility import Definitions diff --git a/asset_list/requirements.txt b/asset_list/requirements.txt index fd43ac64..99943397 100644 --- a/asset_list/requirements.txt +++ b/asset_list/requirements.txt @@ -3,7 +3,7 @@ pandas usaddress pydantic-settings==2.6.0 epc-api-python==1.0.2 -fuzzywuzzy +thefuzz boto3 openpyxl openai diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index e19a776d..0010191a 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -14,7 +14,7 @@ from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from BaseUtility import Definitions from utils.logger import setup_logger from typing import List -from fuzzywuzzy import process +from thefuzz import process from backend.app.utils import sap_to_epc logger = setup_logger() diff --git a/backend/engine/requirements.txt b/backend/engine/requirements.txt index f5e1b5f6..b565e9d3 100644 --- a/backend/engine/requirements.txt +++ b/backend/engine/requirements.txt @@ -10,8 +10,8 @@ boto3==1.35.44 # ML, Data Science usaddress==0.5.11 epc-api-python==1.0.2 -fuzzywuzzy==0.18.0 -python-Levenshtein==0.26.0 +thefuzz +python-Levenshtein>=0.24.0,!=0.26.0 textblob==0.18.0.post0 msgpack==1.1.0 scikit-learn==1.5.2 diff --git a/backend/tests/test_search_epc.py b/backend/tests/test_search_epc.py index 562585ad..d4313f61 100644 --- a/backend/tests/test_search_epc.py +++ b/backend/tests/test_search_epc.py @@ -9,7 +9,7 @@ EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") class TestSearchEpcIntegration: @pytest.mark.parametrize( - "address, postcode, uprn, skip_os, expected_partial_address", + "address, postcode, uprn, skip_os, lmk_key, n_old_epcs", [ # Test case 1: Valid address and postcode, skipping OS # In this case, the property is an individual flat but the uprn associated to the @@ -21,7 +21,11 @@ class TestSearchEpcIntegration: # In this case, the newest EPC, does not have a uprn associated to it. If we did a search by # uprn, we would get an old EPC ("Flat 8, Hainton House", "DN32 9AQ", 10090082018, True, - "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 3), + "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 2), + # Test case 3: When we make a request to the API for this property, we get back results for + # flats 1, 2 and 3. We have some logic to handle the response so that we get back flat 1 + ("Flat 1, 1 Tottenham Street, London", "W1T 2AE", 5167411, True, + "3e6414d7f15f4cf7a69dc20c469bcf043d31a49239b183f1bd0c0e1aafa23c93", 0), ], ) diff --git a/etl/spatial/tests/test_borehole_client.py b/etl/spatial/tests/test_borehole_client.py deleted file mode 100644 index 38bf4495..00000000 --- a/etl/spatial/tests/test_borehole_client.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest -from etl.spatial.BoreholeClient import BoreholeClient - - -@pytest.fixture -def mock_borehole_data(monkeypatch): - # Define the mock data to be returned by the read() method - mock_data = [ - { - 'X': 464343.0, - 'Y': 415553.0 - }, - { - 'X': 464341.0, - 'Y': 415556.0 - }, - # Add more mock data entries here... - ] - - # Monkeypatch the read() method to return the mock data - def mock_read(self): - return mock_data - - # Apply the monkeypatch to the BoreholeClient class - monkeypatch.setattr(BoreholeClient, 'read', mock_read) - - -@pytest.mark.usefixtures('mock_borehole_data') -def test_distance_between_bng_coords(): - # Create an instance of BoreholeClient - borehole_client = BoreholeClient("path/to/dbf/file") - - # Test the distance calculation - distance_m, distance_km = borehole_client.distance_between_bng_coords( - x1_bng=123456, y1_bng=789012, x2_bng=464343.0, y2_bng=415553.0 - ) - - # Perform assertions to verify the results - assert distance_m == 505643.71987596166 - assert distance_km == 505.64371987596166 diff --git a/pytest.ini b/pytest.ini index b2453c82..84c686b1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] pythonpath = . -addopts = --cov-report term-missing --cov=etl --cov=recommendations --cov=backend -testpaths = etl/*/tests recommendations/tests backend/tests +addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial +testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests diff --git a/backend/app/requirements/dev.txt b/test.requirements.txt similarity index 85% rename from backend/app/requirements/dev.txt rename to test.requirements.txt index a466954c..d31371a6 100644 --- a/backend/app/requirements/dev.txt +++ b/test.requirements.txt @@ -2,3 +2,4 @@ pytest mock pytest-cov pytest-mock +dotenv \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..d9384f58 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py311 +skipsdist = True + +[testenv] +description = Install dependencies and run tests +deps = + -rbackend/engine/requirements.txt + -rtest.requirements.txt +commands = pytest + From 5201cab4a9e3b507c2362659b5e587ec8c048c34 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 15:40:42 +0100 Subject: [PATCH 3/9] mocking data for property class tests --- backend/tests/test_property.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 78f08f3c..7f7cc140 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -103,6 +103,11 @@ class TestProperty: property_instance.number_of_rooms = 5 property_instance.floor_area = 100 property_instance.floor_height = 2.5 + + # Fill these values that come from the epc_record + property_instance.energy["primary_energy_consumption"] = 1234 + property_instance.energy["epc_co2_emissions"] = 5 + return property_instance @pytest.fixture From b2beb2f1229c6a11814deafd57be4df38e0d2841 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 16:45:24 +0100 Subject: [PATCH 4/9] fixing existing unit tests --- Makefile | 31 +++++++++++++++++-- .../tests/test_solar_pv_recommendations.py | 2 +- .../tests/test_ventilation_recommendations.py | 10 +++--- .../tests/test_window_recommendations.py | 18 +++++++---- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index e26ab667..00942acd 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,30 @@ -setup: - pip install tox +# Project Makefile +PYTHON = python + +.PHONY: setup test lint typecheck check clean + +# Install dev dependencies + tox +setup: + $(PYTHON) -m pip install --upgrade pip + $(PYTHON) -m pip install tox black ruff mypy + +# Run tests (pass ARGS="..." for specific tests) test: - tox \ No newline at end of file + tox -- $(ARGS) + +# Code formatting check + linting +lint: + ruff . + black --check . + +# Static type checks +typecheck: + mypy . + +# Full quality check (all checks + tests) +check: lint typecheck test + +# Clean up tox environments +clean: + rm -rf .tox diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index a18291e5..a72e4c1d 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -46,7 +46,7 @@ class TestSolarPvRecommendations: property_instance_valid_all = Property(id=1, address="", postcode="", epc_record=epc_record) property_instance_valid_all.roof_area = 40 property_instance_valid_all.number_of_floors = 2 - property_instance_valid_all.roof = {"is_flat": True} + property_instance_valid_all.roof = {"is_flat": True, "thermal_transmittance": None} property_instance_valid_all.solar_panel_configuration = { "panel_performance": pd.DataFrame( [ diff --git a/recommendations/tests/test_ventilation_recommendations.py b/recommendations/tests/test_ventilation_recommendations.py index 441f9a22..787efa52 100644 --- a/recommendations/tests/test_ventilation_recommendations.py +++ b/recommendations/tests/test_ventilation_recommendations.py @@ -18,7 +18,7 @@ class TestVentilationRecommendations: assert not recommender.recommendation - recommender.recommend() + recommender.recommend(phase=None) assert len(recommender.recommendation) == 1 @@ -40,7 +40,7 @@ class TestVentilationRecommendations: assert not recommender2.recommendation - recommender2.recommend() + recommender2.recommend(phase=None) assert len(recommender2.recommendation) == 1 @@ -62,7 +62,7 @@ class TestVentilationRecommendations: assert not recommender3.recommendation - recommender3.recommend() + recommender3.recommend(phase=None) assert len(recommender3.recommendation) == 1 @@ -84,7 +84,7 @@ class TestVentilationRecommendations: assert not recommender4.recommendation - recommender4.recommend() + recommender4.recommend(phase=None) assert not recommender4.recommendation assert recommender4.has_ventilaion @@ -101,7 +101,7 @@ class TestVentilationRecommendations: assert not recommender5.recommendation - recommender5.recommend() + recommender5.recommend(phase=None) assert not recommender5.recommendation assert recommender5.has_ventilaion diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index baef3574..1c32d2bc 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -56,7 +56,8 @@ class TestWindowRecommendations: 'has_glazing_ending': True, 'glazing_type_ending': 'double', 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'double glazing installed during or after 2002' - } + }, + "survey": None } ] @@ -106,7 +107,8 @@ class TestWindowRecommendations: 'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'double', 'glazed_type_ending': 'double glazing installed during or after 2002' - } + }, + "survey": None } ] @@ -206,7 +208,8 @@ class TestWindowRecommendations: 'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'secondary', 'glazed_type_ending': 'secondary glazing' - } + }, + "survey": None } ] @@ -254,8 +257,9 @@ class TestWindowRecommendations: 'has_glazing_ending': True, 'glazing_coverage_ending': 'full', 'glazing_type_ending': 'secondary', 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'secondary glazing' - } - } + }, + "survey": None + }, ] def test_full_triple_glazed(self): @@ -383,6 +387,7 @@ class TestWindowRecommendations: 'glazing_type': 'single', 'no_data': False } + property_9.perimeter = 23.430749027719962 property_9.number_of_windows = 7 property_9.restricted_measures = False @@ -410,7 +415,8 @@ class TestWindowRecommendations: 'glazing_type_ending': 'double', 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'double glazing installed during or after 2002' - } + }, + "survey": None } ] From b609a3f2bbe39dabef8a29b052fd635782a57997 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 16:46:37 +0100 Subject: [PATCH 5/9] fixing existing unit tests --- .../tests/test_solar_pv_recommendations.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index a72e4c1d..87406e49 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -3,6 +3,7 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations from backend.Property import Property from etl.epc.Record import EPCRecord import pandas as pd +import numpy as np class TestSolarPvRecommendations: @@ -82,23 +83,16 @@ class TestSolarPvRecommendations: solar_pv.recommend(phase=0) assert len(solar_pv.recommendation) == 2 assert solar_pv.recommendation == [ - { - 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', - 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' - 'roof.', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, - 'total': 4850.0, 'subtotal': 4041.666666666667, 'vat': 808.333333333333, 'labour_hours': 48, - 'labour_days': 2, 'photo_supply': 50.0, 'has_battery': False, 'initial_ac_kwh_per_year': 3800, - 'description_simulation': {'photo-supply': 50.0} - }, - { - 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', - 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' - 'roof, ' - 'with a battery storage system.', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, - 'total': 7550.0, 'subtotal': 6291.666666666667, 'vat': 1258.333333333333, 'labour_hours': 48, - 'labour_days': 2, 'photo_supply': 50.0, 'has_battery': True, 'initial_ac_kwh_per_year': 3800, - 'description_simulation': {'photo-supply': 50.0} - } + {'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system.', 'starting_u_value': None, + 'new_u_value': None, 'sap_points': np.float64(10.0), 'already_installed': False, + 'total': 6013.139999999999, 'subtotal': 5010.95, 'vat': 0, 'labour_hours': 48, 'labour_days': 2, + 'photo_supply': np.float64(50.0), 'has_battery': False, 'initial_ac_kwh_per_year': np.int64(3800), + 'description_simulation': {'photo-supply': np.float64(50.0)}}, + {'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system, with a battery.', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(10.0), 'already_installed': False, + 'total': 10537.008, 'subtotal': 8780.84, 'vat': 0, 'labour_hours': 48, 'labour_days': 2, + 'photo_supply': np.float64(50.0), 'has_battery': True, 'initial_ac_kwh_per_year': np.int64(3800), + 'description_simulation': {'photo-supply': np.float64(50.0)}} ] From 53f70ab14dec1eee7d1937b4cfb54e3aeba0c239 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 16:49:03 +0100 Subject: [PATCH 6/9] fixing solar recs --- recommendations/tests/test_solar_pv_recommendations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index 87406e49..b16fcc3b 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -26,7 +26,9 @@ class TestSolarPvRecommendations: "county": "Huntingdonshire", "property-type": "House", "photo-supply": None } property_instance_invalid_roof = Property(id=1, address="", postcode="", epc_record=epc_record) - property_instance_invalid_roof.roof = {"is_flat": False, "is_pitched": False, "is_roof_room": False} + property_instance_invalid_roof.roof = { + "is_flat": False, "is_pitched": False, "is_roof_room": False, "thermal_transmittance": None + } return property_instance_invalid_roof @pytest.fixture @@ -36,7 +38,7 @@ class TestSolarPvRecommendations: epc_record.prepared_epc = {"photo-supply": "40", "county": "Huntingdonshire", "property-type": "House"} property_instance_has_solar_pv = Property(id=1, address="", postcode="", epc_record=epc_record) - property_instance_has_solar_pv.roof = {"is_flat": True} + property_instance_has_solar_pv.roof = {"is_flat": True, "thermal_transmittance": None} return property_instance_has_solar_pv @pytest.fixture From 66d6266002be0f2b5ee05d4719df43de4e6efcfe Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 20:17:42 +0100 Subject: [PATCH 7/9] fixing unit tests --- recommendations/HeatingRecommender.py | 13 +- .../test_data/heating_recommendations_data.py | 128 ++---------------- recommendations/tests/test_data/materials.py | 24 ++++ .../tests/test_heating_recommendations.py | 10 -- .../tests/test_lighting_recommendations.py | 28 ++-- 5 files changed, 57 insertions(+), 146 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 20f5e7ad..18e1110b 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -185,17 +185,24 @@ class HeatingRecommender: self.property.main_heating["clean_description"] ]["dual"]["types"] - recommendation_system_types = list(set([x["system_type"] for x in self.heating_recommendations])) + heating_measures = [ + m for m in self.heating_recommendations if + m["measure_type"] not in ["time_temperature_zone_control", "roomstat_programmer_trvs"] + ] + + recommendation_system_types = list( + set([x["system_type"] for x in heating_measures]) + ) # We check if we have the required type if not any([x in recommendation_system_types for x in dual_heating_description]): return type_1_recommendations = [ - x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[0] + x for x in heating_measures if x["system_type"] == dual_heating_description[0] ] type_2_recommendations = [ - x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[1] + x for x in heating_measures if x["system_type"] == dual_heating_description[1] ] # we combine the two recommendations combined_recommendations = [] diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 26263826..53f8bd25 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -39,8 +39,7 @@ testing_examples = [ 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 7.0, 'uprn': 100110195416.0, 'uprn-source': 'Address Matched' }, - "heating_measure_types": ["air_source_heat_pump"], - "heating_controls_measure_types": ["time_temperature_zone_control"], + "heating_measure_types": ["air_source_heat_pump", "time_temperature_zone_control"], "notes": "This property has a boiler, radiators & mains gas with good efficiency so the only recommendation" "we expect here is for an air source heat pump. The heating controls are a programmer, room thermostat" "and TRVs and so we should expect a TTZC recommendation" @@ -89,7 +88,6 @@ testing_examples = [ "heating_measure_types": [ "high_heat_retention_storage_heater", ], - "heating_controls_measure_types": [], "notes": "This property has electric room heaters and is off gas so a boiler recommendation is not appropriate." "We would expect a high heat retention storage recommendation. The property is a flat and therefore" "we don't expect an air source heat pump recommendation. We also wouldn't expect a specific heating" @@ -137,7 +135,6 @@ testing_examples = [ 'uprn': 100090311351.0, 'uprn-source': 'Address Matched', 'property-type_y': None, 'built-form_y': None, }, "heating_measure_types": ['high_heat_retention_storage_heater', 'air_source_heat_pump'], - "heating_controls_measure_types": [], "notes": "This test has electric storage heaters with automatic charge control - we recommend hhr storage" "heaters in this case, but because there are already electic storage heaters in place, we " "note, in the description of the recommendation, that this upgrade may be possible by retrofitting" @@ -181,8 +178,8 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, - "heating_measure_types": ['boiler_upgrade'], - "heating_controls_measure_types": [ + "heating_measure_types": [ + 'boiler_upgrade', 'roomstat_programmer_trvs', 'time_temperature_zone_control', ], @@ -231,10 +228,9 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 9.0, 'low-energy-fixed-light-count': 5.0, 'uprn': 100021936225.0, 'uprn-source': 'Address Matched', }, - "heating_measure_types": [], - "heating_controls_measure_types": [ + "heating_measure_types": [ 'roomstat_programmer_trvs', - 'time_temperature_zone_control', + 'time_temperature_zone_control' ], "notes": "Because this property already has a boiler, we don't recommend HHR. We don't recommend an ashp " "because the home is mid-terraced. Because the heating controls are " @@ -284,7 +280,6 @@ testing_examples = [ "heating_measure_types": [ 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property is a flat so we don't have an ASHP recommendation. It also doesn't have access to the " "mains and so it can't have a gas boiler. We don't expect any controls recommendations" }, @@ -326,8 +321,8 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100080513604.0, 'uprn-source': 'Address Matched' }, - "heating_measure_types": ['air_source_heat_pump'], - "heating_controls_measure_types": [ + "heating_measure_types": [ + 'air_source_heat_pump', 'roomstat_programmer_trvs', 'time_temperature_zone_control', ], @@ -381,7 +376,6 @@ testing_examples = [ 'high_heat_retention_storage_heater', 'boiler_upgrade' ], - "heating_controls_measure_types": [], "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." "We can recommend a boiler upgrade and high heat retention storage heaters" }, @@ -427,7 +421,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" "an air source heat pump and HHR (since if the home has a non-gas boiler, we recommend HHR)" }, @@ -477,7 +470,6 @@ testing_examples = [ 'air_source_heat_pump', 'boiler_upgrade' # TTZs ], - "heating_controls_measure_types": [], "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" "well as an air source heat pump and HHR (since the home has a room heater set up)" }, @@ -525,7 +517,6 @@ testing_examples = [ 'boiler_upgrade', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property has assumed electric heaters. Boiler upgrade, HHR are recommended. We don't recommend" "an ASHP off of the bat because it's mid-terrace." }, @@ -572,7 +563,6 @@ testing_examples = [ 'high_heat_retention_storage_heater', 'boiler_upgrade' ], - "heating_controls_measure_types": [], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" }, @@ -620,7 +610,6 @@ testing_examples = [ 'boiler_upgrade', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" "the ashp is not suitable" }, @@ -668,7 +657,6 @@ testing_examples = [ 'high_heat_retention_storage_heater', 'air_source_heat_pump', ], - "heating_controls_measure_types": [], "notes": "This property has an LFG boiler but it doesn't have a mains gas connection so we can only recommend" "an air source heat pump and hhr storage" }, @@ -714,7 +702,6 @@ testing_examples = [ 'high_heat_retention_storage_heater', 'air_source_heat_pump', ], - "heating_controls_measure_types": [], "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " "recommend a boiler upgrade. We recommend HHR and ASHP" }, @@ -762,7 +749,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater' ], - "heating_controls_measure_types": [], "notes": "This property has a dual fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -807,7 +793,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -855,7 +840,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property has a smokeless fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -901,7 +885,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater', ], - "heating_controls_measure_types": [], "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -948,7 +931,6 @@ testing_examples = [ 'high_heat_retention_storage_heater', 'air_source_heat_pump', ], - "heating_controls_measure_types": [], "notes": "This is an end-terrace house, without mains gas connection, so we recommend is HHR & ASHP" }, { @@ -990,7 +972,6 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_measure_types": [], - "heating_controls_measure_types": [], "notes": "This property already has an ashp. We don't recommend any heating upgrades" }, { @@ -1032,9 +1013,7 @@ testing_examples = [ }, "heating_measure_types": [ 'air_source_heat_pump', - 'high_heat_retention_storage_heater' - ], - "heating_controls_measure_types": [ + 'high_heat_retention_storage_heater', 'time_temperature_zone_control', ], "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" @@ -1081,9 +1060,7 @@ testing_examples = [ 'air_source_heat_pump', 'boiler_upgrade', 'boiler_upgrade+high_heat_retention_storage_heater', - 'high_heat_retention_storage_heater' - ], - "heating_controls_measure_types": [ + 'high_heat_retention_storage_heater', 'time_temperature_zone_control' ], "notes": "This property is a modified version of the previous dual heating property, where we lower the" @@ -1132,7 +1109,6 @@ testing_examples = [ 'air_source_heat_pump', 'high_heat_retention_storage_heater' ], - "heating_controls_measure_types": [], "notes": "This property has anthracite heating without mains. " "We recommend ASHP and HHR, but no gas condensing boiler" }, @@ -1180,7 +1156,6 @@ testing_examples = [ 'boiler_upgrade', 'high_heat_retention_storage_heater' ], - "heating_controls_measure_types": [], "notes": "This property has room heaters with two different fuel sources, so we recommend HHR, ASHP, and a " "boiler upgrade" }, @@ -1224,7 +1199,6 @@ testing_examples = [ "heating_measure_types": [ 'high_heat_retention_storage_heater' ], - "heating_controls_measure_types": [], "notes": "This property is a flag, without mains gas connection. Currently has underfloor electric heating" "so we recommend HHR" }, @@ -1272,7 +1246,6 @@ testing_examples = [ 'boiler_upgrade', 'high_heat_retention_storage_heater' ], - "heating_controls_measure_types": [], "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" "connection so we recommend a gas condensing boiler" }, @@ -1320,89 +1293,6 @@ testing_examples = [ 'boiler_upgrade', 'boiler_upgrade', ], - "heating_controls_measure_types": [], "notes": "This property has warm air mains gas heating, so we recommend a gas condensing boiler" } ] - -# import random -# from pathlib import Path -# import inspect -# import pandas as pd -# -# # this can be used to get example data to build the test cases -# src_file_path = inspect.getfile(lambda: None) -# EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" -# epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] -# directory = random.sample(epc_directories, 1)[0] -# data = pd.read_csv(directory / "certificates.csv", low_memory=False) -# # Rename the columns to the same format as the api returns -# data.columns = [c.replace("_", "-").lower() for c in data.columns] -# data["floor-height"] = data["floor-height"].fillna(2.45) -# -# used_examples = pd.DataFrame( -# [ -# { -# "mainheat-description": x["epc"]["mainheat-description"], -# "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], -# "property-type": x["epc"]["property-type"], -# "built-form": x["epc"]["built-form"], -# "used": True -# } for x in testing_examples -# ] -# ) -# -# data = data.merge( -# used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] -# ) -# data = data[pd.isnull(data["used"])].drop(columns=["used"]) -# -# eg = data.sample(1).to_dict("records")[0] -# print(eg["mainheat-description"]) -# print(eg["mainheat-energy-eff"]) -# print(eg["property-type"]) -# print(eg["built-form"]) -# print(eg["mainheatcont-description"]) -# -# ### We also use the Midlands EPC F/G portfolio to get examples to create tests -# -# completed_descriptions = [ -# "Portable electric heaters assumed for most rooms", -# "Boiler and radiators, oil", -# "Boiler and radiators, mains gas", -# "Room heaters, mains gas", -# "No system present: electric heaters assumed", -# "Room heaters, electric", -# "Electric storage heaters", -# "Boiler and radiators, LPG", -# "Boiler and radiators, electric", -# "Boiler and radiators, dual fuel (mineral and wood)", -# "Boiler and radiators, coal", -# "Boiler and radiators, smokeless fuel", -# "Boiler and radiators, wood pellets", -# "Room heaters, dual fuel (mineral and wood)", -# "Air source heat pump, radiators, electric", -# "Portable electric heaters assumed for most rooms, Room heaters, electric", -# "Boiler and radiators, mains gas, Electric storage heaters", -# "Room heaters, anthracite", -# "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", -# "Electric underfloor heating", -# "Warm air, Electricaire" -# ] -# -# portfolio = pd.read_excel( -# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" -# ) -# portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] -# portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -# portfolio['sheating-energy-eff'] = None -# portfolio['sheating-env-eff'] = None -# portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) -# -# print(portfolio["mainheat-description"].value_counts()) -# -# eg = portfolio[ -# (portfolio["mainheat-description"] == "Warm air, mains gas") -# ].sample(1) -# eg = eg.squeeze().to_dict() -# print(eg) diff --git a/recommendations/tests/test_data/materials.py b/recommendations/tests/test_data/materials.py index 194971e9..13b1ea08 100644 --- a/recommendations/tests/test_data/materials.py +++ b/recommendations/tests/test_data/materials.py @@ -1,6 +1,30 @@ import datetime materials = [ + { + 'id': 2484, + 'type': 'room_roof_insulation', + 'description': 'Room in roof insulation', + 'depth': 100, + 'depth_unit': 'mm', + 'cost_unit': 'gbp_per_m2', + 'r_value_per_mm': 0.038, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SCIS', + 'created_at': '2025-03-16 15:26:22.379', + 'cost': None, + 'is_active': True, + 'prime_material_cost': None, + 'material_cost': 0.0, + 'labour_cost': 0.0, + 'labour_hours_per_unit': 0.0, + 'plant_cost': 0.0, + 'total_cost': 210.0, + 'notes': None, + 'is_installer_quote': True + }, {'id': 1997, 'type': 'cavity_wall_insulation', 'description': 'Imperial Bead cavity wall insulation', 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index ed2e037d..b1ac4d18 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -92,23 +92,13 @@ class TestHeatingRecommendations: recommender = HeatingRecommender(property_instance=p) # Check they're empty assert not recommender.heating_recommendations - assert not recommender.heating_control_recommendations recommender.recommend(has_cavity_or_loft_recommendations=False) assert len(recommender.heating_recommendations) == len(test_case["heating_measure_types"]) - assert ( - len(recommender.heating_control_recommendations) == - len(test_case["heating_controls_measure_types"]) - ) # Check the exact measure types assert ( {x["measure_type"] for x in recommender.heating_recommendations} == set(test_case["heating_measure_types"]) ) - - assert ( - {x["measure_type"] for x in recommender.heating_control_recommendations} == - set(test_case["heating_controls_measure_types"]) - ) diff --git a/recommendations/tests/test_lighting_recommendations.py b/recommendations/tests/test_lighting_recommendations.py index 32d607de..4b1e1d84 100644 --- a/recommendations/tests/test_lighting_recommendations.py +++ b/recommendations/tests/test_lighting_recommendations.py @@ -41,18 +41,18 @@ class TestLightingRecommendations: assert len(lr.recommendation) == 1 assert lr.recommendation == [ - { - 'phase': 0, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting', - 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, - 'new_u_value': None, - 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, - 'description_simulation': { - 'lighting-energy-eff': 'Very Good', - 'lighting-description': 'Low energy lighting in all fixed outlets', - 'low-energy-lighting': 100 - }, - 'total': 240.24, 'subtotal': 200.20000000000002, - 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, - 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False - } + {'phase': 0, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting', + 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, 'new_u_value': None, + 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'energy_cost_savings': 54.4434, + 'co2_equivalent_savings': 0.035478, 'description_simulation': { + 'lighting-energy-eff': 'Very Good', + 'lighting-description': 'Low energy ' + 'lighting in all ' + 'fixed outlets', + 'low-energy-lighting': 100 + }, + 'total': 240.24, + 'subtotal': 200.20000000000002, 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, + 'material': 80.0, 'profit': 28.6, 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, + 'survey': False} ] From 382e04ea7a441f56570dbd1918fd28fba76165aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 15 May 2025 15:02:35 +0100 Subject: [PATCH 8/9] fixing costing unit tests --- .idea/Model.iml | 7 - asset_list/app.py | 28 ++++ .../places_for_people/finalise_programme.py | 143 ++++++++++++++++++ recommendations/tests/test_costs.py | 9 +- .../test_data/heating_recommendations_data.py | 5 +- .../tests/test_floor_recommendations.py | 2 + 6 files changed, 178 insertions(+), 16 deletions(-) create mode 100644 etl/customers/places_for_people/finalise_programme.py diff --git a/.idea/Model.iml b/.idea/Model.iml index df6c4faa..c6561970 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -10,11 +10,4 @@ - - - \ No newline at end of file diff --git a/asset_list/app.py b/asset_list/app.py index e3c612a7..bb898c09 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -182,6 +182,34 @@ def app(): # master_filepaths = [] # master_to_asset_list_filepath = None + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West" + data_filename = "Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE.xlsx" + sheet_name = "CHECKED" + postcode_column = 'Postcode' + fulladdress_column = None + address1_column = "AddressLine1" + address1_method = None + address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"] + missing_postcodes_method = None + landlord_year_built = None + landlord_os_uprn = None + landlord_property_type = "Archetype (PFP)" + landlord_built_form = "Archetype (PFP)" + landlord_wall_construction = None + landlord_roof_construction = None + landlord_heating_system = None + landlord_existing_pv = None + landlord_property_id = "Uprn" + outcomes_filename = None + outcomes_sheetname = None + outcomes_postcode = None + outcomes_houseno = None + outcomes_id = None + master_filepaths = [] + master_to_asset_list_filepath = None + landlord_sap = None + phase = None + # Maps addresses to uprn in problematic cases manual_uprn_map = {} diff --git a/etl/customers/places_for_people/finalise_programme.py b/etl/customers/places_for_people/finalise_programme.py new file mode 100644 index 00000000..bb612ebf --- /dev/null +++ b/etl/customers/places_for_people/finalise_programme.py @@ -0,0 +1,143 @@ +""" +Having produced the 4 standardsied asset lists for PFP, this script performs a final review +on those assets, reconciling against a list of properties that they sent us that indicates the +properties that they have retained, acquired and then the list will also include some properties that we +have never seen before and so might require additional inspections +""" + +import pandas as pd +import numpy as np +import os +from tqdm import tqdm + + +def match_to_list(pfp_reconciliation_list, asset_list): + lookup = [] + for _, asset in tqdm(pfp_reconciliation_list.iterrows(), total=pfp_reconciliation_list.shape[0]): + _id = str(asset['PRO PROPREF']) + # Match to the asset list - we check the bas ID and then we test removing leading zeros + matched = asset_list[asset_list["landlord_property_id"] == _id] + if matched.empty: + _id_stripped = _id.lstrip("0") + matched = asset_list[asset_list["landlord_property_id"] == _id_stripped] + + if not matched.empty: + lookup.append( + { + "reconciliation_id": _id, + "landlord_property_id": matched["landlord_property_id"].values[0], + } + ) + + lookup = pd.DataFrame(lookup) + asset_list["reconciliation"] = np.where( + asset_list["landlord_property_id"].isin( + lookup["landlord_property_id"].values + ), + "Property still owned by PFP", + "Property not owned by PFP" + ) + + return asset_list, lookup + + +data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/Finalise Programme" + +pfp_reconciliation_list = pd.read_excel( + os.path.join(data_folder, "PFP properties w repair responsibility.xlsx"), +) +# London +pfp_london = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - areas surrounding London - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_london["landlord_property_id"] = pfp_london["landlord_property_id"].astype(str) + +# North-East +pfp_ne = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - North East - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_ne["landlord_property_id"] = pfp_ne["landlord_property_id"].astype(str) + +# North-West +pfp_nw = pd.read_excel( + os.path.join( + data_folder, + "Standardised Asset Lists/Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE - " + "Standardised.xlsx" + ), + sheet_name="Standardised Asset List" +) +pfp_nw["landlord_property_id"] = pfp_nw["landlord_property_id"].astype(str) + +# East +pfp_east = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - East - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_east["landlord_property_id"] = pfp_east["landlord_property_id"].astype(str) + +pfp_london, lookup_london = match_to_list(pfp_reconciliation_list, pfp_london) +pfp_ne, lookup_ne = match_to_list(pfp_reconciliation_list, pfp_ne) +pfp_nw, lookup_nw = match_to_list(pfp_reconciliation_list, pfp_nw) +pfp_east, lookup_east = match_to_list(pfp_reconciliation_list, pfp_east) + +pfp_london["reconciliation"].value_counts() +pfp_ne["reconciliation"].value_counts() +pfp_nw["reconciliation"].value_counts() +pfp_east["reconciliation"].value_counts() + +# We store the reconciled datasets +pfp_london.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - areas surrounding London - reconciled.csv"), + index=False +) +pfp_ne.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - North East - reconciled.csv"), + index=False +) +pfp_nw.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - North West - reconciled.csv"), + index=False +) +pfp_east.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - East - reconciled.csv"), + index=False +) + +pd.set_option('display.max_columns', None) +pd.set_option('display.width', 1000) + +# We look at what was on the reconciled list, that was NOT on the original list +all_ids = lookup_london["reconciliation_id"].tolist() + \ + lookup_ne["reconciliation_id"].tolist() + \ + lookup_nw["reconciliation_id"].tolist() + \ + lookup_east["reconciliation_id"].tolist() +missed_inspections = pd.read_excel( + os.path.join( + data_folder, + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West/Places for People NORTH " + "WEST - INSPECTIONS MASTER - UPDATE.xlsx" + ), + sheet_name="MISSING STILL" +) +missed_inspections.columns = ["landlord_id", "address"] + +not_seen = pfp_reconciliation_list[ + ~pfp_reconciliation_list["PRO PROPREF"].astype(str).isin(all_ids) +].copy() + +not_seen["Note"] = None +not_seen["Note"] = np.where( + not_seen["PRO PROPREF"].astype(str).isin(missed_inspections["landlord_id"].astype(str).values) | + not_seen["PRO PROPREF"].astype(str).str.lstrip("0").isin(missed_inspections["landlord_id"].astype(str).values), + "Property not inspected", + not_seen["Note"] +) +not_seen["Note"] = not_seen["Note"].fillna("Property not in original lists") + +# Store +not_seen = os.path.join( + data_folder, "Reconciled Programme/Properties not inspected by Domna.xlsx" +) diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 74a210c1..4b8d74db 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -1,6 +1,5 @@ from recommendations.Costs import Costs from unittest.mock import Mock -import datetime import pytest @@ -298,10 +297,10 @@ class TestCosts: # Test for different wattages @pytest.mark.parametrize("n_panels, expected_cost", [ - (7, 4055.0), - (10, 4540.0), - (12, 4863.0), - (15, 5707.0), + (7, 5458.727999999999), + (10, 6013.139999999999), + (12, 6386.447999999999), + (15, 7594.451999999999), ]) def test_solar_pv_different_wattages(self, n_panels, expected_cost): mock_property = Mock() diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 53f8bd25..a7f4121e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -179,13 +179,10 @@ testing_examples = [ 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, "heating_measure_types": [ - 'boiler_upgrade', 'roomstat_programmer_trvs', 'time_temperature_zone_control', ], - "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to " - "the current water heating efficiency) the only recommendation we expect is for " - "a boiler upgrade. The heating controls are programmer and thermostat, so we can also recommend" + "notes": "The heating controls are programmer and thermostat, so we can also recommend" "better heating controls" }, { diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index 17f1f82e..eb4f30d2 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -65,6 +65,7 @@ class TestFloorRecommendations: input_properties[2].number_of_floors = 1 input_properties[2].floor_level = 0 input_properties[2].already_installed = [] + input_properties[2].non_invasive_recommendations = {} recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials) assert recommender.estimated_u_value is None @@ -115,6 +116,7 @@ class TestFloorRecommendations: input_properties[4].number_of_floors = 1 input_properties[4].floor_level = 0 input_properties[4].already_installed = [] + input_properties[4].non_invasive_recommendations = {} # In this case, we have no county, so in this case, it should yse the local-authority-label if possible input_properties[4].data["county"] = "" From bea5901bc280e4f931a06650d07f49a4856fd459 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 15 May 2025 16:25:05 +0100 Subject: [PATCH 9/9] Added year built test --- etl/epc/tests/test_epcrecord.py | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/etl/epc/tests/test_epcrecord.py b/etl/epc/tests/test_epcrecord.py index cf0361b1..37a00cd4 100644 --- a/etl/epc/tests/test_epcrecord.py +++ b/etl/epc/tests/test_epcrecord.py @@ -356,3 +356,64 @@ class TestEpcRecord: assert record.prepared_epc["solar-water-heating-flag"] == "N" assert record.solar_water_heating_flag_bool is False + + def test_year_built(self, cleaning_data): + # This test handles a specific test case + # Mock the property object + + epc_records = { + "original_epc": { + 'low-energy-fixed-light-count': '', 'address': '19 Waterloo Road, Shoeburyness', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.65', 'heating-cost-potential': '436', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '100', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '89', 'glazed-type': 'double glazing installed during or after 2002', + 'heating-cost-current': '888', 'address3': '', + 'mainheatcont-description': 'Programmer and room thermostat', + 'sheating-energy-eff': 'N/A', 'report-type': '100', 'property-type': 'House', + 'local-authority-label': 'Southend-on-Sea', 'fixed-lighting-outlets-count': '9', + 'energy-tariff': 'Single', + 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '386', 'county': '', + 'postcode': 'SS3 9EQ', + 'solar-water-heating-flag': 'N', 'constituency': 'E14001501', 'co2-emissions-potential': '0.7', + 'number-heated-rooms': '4', 'floor-description': 'Suspended, no insulation (assumed)', + 'energy-consumption-potential': '49', 'local-authority': 'E06000033', 'built-form': 'Mid-Terrace', + 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', + 'inspection-date': '2025-03-17', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '58', + 'address1': '19 Waterloo Road', 'heat-loss-corridor': '', 'flat-storey-count': '', + 'constituency-label': '', + 'roof-energy-eff': 'Average', 'total-floor-area': '78.0', 'building-reference-number': '10007286268', + 'environment-impact-current': '48', 'co2-emissions-current': '4.5', + 'roof-description': 'Pitched, 100 mm loft insulation', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '4', 'address2': 'Shoeburyness', 'hot-water-env-eff': 'Average', + 'posttown': 'SOUTHEND-ON-SEA', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 78% of fixed outlets', + 'roof-env-eff': 'Average', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '101', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', + 'main-heating-controls': '', 'lodgement-datetime': '2025-03-25 16:59:15', 'flat-top-storey': '', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'marketed sale', 'uprn': 100090702270, 'current-energy-efficiency': '56', + 'energy-consumption-current': '329', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '101', 'lodgement-date': '2025-03-25', 'extension-count': '1', + 'mainheatc-env-eff': 'Average', + 'lmk-key': 'ff00a1e150063f7bbcac1644be57fdcf05b6c9c60053f80c5d218bf2863fea93', + 'wind-turbine-count': '0', + 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '89', + 'hot-water-energy-eff': 'Average', 'low-energy-lighting': '78', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "full_sap_epc": {}, + "old_data": [] + } + + prepared_epc = EPCRecord( + epc_records=epc_records, + run_mode="newdata", + cleaning_data=cleaning_data + ) + + assert prepared_epc.get("year_built") == 1900