fixing costing unit tests

.idea/Model.iml

@@ -10,11 +10,4 @@
     <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
-  <component name="PyNamespacePackagesService">
-    <option name="namespacePackageFolders">
-      <list>
-        <option value="$MODULE_DIR$/local_data" />
-      </list>
-    </option>
-  </component>
 </module>

asset_list/app.py

@@ -182,6 +182,34 @@ def app():
     # master_filepaths = []
     # master_to_asset_list_filepath = None
 
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West"
+    data_filename = "Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE.xlsx"
+    sheet_name = "CHECKED"
+    postcode_column = 'Postcode'
+    fulladdress_column = None
+    address1_column = "AddressLine1"
+    address1_method = None
+    address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"]
+    missing_postcodes_method = None
+    landlord_year_built = None
+    landlord_os_uprn = None
+    landlord_property_type = "Archetype (PFP)"
+    landlord_built_form = "Archetype (PFP)"
+    landlord_wall_construction = None
+    landlord_roof_construction = None
+    landlord_heating_system = None
+    landlord_existing_pv = None
+    landlord_property_id = "Uprn"
+    outcomes_filename = None
+    outcomes_sheetname = None
+    outcomes_postcode = None
+    outcomes_houseno = None
+    outcomes_id = None
+    master_filepaths = []
+    master_to_asset_list_filepath = None
+    landlord_sap = None
+    phase = None
+
     # Maps addresses to uprn in problematic cases
     manual_uprn_map = {}
 

etl/customers/places_for_people/finalise_programme.py

@@ -0,0 +1,143 @@
+"""
+Having produced the 4 standardsied asset lists for PFP, this script performs a final review
+on those assets, reconciling against a list of properties that they sent us that indicates the
+properties that they have retained, acquired and then the list will also include some properties that we
+have never seen before and so might require additional inspections
+"""
+
+import pandas as pd
+import numpy as np
+import os
+from tqdm import tqdm
+
+
+def match_to_list(pfp_reconciliation_list, asset_list):
+    lookup = []
+    for _, asset in tqdm(pfp_reconciliation_list.iterrows(), total=pfp_reconciliation_list.shape[0]):
+        _id = str(asset['PRO PROPREF'])
+        # Match to the asset list - we check the bas ID and then we test removing leading zeros
+        matched = asset_list[asset_list["landlord_property_id"] == _id]
+        if matched.empty:
+            _id_stripped = _id.lstrip("0")
+            matched = asset_list[asset_list["landlord_property_id"] == _id_stripped]
+
+        if not matched.empty:
+            lookup.append(
+                {
+                    "reconciliation_id": _id,
+                    "landlord_property_id": matched["landlord_property_id"].values[0],
+                }
+            )
+
+    lookup = pd.DataFrame(lookup)
+    asset_list["reconciliation"] = np.where(
+        asset_list["landlord_property_id"].isin(
+            lookup["landlord_property_id"].values
+        ),
+        "Property still owned by PFP",
+        "Property not owned by PFP"
+    )
+
+    return asset_list, lookup
+
+
+data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/Finalise Programme"
+
+pfp_reconciliation_list = pd.read_excel(
+    os.path.join(data_folder, "PFP properties w repair responsibility.xlsx"),
+)
+# London
+pfp_london = pd.read_excel(
+    os.path.join(data_folder, "Standardised Asset Lists/PFP - areas surrounding London - Standardised.xlsx"),
+    sheet_name="Standardised Asset List"
+)
+pfp_london["landlord_property_id"] = pfp_london["landlord_property_id"].astype(str)
+
+# North-East
+pfp_ne = pd.read_excel(
+    os.path.join(data_folder, "Standardised Asset Lists/PFP - North East - Standardised.xlsx"),
+    sheet_name="Standardised Asset List"
+)
+pfp_ne["landlord_property_id"] = pfp_ne["landlord_property_id"].astype(str)
+
+# North-West
+pfp_nw = pd.read_excel(
+    os.path.join(
+        data_folder,
+        "Standardised Asset Lists/Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE - "
+        "Standardised.xlsx"
+    ),
+    sheet_name="Standardised Asset List"
+)
+pfp_nw["landlord_property_id"] = pfp_nw["landlord_property_id"].astype(str)
+
+# East
+pfp_east = pd.read_excel(
+    os.path.join(data_folder, "Standardised Asset Lists/PFP - East - Standardised.xlsx"),
+    sheet_name="Standardised Asset List"
+)
+pfp_east["landlord_property_id"] = pfp_east["landlord_property_id"].astype(str)
+
+pfp_london, lookup_london = match_to_list(pfp_reconciliation_list, pfp_london)
+pfp_ne, lookup_ne = match_to_list(pfp_reconciliation_list, pfp_ne)
+pfp_nw, lookup_nw = match_to_list(pfp_reconciliation_list, pfp_nw)
+pfp_east, lookup_east = match_to_list(pfp_reconciliation_list, pfp_east)
+
+pfp_london["reconciliation"].value_counts()
+pfp_ne["reconciliation"].value_counts()
+pfp_nw["reconciliation"].value_counts()
+pfp_east["reconciliation"].value_counts()
+
+# We store the reconciled datasets
+pfp_london.to_csv(
+    os.path.join(data_folder, "Reconciled Programme/PFP - areas surrounding London - reconciled.csv"),
+    index=False
+)
+pfp_ne.to_csv(
+    os.path.join(data_folder, "Reconciled Programme/PFP - North East - reconciled.csv"),
+    index=False
+)
+pfp_nw.to_csv(
+    os.path.join(data_folder, "Reconciled Programme/PFP - North West - reconciled.csv"),
+    index=False
+)
+pfp_east.to_csv(
+    os.path.join(data_folder, "Reconciled Programme/PFP - East - reconciled.csv"),
+    index=False
+)
+
+pd.set_option('display.max_columns', None)
+pd.set_option('display.width', 1000)
+
+# We look at what was on the reconciled list, that was NOT on the original list
+all_ids = lookup_london["reconciliation_id"].tolist() + \
+          lookup_ne["reconciliation_id"].tolist() + \
+          lookup_nw["reconciliation_id"].tolist() + \
+          lookup_east["reconciliation_id"].tolist()
+missed_inspections = pd.read_excel(
+    os.path.join(
+        data_folder,
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West/Places for People NORTH "
+        "WEST - INSPECTIONS MASTER - UPDATE.xlsx"
+    ),
+    sheet_name="MISSING STILL"
+)
+missed_inspections.columns = ["landlord_id", "address"]
+
+not_seen = pfp_reconciliation_list[
+    ~pfp_reconciliation_list["PRO PROPREF"].astype(str).isin(all_ids)
+].copy()
+
+not_seen["Note"] = None
+not_seen["Note"] = np.where(
+    not_seen["PRO PROPREF"].astype(str).isin(missed_inspections["landlord_id"].astype(str).values) |
+    not_seen["PRO PROPREF"].astype(str).str.lstrip("0").isin(missed_inspections["landlord_id"].astype(str).values),
+    "Property not inspected",
+    not_seen["Note"]
+)
+not_seen["Note"] = not_seen["Note"].fillna("Property not in original lists")
+
+# Store
+not_seen = os.path.join(
+    data_folder, "Reconciled Programme/Properties not inspected by Domna.xlsx"
+)

recommendations/tests/test_costs.py

@@ -1,6 +1,5 @@
 from recommendations.Costs import Costs
 from unittest.mock import Mock
-import datetime
 import pytest
 
 
@@ -298,10 +297,10 @@ class TestCosts:
 
     # Test for different wattages
     @pytest.mark.parametrize("n_panels, expected_cost", [
-        (7, 4055.0),
+        (7, 5458.727999999999),
-        (10, 4540.0),
+        (10, 6013.139999999999),
-        (12, 4863.0),
+        (12, 6386.447999999999),
-        (15, 5707.0),
+        (15, 7594.451999999999),
     ])
     def test_solar_pv_different_wattages(self, n_panels, expected_cost):
         mock_property = Mock()

recommendations/tests/test_data/heating_recommendations_data.py

@@ -179,13 +179,10 @@ testing_examples = [
             'uprn': 100021560521.0, 'uprn-source': 'Address Matched',
         },
         "heating_measure_types": [
-            'boiler_upgrade',
             'roomstat_programmer_trvs',
             'time_temperature_zone_control',
         ],
-        "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to "
+        "notes": "The heating controls are programmer and thermostat, so we can also recommend"
-                 "the current water heating efficiency) the only recommendation we expect is for "
-                 "a boiler upgrade. The heating controls are programmer and thermostat, so we can also recommend"
                  "better heating controls"
     },
     {

recommendations/tests/test_floor_recommendations.py

@@ -65,6 +65,7 @@ class TestFloorRecommendations:
         input_properties[2].number_of_floors = 1
         input_properties[2].floor_level = 0
         input_properties[2].already_installed = []
+        input_properties[2].non_invasive_recommendations = {}
 
         recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials)
         assert recommender.estimated_u_value is None
@@ -115,6 +116,7 @@ class TestFloorRecommendations:
         input_properties[4].number_of_floors = 1
         input_properties[4].floor_level = 0
         input_properties[4].already_installed = []
+        input_properties[4].non_invasive_recommendations = {}
 
         # In this case, we have no county, so in this case, it should yse the local-authority-label if possible
         input_properties[4].data["county"] = ""