Merge pull request #295 from Hestia-Homes/immo-pilot

Immo pilot

backend/SearchEpc.py

@@ -709,8 +709,13 @@ class SearchEpc:
                 self.full_sap_epc = {}
 
                 # Finally, set a standardised address 1 and postcode
-                self.address_clean = self.ordnance_survey_client.address_os
+                self.address_clean = (
-                self.postcode_clean = self.ordnance_survey_client.postcode_os
+                    self.ordnance_survey_client.address_os if self.ordnance_survey_client.address_os else self.address1
+                )
+                self.postcode_clean = (
+                    self.ordnance_survey_client.postcode_os if self.ordnance_survey_client.postcode_os else
+                    self.postcode
+                )
             return
 
         os_response = self.ordnance_survey_client.get_places_api()

backend/app/plan/router.py

@@ -52,6 +52,10 @@ def patch_epc(patch, epc_records):
     """
 
     for patch_variable, patch_value in patch.items():
+
+        if patch_variable in ["address", "postcode"]:
+            continue
+
         if patch_value == "":
             continue
         if patch_variable in epc_records["original_epc"]:
@@ -268,23 +272,26 @@ async def trigger_plan(body: PlanTriggerRequest):
                 postcode=config["postcode"],
                 uprn=uprn,
                 auth_token=get_settings().EPC_AUTH_TOKEN,
-                os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY
+                os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY,
             )
-            epc_searcher.find_property()
+            epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None)
+            epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None)
+            # For the moment, our OS API access is unavailable, so we skip and interpolate
+            epc_searcher.find_property(skip_os=True)
             # Create a record in db
             property_id, is_new = create_property(
                 session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn
             )
-            if not is_new:
+            # if not is_new:
-                continue
+            #     continue
-
+            #
-            create_property_targets(
+            # create_property_targets(
-                session,
+            #     session,
-                property_id=property_id,
+            #     property_id=property_id,
-                portfolio_id=body.portfolio_id,
+            #     portfolio_id=body.portfolio_id,
-                epc_target=body.goal_value,
+            #     epc_target=body.goal_value,
-                heat_demand_target=None
+            #     heat_demand_target=None
-            )
+            # )
 
             epc_records = {
                 'original_epc': epc_searcher.newest_epc.copy(),
@@ -373,6 +380,7 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         logger.info("Preparing data for scoring in sap change api")
         recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
+
         recommendations_scoring_data = recommendations_scoring_data.drop(
             columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
                      "carbon_ending"]

backend/ml_models/Valuation.py

@@ -63,6 +63,14 @@ class PropertyValuation:
         90093693: 279_000,  # Based on Zoopla
         90055152: 149_000,  # Based on Zoopla
         90028499: 238_000,  # Based on Zoopla
+        # IMMO Dudley Pilot 2- search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
+        90039318: 177_000,  # Based on Zoopla
+        90038384: 170_000,  # Based on Zoopla
+        90105380: 185_000,  # Based on Zoopla
+        90124001: 165_000,  # Based on Zoopla
+        90013980: 148_000,  # Based on Zoopla
+        90087154: 184_000,  # Based on Zoopla
+        90046817: 167_000,  # Based on Zoopla
     }
 
     # We base our valuation uplifts on a number of sources

etl/customers/gla_croydon_demo/asset_list.py

@@ -34,8 +34,9 @@ def app():
         low_memory=False
     )
 
-    z = epc_data.groupby(["WALLS_DESCRIPTION", "WALLS_ENERGY_EFF"]).size().reset_index(name="count")
+    z = epc_data[epc_data["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"]
-    z = z[z["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"]
+    z["HOTWATER_DESCRIPTION"].value_counts()
+    z["MAIN_FUEL"].value_counts()
 
     # Filter on entries where we have a UPRN
     epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]

etl/customers/immo/pilot/asset_list_2.py

@@ -0,0 +1,152 @@
+import pandas as pd
+from utils.s3 import read_excel_from_s3
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 72
+
+# For
+patches = [
+    {
+        'address': '116 Parkes Hall Road',
+        'postcode': 'DY1 3RJ',
+        'uprn': '90046817',
+        'walls-description': 'Cavity wall, filled cavity',
+        'walls-energy-eff': 'Average',
+        'roof-description': 'Pitched, 270 mm loft insulation',
+        'roof-energy-eff': 'Good',
+        'windows-description': 'Fully double glazed',
+        'windows-energy-eff': 'Good',
+        'mainheat-description': 'Boiler and radiators, mains gas',
+        'mainheat-energy-eff': 'Good',
+        'mainheatcont-description': 'Programmer, room thermostat and TRVs',
+        'mainheatc-energy-eff': 'Good',
+        'lighting-description': 'Low energy lighting in 27% of fixed outlets',
+        'lighting-energy-eff': 'Average',
+        'floor-description': 'Solid, no insulation (assumed)',
+        'secondheat-description': 'None',
+        'current-energy-efficiency': '73',
+        'current-energy-rating': 'C',
+        'energy-consumption-current': '184',
+        'co2-emissions-current': '2.4',
+        'potential-energy-efficiency': '88',
+        'total-floor-area': '73',
+        'construction-age-band': 'England and Wales: 1930-1949',
+        'property-type': 'House',
+        'built-form': 'Mid-Terrace',
+    }
+]
+
+# This is information that is found as a result of the non-invasives, that mean that certain measures
+# have been installed already. To reflect this in the front end, it is included in the recommendation, however
+# the cost is removed and instead, a message is presented saying that the measure is already installed.
+already_installed = [
+    {
+        'address': '28 Sangwin Road', 'postcode': 'WV14 9EQ', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '51 Hillwood Road', 'postcode': 'B62 8NQ', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '47 Watsons Close', 'postcode': 'DY2 7HL', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '44 Hatfield Road',
+        'postcode': 'DY9 7LW',
+        "already_installed": ["loft_insulation", "cavity_wall_insulation"]
+    }
+]
+
+non_invasive_recommendations = []
+
+
+def app():
+    raw_asset_list = read_excel_from_s3(
+        bucket_name="retrofit-datalake-dev",
+        file_key="customers/Immo/Dudley Asset List - Hestia - pilot2.xlsx",
+        header_row=0
+    )
+
+    raw_asset_list = raw_asset_list[raw_asset_list["in_pilot"]].copy()
+
+    # Extract address and postcode
+    raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0]
+    raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip()
+
+    # We're provided with number of bathrooms and number of bedrooms.
+    # THe UPRNs are not the official ones
+    asset_list = raw_asset_list.rename(
+        columns={
+            "No. of Beds": "n_bedrooms",
+            "No. of WC's": "n_bathrooms",
+            'Property Type': 'property_type',
+            'Architype': 'built_form'
+        }
+    )
+
+    # Remap the values
+    asset_list["built_form"] = asset_list["built_form"].map({
+        "SEMI DETACHED": "Semi-Detached",
+        "MID TERRACE": "Mid-Terrace",
+        "END TERRACE": "End-Terrace",
+    })
+
+    # Store the asset list in s3
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    # Store overrides in s3
+    already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(already_installed),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=already_installed_filename
+    )
+
+    # Store patches in s3
+    patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(patches),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=patches_filename
+    )
+
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    # EPC C portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)
+
+    # EPC B portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID + 1),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "B",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)

etl/epc/Record.py

@@ -191,7 +191,7 @@ class EPCRecord:
         This method will clean the records using the data processor
         """
         epc_data_processor = EPCDataProcessor(
-            data=self.epc_record_as_dataframe("prepared_epc"),
+            data=self.epc_record_as_dataframe("prepared_epc").copy(),
             run_mode="newdata",
             cleaning_averages=self.cleaning_data,
         )

recommendations/Costs.py

@@ -67,18 +67,12 @@ LOW_CARBON_COMBI_BOILER = 2200
 # https://www.greenmatch.co.uk/boilers/35kw-boiler
 # https://www.greenmatch.co.uk/boilers/40kw-boiler
 # These are exclusive of installation costs
-COMBI_BOILER_COSTS = {
+CONDENSING_BOILER_COSTS = {
     "30kw": 1550,
     "35kw": 1610,
     "40kw": 1625
 }
 
-CONVENTIONAL_BOILER_COSTS = {
-    "30kw": 1117,
-    "35kw": 1546,
-    "40kw": 1776
-}
-
 # Assumes 3 hours to remove each heater (including re-decorating)
 ROOM_HEATER_REMOVAL_COST = 120
 ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3
@@ -1179,7 +1173,7 @@ class Costs:
         estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators)
         return round(estimated_radiators)
 
-    def boiler(self, is_combi, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms):
+    def boiler(self, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms):
         """
         Based on a basic estimate of median value £2600 to install a low carbon combi boiler
         First time central heating vosts can als be found here:
@@ -1187,7 +1181,7 @@ class Costs:
         :return:
         """
 
-        unit_cost = COMBI_BOILER_COSTS[size] if is_combi else CONVENTIONAL_BOILER_COSTS[size]
+        unit_cost = CONDENSING_BOILER_COSTS[size]
         # The unit cost is the cost without VAT
         # We now need to estimate the cost of the works
         labour_days = 2

recommendations/HeatingRecommender.py

@@ -15,7 +15,8 @@ class HeatingRecommender:
         self.property = property_instance
         self.costs = Costs(self.property)
 
-        self.recommendations = []
+        self.heating_recommendations = []
+        self.heating_control_recommendations = []
 
     def recommend(self, phase=0):
 
@@ -23,7 +24,8 @@ class HeatingRecommender:
         #       the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this
         #       in the Costs class, stored as SYSTEM_FLUSH_COST
 
-        self.recommendations = []
+        self.heating_recommendations = []
+        self.heating_control_recommendations = []
         # This first iteration of the recommender will provide very basic recommendation
         # We recommend heating controls based on the main heating system
 
@@ -254,7 +256,7 @@ class HeatingRecommender:
             system_change=system_change
         )
 
-        self.recommendations.extend(recommendations)
+        self.heating_recommendations.extend(recommendations)
 
     @staticmethod
     def estimate_boiler_size(property_type, built_form, floor_area, floor_height, num_heated_rooms):
@@ -312,7 +314,15 @@ class HeatingRecommender:
         simulation_config = {}
         boiler_costs = {}
         boiler_recommendation = {}
-        if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]:
+
+        has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]
+
+        has_inefficient_mains_water = (
+            self.property.hotwater["clean_description"] in ["From main system"] and
+            self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"]
+        )
+
+        if has_inefficient_space_heating or has_inefficient_mains_water:
             boiler_size = self.estimate_boiler_size(
                 property_type=self.property.data["property-type"],
                 built_form=self.property.data["built-form"],
@@ -321,22 +331,12 @@ class HeatingRecommender:
                 num_heated_rooms=self.property.data["number-heated-rooms"],
             )
 
-            # We recommend a combi boiler under the following conditions
+            description = "Upgrade to a new condensing boiler"
-            # 1) If there are 4 or fewer rooms (we don't use heqted rooms because none of the rooms could be
-            #    heated if there is no existing heating system).
-            # 2) There 1 or fewer bathrooms
-            # Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple
-            # bathrooms
-            is_combi = (
-                (self.property.number_of_rooms <= 4) and
-                (self.property.n_bathrooms in [None, 0, 1])
-            )
-            if is_combi:
-                description = "Upgrade to a new combi boiler"
-            else:
-                description = "Upgrade to a new gas condensing boiler"
 
-            simulation_config = {"mainheat_energy_eff_ending": "Good"}
+            simulation_config = {
+                "mainheat_energy_eff_ending": "Good",
+                "hot_water_energy_eff_ending": "Good"
+            }
             if system_change:
                 # Installation of a boiler improves the hot water system so we need to reflect this in
                 # the outcome of the recommendation
@@ -363,7 +363,6 @@ class HeatingRecommender:
                 }
 
             boiler_costs = self.costs.boiler(
-                is_combi=is_combi,
                 size=f"{boiler_size}kw",
                 exising_room_heaters=exising_room_heaters,
                 system_change=system_change,
@@ -397,9 +396,13 @@ class HeatingRecommender:
         controls_recommender.recommend(heating_description="Boiler and radiators, mains gas")
         # We may have 2 recommendations from the heating controls
 
-        if not controls_recommender.recommendation:
+        if not controls_recommender.recommendation and not boiler_recommendation:
             return
 
+        if not system_change and len(boiler_recommendation):
+            # If there is not a system change, we add the boiler recommendation at point.
+            self.heating_recommendations.extend([boiler_recommendation])
+
         if system_change:
             # We combine the heating and controls recommendations, in the case of a system change
             combined_recommendations = []
@@ -416,12 +419,12 @@ class HeatingRecommender:
                 combined_recommendations.extend(combined_recommendation)
 
             # Overwrite the existing boiler recommendation
-            self.recommendations.extend(combined_recommendations)
+            self.heating_recommendations.extend(combined_recommendations)
         else:
             # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade
             # but we'll only upgrade if we have a heating recommendation
             has_heating_recommendation = any(
-                recommendation["type"] == "heating" for recommendation in self.recommendations
+                rec["type"] == "heating" for rec in self.heating_recommendations
             )
             if has_heating_recommendation:
                 recommendation_phase += 1
@@ -430,6 +433,6 @@ class HeatingRecommender:
             for recommendation in controls_recommender.recommendation:
                 recommendation["phase"] = recommendation_phase
 
-            self.recommendations.extend(controls_recommender.recommendation)
+            self.heating_control_recommendations.extend(controls_recommender.recommendation)
 
         return

recommendations/Recommendations.py

@@ -110,12 +110,24 @@ class Recommendations:
         # Heating and Electical systems
         if "heating" not in self.exclusions:
             self.heating_recommender.recommend(phase=phase)
-            if self.heating_recommender.recommendations:
+            if (
-                property_recommendations.append(self.heating_recommender.recommendations)
+                self.heating_recommender.heating_recommendations or
+                self.heating_recommender.heating_control_recommendations
+            ):
+                if self.heating_recommender.heating_recommendations:
+                    property_recommendations.append(self.heating_recommender.heating_recommendations)
+
+                if self.heating_recommender.heating_control_recommendations:
+                    property_recommendations.append(self.heating_recommender.heating_control_recommendations)
+
                 # We check if we have distinct heating and heating controls recommendations
                 # If so, we increment by 2 (one of the heating system, one for the heating controls)
                 # otherwise we incremenet by 1
-                max_used_phase = max([rec["phase"] for rec in self.heating_recommender.recommendations])
+                max_used_phase = max(
+                    [rec["phase"] for rec in
+                     self.heating_recommender.heating_recommendations +
+                     self.heating_recommender.heating_control_recommendations]
+                )
                 amount_to_increment = max_used_phase - phase + 1
                 phase += amount_to_increment
 

recommendations/SolarPvRecommendations.py

@@ -56,14 +56,18 @@ class SolarPvRecommendations:
         if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv:
             return
 
+        solar_pv_percentage = self.property.solar_pv_percentage
+        # We round up to the neaest 10%
+        solar_pv_percentage = np.ceil(solar_pv_percentage * 10) / 10
+
         # For the solar recommendations, we produce the following scenarios:
         # 1) Solar panels only, we present a high, medium and low coverage
         # 2) With and without battery
         roof_coverage_scenarios = [
-            self.property.solar_pv_percentage - 0.1, self.property.solar_pv_percentage,
+            solar_pv_percentage - 0.1, solar_pv_percentage,
         ]
-        if self.property.solar_pv_percentage <= 0.4:
+        if solar_pv_percentage <= 0.4:
-            roof_coverage_scenarios.append(self.property.solar_pv_percentage + 0.1)
+            roof_coverage_scenarios.append(solar_pv_percentage + 0.1)
         # We make sure we haven't gone too low or high - we allow no more than 60% coverage
         roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 0.6]
         # If we only have two scenarios, we add a coverage scenario 10% less than the smallest