adding additional coverage to RetrieveFindMyEpc

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Stonewater-wave-3" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Stonewater-wave-3" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

backend/app/plan/router.py

@@ -366,7 +366,7 @@ def extract_property_request_data(
         property_non_invasive_recommendations["recommendations"] = str(transformed)
 
     property_valution = next((
-        float(x["value"]) for x in valuation_data if
+        float(x["valuation"]) for x in valuation_data if
         (str(x["uprn"]) == str(uprn))
     ), None)
 
@@ -611,6 +611,7 @@ async def trigger_plan(body: PlanTriggerRequest):
                     property_instance=property_instance,
                     all_predictions=all_predictions,
                     recommendations=recommendations,
+                    representative_recommendations=representative_recommendations
                 )
             )
 

etl/customers/cottons/remote_assessments.py

@@ -10,8 +10,8 @@ from utils.s3 import save_csv_to_s3
 
 load_dotenv(dotenv_path="backend/.env")
 EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
-PORTFOLIO_ID = 121
 USER_ID = 8
+PORTFOLIO_ID = 121
 
 
 def app():
@@ -22,7 +22,8 @@ def app():
 
     # Read in the asset list
     cottons_asset_list = pd.read_excel(
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Cottons/Cottons Asset List EPC Data Pull.xlsx"
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Cottons/Cottons Asset List EPC Data Pull with "
+        "valuations.xlsx"
     )
     # A number are missing EPCs due to the space in the postcode
     # Breakdowns:
@@ -79,6 +80,9 @@ def app():
         } for r in extracted_data
     ]
 
+    valuations_data = asset_list[["uprn", "Zoopla Valuation"]].copy().rename(columns={"Zoopla Valuation": "valuation"})
+    valuations_data = valuations_data[~pd.isnull(valuations_data["valuation"])]
+
     filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
     save_csv_to_s3(
         dataframe=pd.DataFrame(model_asset_list),
@@ -94,6 +98,14 @@ def app():
         file_name=non_invasive_recommendations_filename
     )
 
+    # Store the valuations data in s3
+    valuations_filename = f"{USER_ID}/{PORTFOLIO_ID}/valuations.csv"
+    save_csv_to_s3(
+        dataframe=valuations_data,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=valuations_filename
+    )
+
     body = {
         "portfolio_id": str(PORTFOLIO_ID),
         "housing_type": "Social",
@@ -103,9 +115,10 @@ def app():
         "already_installed_file_path": "",
         "patches_file_path": "",
         "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
-        "valuation_file_path": "",
+        "valuation_file_path": valuations_filename,
         "scenario_name": "Wave 3 Packages",
         "multi_plan": True,
         "budget": None,
+        "exclusions": ['air_source_heat_pump', 'boiler_upgrade', 'floor_insulation']
     }
     print(body)

etl/find_my_epc/RetrieveFindMyEpc.py

@@ -282,7 +282,8 @@ class RetrieveFindMyEpc:
             "Low energy lighting for all fixed outlets": ["low_energy_lighting"],
             "Cylinder thermostat recommendation": [],
             "Heating controls recommendation": [],
-            "Replace boiler with Band A condensing boiler": [],
+            "Replace boiler with Band A condensing boiler": ["boiler_upgrade"],
+            "Band A condensing gas boiler": ["boiler_upgrade"],
             "Solar panel recommendation": [],
             "Double glazing recommendation": [],
             "Solid wall insulation recommendation": [],
@@ -296,6 +297,17 @@ class RetrieveFindMyEpc:
             "Cylinder thermostat": ["cylinder_thermostat"],
             "Heat recovery system for mixer showers": ["heat_recovery_shower"],
             "Room-in-roof insulation": ["room_in_roof_insulation"],
+            "Fan assisted storage heaters": [],
+            "Fan-assisted storage heaters": [],
+            "Step 1:": [],
+            "Biomass stove with boiler": [],
+            "Replace boiler with biomass boiler": [],
+            "Heating controls (room thermostat and thermostatic radiator valves)": [
+                "roomstat_programmer_trvs", "time_temperature_zone_control"
+            ],
+            "Heating controls (programmer, and thermostatic radiator valves)": [
+                "roomstat_programmer_trvs", "time_temperature_zone_control"
+            ],
         }
 
         survey = True

etl/route_march_data_pull/app.py

@@ -25,6 +25,7 @@ def get_data(asset_list, fulladdress_column, address1_column, postcode_column):
     epc_data = []
     errors = []
     no_epc = []
+    # home = asset_list[asset_list["row_id"] == errors[15]].squeeze()
     for _, home in tqdm(asset_list.iterrows(), total=len(asset_list)):
         try:
             postcode = home[postcode_column]
@@ -94,7 +95,7 @@ def get_data(asset_list, fulladdress_column, address1_column, postcode_column):
                 )
                 find_epc_data = find_epc_searcher.retrieve_newest_find_my_epc_data()
             except ValueError as e:
-                if "No EPC found" in str(e):
+                if "No EPC found" in str(e) and "address1" in searcher.newest_epc:
                     find_epc_searcher = RetrieveFindMyEpc(
                         address=searcher.newest_epc["address1"], postcode=searcher.newest_epc["postcode"]
                     )
@@ -151,17 +152,17 @@ def app():
     Property UPRN
 
     """
-    DATA_FOLDER = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Cottons/"
-    DATA_FILENAME = "Cottons Asset List.xlsx"
-    SHEET_NAME = "Sheet1"
-    POSTCODE_COLUMN = "postcode"
-    FULLADDRESS_COLUMN = "Property Address"
-    ADDRESS1_COLUMN = "address1"
-    ADDRESS1_METHOD = None
-    ADDRESS_COLS_TO_CONCAT = []
+    DATA_FOLDER = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford"
+    DATA_FILENAME = "BROMFORD - SOLAR PV ROOFs INSPECTED - Electric only properties getting to C list.xlsx"
+    SHEET_NAME = "MAIN"
+    POSTCODE_COLUMN = "Post Code"
+    FULLADDRESS_COLUMN = "Full Address"
+    ADDRESS1_COLUMN = None
+    ADDRESS1_METHOD = "first_two_words"
+    ADDRESS_COLS_TO_CONCAT = ["House No", "Street", "District"]
 
     asset_list = pd.read_excel(os.path.join(DATA_FOLDER, DATA_FILENAME), header=0, sheet_name=SHEET_NAME)
-    # asset_list = asset_list[~pd.isnull(asset_list[POSTCODE_COLUMN])].reset_index()
+    asset_list = asset_list[~pd.isnull(asset_list[POSTCODE_COLUMN])].reset_index()
     asset_list["row_id"] = asset_list.index
 
     # We clean up portential non-breaking spaces, and double spaces
@@ -249,6 +250,8 @@ def app():
         [
             "row_id",
             "uprn",
+            "address1",
+            "postcode",
             "property-type",
             "built-form",
             "inspection-date",
@@ -256,6 +259,7 @@ def app():
             "current-energy-efficiency",
             "roof-description",
             "walls-description",
+            "floor-description",
             "transaction-type",
             # New fields needed
             "secondheat-description",
@@ -268,7 +272,7 @@ def app():
             "energy-consumption-current",  # kwh/m2
             "photo-supply",
         ]
-    ]
+    ].rename(columns={"address1": "Address1 on EPC", "postcode": "Postcode on EPC"})
 
     asset_list = asset_list.merge(
         epc_df,
@@ -308,6 +312,7 @@ def app():
         "number-habitable-rooms": "Number of Habitable Rooms",
         "walls-description": "Wall Construction",
         "roof-description": "Roof Construction",
+        "floor-description": "Floor Construction",
         "mainheat-description": "Heating Type",
         "secondheat-description": "Secondary Heating",
         "transaction-type": "Reason for last EPC",
@@ -363,3 +368,7 @@ def app():
     # Store as an excel
     filename = os.path.join(DATA_FOLDER, ".".join(DATA_FILENAME.split(".")[:-1])) + " EPC Data Pull.xlsx"
     asset_list.to_excel(filename, index=False)
+
+    matches_review = asset_list[
+        [FULLADDRESS_COLUMN, ADDRESS1_COLUMN, POSTCODE_COLUMN, "Address1 on EPC", "Postcode on EPC"]
+    ]

recommendations/HeatingRecommender.py

@@ -1111,17 +1111,14 @@ class HeatingRecommender:
         if not controls_recommender.recommendation and not boiler_recommendation:
             return
 
-        # If this is true, we set SAP points to None and survey to False for the boiler recommendation
-        if boiler_recommendation:
-            boiler_recommendation["sap_points"] = None
-            boiler_recommendation["survey"] = False
-
         if not system_change and len(boiler_recommendation):
             # If there is not a system change, we add the boiler recommendation at point.
             self.heating_recommendations.extend([boiler_recommendation])
 
         if system_change:
             # We combine the heating and controls recommendations, in the case of a system change
+            # If this is true, we set SAP points to None and survey to False for the boiler recommendation
+
             combined_recommendations = []
             for controls_recommendation in controls_recommender.recommendation:
                 combined_recommendation = self.combine_heating_and_controls(

recommendations/Recommendations.py

@@ -311,7 +311,7 @@ class Recommendations:
                 continue
 
             has_u_value = recommendations_by_type[0].get("new_u_value") is not None
-            has_sap_points = recommendations_by_type[0].get("sap_points") is not None
+            has_sap_points = all([r.get("sap_points") is not None for r in recommendations_by_type])
             has_rank = recommendations_by_type[0].get("rank") is not None
 
             # When check if these recommendations have two different types, such as solid wall insulation
@@ -449,6 +449,7 @@ class Recommendations:
         property_instance,
         all_predictions,
         recommendations,
+        representative_recommendations,
     ):
 
         """
@@ -473,6 +474,9 @@ class Recommendations:
 
         property_recommendations = recommendations[property_instance.id].copy()
 
+        representative_recs = representative_recommendations[property_instance.id].copy()
+        representative_ids = [r["recommendation_id"] for r in representative_recs]
+
         increasing_variables = ["sap"]
         decreasing_variables = ["carbon", "heat_demand"]
 
@@ -530,7 +534,9 @@ class Recommendations:
 
                 else:
 
-                    previous_phase_values_multiple = [x for x in impact_summary if x["phase"] == (rec["phase"] - 1)]
+                    previous_phase_values_multiple = [
+                        x for x in impact_summary if x["phase"] == (rec["phase"] - 1) and x["representative"]
+                    ]
                     if len(previous_phase_values_multiple) != 1:
                         # Take an average of each of the previous phases
                         keys_to_median = ["sap", "carbon", "heat_demand"]
@@ -628,7 +634,9 @@ class Recommendations:
                 impact_summary.append(
                     {
                         "phase": rec["phase"],
+                        "representative": rec["recommendation_id"] in representative_ids,
                         "recommendation_id": rec["recommendation_id"],
+                        "measure_type": rec["measure_type"],
                         **current_phase_values
                     }
                 )