From 753ef66f4891b0070d549f32c1f91b4bb70e720f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 1 Dec 2025 13:40:07 +0000 Subject: [PATCH] allowing us not to retrieve find my epc data, when we very clearly don't have the right epc --- backend/apis/GoogleSolarApi.py | 2 +- .../db/functions/recommendations_functions.py | 32 ++++++----------- backend/app/plan/utils.py | 2 +- etl/find_my_epc/RetrieveFindMyEpc.py | 35 ++++++++++++------- 4 files changed, 34 insertions(+), 37 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index a0970ac5..f7aa311f 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -723,7 +723,7 @@ class GoogleSolarApi: # We set the target rating to EPC C, which is the typical EPC rating we would expect the # property to achieve post retrofit of just the fabric "energy_consumption": cls.estimate_new_consumption( - current_energy_efficiency=min(p.data["current-energy-efficiency"], 100), + current_energy_efficiency=min(int(p.data["current-energy-efficiency"]), 100), target_efficiency="69", current_consumption=p.estimate_electrical_consumption( assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 9d01f288..55bf5824 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -175,31 +175,19 @@ def chunked(iterable, size=100): def fast_delete_recommendations(session, chunk): - placeholders = ",".join([f"({i})" for i in range(len(chunk))]) + placeholders = ",".join(["(:p{})".format(i) for i in range(len(chunk))]) + params = {f"p{i}": chunk[i] for i in range(len(chunk))} sql = text(f""" - WITH ids(property_id) AS ( - VALUES {placeholders} - ) - DELETE FROM recommendation r - USING ids - WHERE r.property_id = ids.property_id; - """) + WITH ids(property_id) AS ( + VALUES {placeholders} + ) + DELETE FROM recommendation r + USING ids + WHERE r.property_id = ids.property_id; + """) - session.execute(sql, execution_options={"synchronize_session": False}) - - # Note; we may be able to go even faster like this: - # def delete_with_temp_table(session, chunk): - # session.execute(text("CREATE TEMP TABLE tmp_ids (id bigint) ON COMMIT DROP;")) - # - # insert_sql = "INSERT INTO tmp_ids (id) VALUES " + ",".join(f"({i})" for i in chunk) - # session.execute(text(insert_sql)) - # - # session.execute(text(""" - # DELETE FROM recommendation r - # USING tmp_ids t - # WHERE r.property_id = t.id; - # """)) + session.execute(sql, params, execution_options={"synchronize_session": False}) def clear_portfolio(session: Session, portfolio_id: int, batch_size=100): diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index d8c54963..ebf1dd9c 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -43,7 +43,7 @@ def patch_epc(patch, epc_records): if patch_variable in ["address", "postcode"]: continue - if patch_value == "": + if patch_value in ["", None]: continue if patch_variable in epc_records["original_epc"]: epc_records["original_epc"][patch_variable] = patch_value diff --git a/etl/find_my_epc/RetrieveFindMyEpc.py b/etl/find_my_epc/RetrieveFindMyEpc.py index 5bb0e89c..2e23c7e6 100644 --- a/etl/find_my_epc/RetrieveFindMyEpc.py +++ b/etl/find_my_epc/RetrieveFindMyEpc.py @@ -407,7 +407,10 @@ class RetrieveFindMyEpc: ) if not extracted_table and not backup_flat: - raise ValueError("No EPC found") + # This is a relatively new change, as of November 2025, but we see cases where properties do not + # have data appearing on the find my EPC website, particularly for older EPCs. In this case, we allo + # for us to not find any information and return nothing + return None, None if not extracted_table: extracted_table = deepcopy(backup_flat) @@ -430,6 +433,11 @@ class RetrieveFindMyEpc: if epc_page_source is None and rrn is None: chosen_epc, rrn = self._find_epc_page() + if chosen_epc is None: + # We have no resulting data + logger.info("No EPC found for address %s, postcode %s", self.address, self.postcode) + return {} + address_response = requests.get(chosen_epc, headers=self.HEADERS) epc_page_source = address_response.text address_res = BeautifulSoup(address_response.text, features="html.parser") @@ -458,10 +466,11 @@ class RetrieveFindMyEpc: current_sap = int(current_rating.split(' ')[-1]) if current_sap != self.sap_rating: - raise ValueError( - f"SAP rating mismatch: expected {self.sap_rating}, got {current_sap} for address {self.address}, " - f"postcode {self.postcode}" - ) + # This means we likely have the wrong data. If we are in this scenario, we return nothing + return { + "epc_certificate": None, + "page_source": None, + } # Retrieve the energy consumption bills = address_res.find('div', {'id': 'bills-affected'}) @@ -775,21 +784,21 @@ class RetrieveFindMyEpc: "uprn": epc["uprn"], "address": epc["address"], "postcode": epc["postcode"], - "recommendations": find_epc_data["recommendations"], + "recommendations": find_epc_data.get("recommendations", []), } # We need to add the patch information patch = { - "current-energy-rating": find_epc_data["current_epc_rating"], - "current-energy-efficiency": find_epc_data["current_epc_efficiency"], - "potential-energy-rating": find_epc_data["potential_epc_rating"], - "potential-energy-efficiency": find_epc_data["potential_epc_efficiency"], - **find_epc_data["epc_data"], + "current-energy-rating": find_epc_data.get("current_epc_rating"), + "current-energy-efficiency": find_epc_data.get("current_epc_efficiency"), + "potential-energy-rating": find_epc_data.get("potential_epc_rating"), + "potential-energy-efficiency": find_epc_data.get("potential_epc_efficiency"), + **find_epc_data.get("epc_data", {}), } page_source = { - "rrn": find_epc_data["epc_certificate"], - "page_source": find_epc_data["page_source"] + "rrn": find_epc_data.get("epc_certificate"), + "page_source": find_epc_data.get("page_source") } return non_invasive_recommendations, patch, page_source