major bulk update

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="AssetList" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

asset_list/AssetList.py

@@ -450,6 +450,8 @@ class AssetList:
         self.non_intrusives_present = "CIGA Check Required" in self.raw_asset_list.columns
         # We detect if we have the old format of non-intruvies
         self.old_format_non_intrusives_present = "WFT Findings" in self.raw_asset_list.columns
+        if self.old_format_non_intrusives_present:
+            self.non_intrusives_present = False
 
         self.non_intrusives_eligibility = "Eligibility (Red/Yellow/Green)" in self.raw_asset_list.columns
 
@@ -1320,7 +1322,7 @@ class AssetList:
 
         # Before we being, we identify if a property has solar already as we use this
         # for identifying cavity jobs
-        if self.non_intrusives_present:
+        if self.non_intrusives_present and not self.old_format_non_intrusives_present:
 
             if self.new_format_non_insturives_present_v2:
                 existing_solar_non_intrusives_check = (
@@ -3011,7 +3013,7 @@ class AssetList:
             outcomes["row_id"] = outcomes.index
 
             if outcomes_houseno[idx] is None:
-                outcomes_houseno = "houseno"
+                outcomes_houseno[idx] = "houseno"
                 outcomes["houseno"] = outcomes[outcomes_address[idx]].apply(
                     lambda x: SearchEpc.get_house_number(x, outcomes[outcomes_postcode])
                 )
@@ -3286,9 +3288,14 @@ class AssetList:
             else:
                 raise ValueError("No install or cancellation date")
 
-            submission_col = (
-                "SUBMISSION DATE" if "SUBMISSION DATE" in master_data.columns else "SUBMISSION DATE TO INSTALLERS"
-            )
+            if "SUBMISSION DATE" in master_data.columns:
+                submission_col = "SUBMISSION DATE"
+            elif "SUBMISSION DATE TO INSTALLERS" in master_data.columns:
+                submission_col = "SUBMISSION DATE TO INSTALLERS"
+            elif "Submission Date" in master_data.columns:
+                submission_col = "Submission Date"
+            else:
+                raise ValueError("No submission date column found in master data")
 
             master_data["row_id"] = master_data.index
 
@@ -3331,11 +3338,21 @@ class AssetList:
                 installer_notes_col = "INSTALLERS NOTES ; REASONS FOR CANCELLATIONS"
             elif "INSTALLERS NOTES" in master_data.columns:
                 installer_notes_col = "INSTALLERS NOTES"
+            elif 'Installers Notes' in master_data.columns:
+                installer_notes_col = 'Installers Notes'
+            elif 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM' in master_data.columns:
+                installer_notes_col = 'NOTES ; REASONS FOR CANCELLATIONS OR WHERE INSTALL DATE WAS OBTAINED FROM'
             else:
                 raise ValueError("No installer notes column found in master data")
 
+            if "INSTALLER" in master_data.columns:
+                installer_col = "INSTALLER"
+            elif "Installer" in master_data.columns:
+                installer_col = "Installer"
+            else:
+                raise ValueError("No installer column found in master data")
+
             measure_mix_col = "MEASURE COMBO"
-            installer_col = "INSTALLER"
             town_colname = "TOWN" if "TOWN" in master_data.columns else 'Town/Area'
 
             logger.info("Matching master data to asset list")

asset_list/app.py

@@ -59,42 +59,227 @@ def app():
     Property UPRN
     """
 
-    # Pickering and Ferens
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Pickering & Ferens"
-    data_filename = "SAP 9 vs SAP 10 Sava Intelligent Energy - Property List (190625).xlsx"
-    sheet_name = "Sava Intelligent Energy - Prope"
+    # CDS
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
+    data_filename = "Founder Estates - Asset List.xlsx"
+    sheet_name = "Combined"
     postcode_column = 'Postcode'
-    fulladdress_column = 'Address'
+    fulladdress_column = "Address"
     address1_column = None
     address1_method = "house_number_extraction"
     address_cols_to_concat = []
     missing_postcodes_method = None
     landlord_year_built = None
     landlord_os_uprn = None
-    landlord_property_type = "Property Type"  # Using the inspections property type
-    landlord_built_form = "Archetype 2"
+    landlord_property_type = None
+    landlord_built_form = None
     landlord_wall_construction = None
-    landlord_roof_construction = None
-    landlord_heating_system = None
+    landlord_heating_system = "Heating Type"
     landlord_existing_pv = None
-    landlord_property_id = "UPRN"
-    landlord_sap = "SAP Rating (RdSAP 10)"
+    landlord_property_id = "Row ID"
     outcomes_filename = []
     outcomes_sheetname = []
     outcomes_postcode = []
     outcomes_houseno = []
-    outcomes_id = []
     outcomes_address = []
-    master_filepaths = [
-        os.path.join(data_folder, "PICKERING & FERENS ROLLING MASTER SHEET HEDGEFUND - 26.7.24 - K.csv"),
-        os.path.join(data_folder, "PICKERING & FERENS NEW MASTER GBIS UPDATED 21.8.24 - M - For Analysis.csv"),
-    ]
+    outcomes_id = []
+    master_filepaths = []
     master_to_asset_list_filepath = None
-    phase = False
-    ecosurv_landlords = "pickering"
     asset_list_header = 0
     landlord_block_reference = None
+    master_id_colnames = []
+    landlord_roof_construction = None
+    phase = False
+    landlord_sap = None
+    ecosurv_landlords = None
+
+    # Plus Dane
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
+    data_filename = "20250711 Plus Dane Asset List.xlsx"
+    sheet_name = "Sheet1"
+    postcode_column = 'Postcode'
+    fulladdress_column = "Address"
+    address1_column = None
+    address1_method = "house_number_extraction"
+    address_cols_to_concat = []
+    missing_postcodes_method = None
+    landlord_year_built = "Property Age"
+    landlord_os_uprn = None
+    landlord_property_type = "Property Type"
+    landlord_built_form = "Built Form"
+    landlord_wall_construction = "Wall Construction"
+    landlord_heating_system = "Full Heating System"
+    landlord_existing_pv = None
+    landlord_property_id = "UPRN"
+    outcomes_filename = [
+        os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
+        os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
+        os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
+    ]
+    outcomes_sheetname = [
+        "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
+    ]
+    outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
+    outcomes_houseno = ["No.", "No", "No"]
+    outcomes_address = ["Address", "Address", "Address"]
+    outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
+    master_filepaths = [
+        os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
+        os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
+    ]
+    master_to_asset_list_filepath = None
+    asset_list_header = 1
+    landlord_block_reference = None
     master_id_colnames = [None, None]
+    landlord_roof_construction = None
+    phase = False
+    landlord_sap = "SAP Rating"
+    ecosurv_landlords = "plus dane"
+
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme"
+    # data_filename = "20250710 Asset List Brentwood.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = None
+    # address1_column = "House Number"
+    # address1_method = None
+    # address_cols_to_concat = ["House Number", "Address Line 1", "Address Line 2", "Address Line 3"]
+    # missing_postcodes_method = None
+    # landlord_year_built = "Year Built"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Dwelling"
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Heating"
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # outcomes_filename = [os.path.join(data_folder, "Brentwood - outcomes for analysis.xlsx")]
+    # outcomes_sheetname = ["OUTCOMES"]
+    # outcomes_postcode = ["POSTCODE"]
+    # outcomes_houseno = [None]
+    # outcomes_address = ["ADDRESS"]
+    # outcomes_id = [None]
+    # master_filepaths = [os.path.join(data_folder, "Submissions.csv")]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 1
+    # landlord_block_reference = None
+    # master_id_colnames = [None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = "brentwood"
+
+    # Brentwood
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme"
+    # data_filename = "20250710 Asset List Brentwood.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = None
+    # address1_column = "House Number"
+    # address1_method = None
+    # address_cols_to_concat = ["House Number", "Address Line 1", "Address Line 2", "Address Line 3"]
+    # missing_postcodes_method = None
+    # landlord_year_built = "Year Built"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Dwelling"
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Heating"
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # outcomes_filename = [os.path.join(data_folder, "Brentwood - outcomes for analysis.xlsx")]
+    # outcomes_sheetname = ["OUTCOMES"]
+    # outcomes_postcode = ["POSTCODE"]
+    # outcomes_houseno = [None]
+    # outcomes_address = ["ADDRESS"]
+    # outcomes_id = [None]
+    # master_filepaths = [os.path.join(data_folder, "Submissions.csv")]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 1
+    # landlord_block_reference = None
+    # master_id_colnames = [None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = "brentwood"
+    #
+    # # Eastlight
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Eastlight/New Programme"
+    # data_filename = "INSPECTIONS MASTER Non Tech.xlsx"
+    # sheet_name = "EASTLIGHT CW"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = None
+    # address1_column = "HouseName"
+    # address1_method = None
+    # address_cols_to_concat = ["HouseName", "Block", "Address1", "Address2", "Address3"]
+    # missing_postcodes_method = None
+    # landlord_year_built = "Built In Year"
+    # landlord_os_uprn = None
+    # landlord_property_type = "AssetType"
+    # landlord_built_form = "Archetype"  # Using inspections archetype
+    # landlord_wall_construction = None
+    # landlord_roof_construction = None
+    # landlord_heating_system = "Main Heating Source"
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # landlord_sap = "SAP Score"
+    # outcomes_filename = [
+    #     os.path.join(data_folder, "Eastlight_CWI_JJC_2025.xlsx"),
+    #     os.path.join(data_folder, "Eastlight_CWI_SCIS_2025.xlsx"),
+    # ]
+    # outcomes_sheetname = ["Outcomes", "Feedback"]
+    # outcomes_postcode = ["Postcode", "Postcode"]
+    # outcomes_houseno = ["No", "No."]
+    # outcomes_id = [None, None]
+    # outcomes_address = ["Address", "Address"]
+    # master_filepaths = [
+    #     os.path.join(data_folder, "ECO 3-Table 1.csv"),
+    #     os.path.join(data_folder, "ECO 4-Table 1.csv"),
+    # ]
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = "eastlight"
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = [None, None]
+    # landlord_sap = None
+
+    # Pickering and Ferens
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Pickering & Ferens"
+    # data_filename = "SAP 9 vs SAP 10 Sava Intelligent Energy - Property List (190625).xlsx"
+    # sheet_name = "Sava Intelligent Energy - Prope"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = 'Address'
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = "Property Type"  # Using the inspections property type
+    # landlord_built_form = "Archetype 2"
+    # landlord_wall_construction = None
+    # landlord_roof_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # landlord_sap = "SAP Rating (RdSAP 10)"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_id = []
+    # outcomes_address = []
+    # master_filepaths = [
+    #     os.path.join(data_folder, "PICKERING & FERENS ROLLING MASTER SHEET HEDGEFUND - 26.7.24 - K.csv"),
+    #     os.path.join(data_folder, "PICKERING & FERENS NEW MASTER GBIS UPDATED 21.8.24 - M - For Analysis.csv"),
+    # ]
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = "pickering"
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = [None, None]
 
     # Colchester
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester"

asset_list/mappings/built_form.py

@@ -365,6 +365,12 @@ BUILT_FORM_MAPPINGS = {
     'DETACHED': 'detached',
     'MID TERRACE': 'mid-terrace',
     'END TERRACE': 'end-terrace',
-    'ENCLOSED MID': 'enclosed mid-terrace'
+    'ENCLOSED MID': 'enclosed mid-terrace',
 
+    'BUILDING': 'unknown',
+    'FLAT COMMUNAL FACILITIES': 'unknown',
+    'MAISONETTE': 'unknown',
+    'HOUSE': 'unknown',
+    'FLAT': 'unknown',
+    'BLOCK': 'unknown'
 }

asset_list/mappings/heating_systems.py

@@ -341,5 +341,28 @@ HEATING_MAPPINGS = {
     'Boiler: D rated Combi': 'gas condensing combi',
     'Heat Pump: (from database)': 'air source heat pump',
     'Community Heating Systems: Community CHP and boilers (RdSAP)': 'communal heating',
-    '': 'unknown'
+    '': 'unknown',
+
+    'Solid Fuel Boiler': 'solid fuel',
+    'Heating (Other)': 'other',
+    'Solid Fuel Fire Only': 'solid fuel',
+    'No Main Heat Source': 'no heating',
+    'Electric Programmable': 'electric storage heaters',
+    'Linked to Communal Boiler': 'communal heating',
+    'Bio Mass Boiler': 'solid fuel',
+    'Electric Non Programmable': 'electric storage heaters',
+
+    'Room heaters, Mains gas': 'room heaters',
+    'Boiler, Solid fuel': 'solid fuel',
+    'Room heaters, Electricity': 'room heaters',
+    'Room heaters, Solid fuel': 'room heaters',
+    'Boiler, Oil': 'oil boiler',
+    'Boiler, Biomass': 'boiler - other fuel',
+    'Community heating, Community (non-gas)': 'communal heating',
+    'Heat pump (wet), Electricity': 'air source heat pump',
+    'Community heating, Community (mains gas)': 'communal gas boiler',
+    'Boiler, Electricity': 'electric boiler',
+    'Boiler, LPG': 'gas boiler, radiators',
+    'Boiler, Mains gas': 'gas boiler, radiators',
+    'Storage heating, Electricity': 'electric storage heaters'
 }

asset_list/mappings/property_type.py

@@ -266,5 +266,10 @@ PROPERTY_MAPPING = {
     'Office Block': 'other',
     'BLOCK (Non-Communal)': 'block of flats',
     'Refuge': 'other',
-    None: 'unknown'
+    None: 'unknown',
+    'HFOP FLAT': 'flat',
+    'HFOP BEDSIT': 'bedsit',
+    'LINKED FLAT': 'flat',
+    'LINKED BUNGALOW': 'bungalow'
+
 }

backend/Property.py

@@ -1200,7 +1200,7 @@ class Property:
         self.heating_energy_source = self.heating_energy_source[0]
 
         if self.heating_energy_source == "Varied (Community Scheme)":
-            if self.main_fuel["fuel_type"] == "mains gas":
+            if self.main_fuel["fuel_type"] in ["mains gas", None]:  # We assume when None as it's unknown
                 self.heating_energy_source = "Natural Gas (Community Scheme)"
             else:
                 raise Exception("Implement me")

backend/app/assumptions.py

@@ -64,6 +64,13 @@ DESCRIPTIONS_TO_FUEL_TYPES = {
     'Boiler and radiators, mains gas, Boiler and radiators, mains gas': {"fuel": "Natural Gas", "cop": 0.85},
     'Room heaters, electric, Electric storage heaters': {"fuel": "Electricity", "cop": 1},
     "Boiler and radiators, mains gas, Electric storage heaters": {"fuel": "Natural Gas", "cop": 0.85},
+    "Boiler and radiators, anthracite": {"fuel": "Anthracite", "cop": 0.85},
+    'Electric immersion, off-peak, plus solar': {"fuel": "Electricity + Solar Thermal", "cop": 1},
+    'Ground source heat pump, radiators, electric': {
+        "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100
+    },
+    'Electric instantaneous at point of use, plus solar': {"fuel": "Electricity + Solar Thermal", "cop": 1},
+    "Electric storage heaters, Room heaters, electric": {"fuel": "Electricity", "cop": 1},
 }
 
 # These are the measure types where if there is a ventilation recommendation, we force the inclusion of it

backend/engine/engine.py

@@ -506,7 +506,7 @@ async def model_engine(body: PlanTriggerRequest):
             )
 
             # if we have a remote assment data type, we pull the additional data and include it
-            if (body.event_type == "remote_assessment") and not (epc_searcher.newest_epc["estimated"]):
+            if (body.event_type == "remote_assessment") and not (epc_searcher.newest_epc.get("estimated")):
                 logger.info("Retrieving find my epc data")
                 try:
                     property_non_invasive_recommendations, patch = RetrieveFindMyEpc.get_from_epc(

backend/ml_models/AnnualBillSavings.py

@@ -28,8 +28,8 @@ class AnnualBillSavings:
 
     # Latest price cap figures from Ofgem are for April 2024
     # https://www.ofgem.gov.uk/energy-price-cap
-    ELECTRICITY_PRICE_CAP = 0.2486
-    GAS_PRICE_CAP = 0.0634
+    ELECTRICITY_PRICE_CAP = 0.2573
+    GAS_PRICE_CAP = 0.0633
     # This is the most recent export payment figure, at 9.28p/kWh
     # Smart export guarantee rates can be found here:
     # https://www.sunsave.energy/solar-panels-advice/exporting-to-the-grid/best-seg-rates
@@ -39,8 +39,8 @@ class AnnualBillSavings:
     PRICE_FACTOR = 0.09549999999999999
 
     # Daily standard charge, based on average across England, Scotland and Wales, and includes VAT
-    DAILY_STANDARD_CHARGE_GAS = 0.3165
-    DAILY_STANDARD_CHARGE_ELECTRICITY = 0.6097
+    DAILY_STANDARD_CHARGE_GAS = 0.2982
+    DAILY_STANDARD_CHARGE_ELECTRICITY = 0.5137
 
     # Based on https://www.nottenergy.com/advice-and-tools/project-energy-cost-comparison
     # For July 2024. These quotes are based on the east midlands region, so we

etl/customers/Brentwood/compile_new_asset_list.py

@@ -0,0 +1,38 @@
+"""
+Brentwood sent us a new asset list in July 2025. This script will combine the data in the new asset list with the
+old, so we have a single picture
+"""
+
+import pandas as pd
+
+new_asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme/All Assets "
+    "29.05.2025.xlsx",
+    sheet_name="Sheet1",
+    header=1
+)
+
+old_asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme/BRENTWOOD Asset "
+    "list.xlsx",
+    sheet_name="Asset List"
+)
+
+# We combine based on the data we want
+compiled = new_asset_list.merge(
+    old_asset_list[["UPRN", "Asset Type", "Year Built", "Dwelling", "Bedrooms", "Ownership", 'Asbestos Full Survey',
+                    'Stock Condition Survey', 'Cat', 'Heating',
+                    'WFT Findings', 'ECO Eligibility', 'CIGA Requested', 'CIGA Guarantee',
+                    'ECO Survey completed']],
+    how="left",
+    on="UPRN"
+)
+
+compiled["WFT Findings"] = compiled["WFT Findings"].fillna("Not Inspected")
+
+# Store this data
+compiled.to_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme/20250710 Asset List "
+    "Brentwood.xlsx",
+    index=False
+)

etl/customers/blakeridge_mill/data.py

@@ -0,0 +1,49 @@
+# Get units for postcodes WF17 8RA, WF17 8RB
+import os
+
+import pandas as pd
+from epc_api.client import EpcClient
+from dotenv import load_dotenv
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+postcodes = [
+    "WF17 8RA",
+    "WF17 8RB",
+]
+
+client = EpcClient(auth_token=EPC_AUTH_TOKEN)
+
+data = []
+for postcode in postcodes:
+    resp = client.domestic.search(
+        params={"postcode": postcode, "address": None, "local-authority": None, "property-type": None,
+                "floor-area": None,
+                "energy-band": None, "from-month": None, "from-year": None, "to-month": None, "to-year": None,
+                'constituency': None},
+        size=1000
+    )
+    data.extend(resp["rows"])
+
+df = pd.DataFrame(data)
+# Get newest field by UPRN, inspection-date
+df["inspection-date"] = pd.to_datetime(df["inspection-date"])
+df = df.sort_values(by=["uprn", "inspection-date"], ascending=[True, False])
+df = df.drop_duplicates(subset=["uprn"], keep="first")
+
+df.to_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Blakeridge Mill/blakeridge_mill_epc_data.xlsx", index=False
+)
+
+df = df[df["address"] != "The Tower Blakeridge Mill, Upper Blakeridge Lane"]
+df["walls-description"].value_counts()
+df["roof-description"].value_counts()
+
+df["total-floor-area"].astype(float).mean()
+df["current-energy-efficiency"] = pd.to_numeric(df["current-energy-efficiency"], errors='coerce')
+
+df.groupby("transaction-type")["current-energy-efficiency"].mean()
+df["transaction-type"].value_counts()
+
+df[df["transaction-type"] == "rental"]["built-form"].value_counts()

etl/customers/ealing/fixing houses asset list.py

@@ -0,0 +1,45 @@
+import pandas as pd
+
+houses_list = pd.read_csv(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Ealing BC - HOUSES(UNCHECKED).csv"
+)
+
+features = pd.read_csv(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Ealing BC - HOUSES(IGNORE - FULL MAIN).csv"
+)
+features = features.drop(
+    columns=[
+        'Archetype', 'Construction', 'Insulated', 'Material',
+        'CIGA Check Required', 'PV, ACCESS ISSUE, SEE NOTES',
+        'OFF GAS - ROOF ORIENTATION', 'Any further surveyor notes', 'Surveyors Name',
+        'Unnamed: 30', 'Unnamed: 31'
+    ]
+)
+
+demolitions = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Ealing - Demolished or due to be.xlsx",
+    sheet_name="Demolished or due to be"
+)
+
+inspections_data = houses_list[
+    [
+        "Property ref", "Postcode", 'Archetype', 'Construction', 'Insulated', 'Material',
+        'CIGA Check Required', 'PV, ACCESS ISSUE, SEE NOTES',
+        'OFF GAS - ROOF ORIENTATION', 'Any further surveyor notes', 'YET TO BE SURVEYED'
+    ]
+].rename(columns={"YET TO BE SURVEYED": "Surveyors Name"})
+
+asset_list = features.drop(
+    columns=[
+        'Archetype', 'Construction', 'Insulated', 'Material', 'CIGA Check Required',
+        'PV, ACCESS ISSUE, SEE NOTES', 'OFF GAS - ROOF ORIENTATION',
+        'Any further surveyor notes', 'Surveyors Name', "Postcode"
+    ]
+).merge(
+    inspections_data,
+    how="inner",
+    on="Property ref",
+)
+
+asset_list.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Ealing_rechecked_cleaned_05042025.csv",
+                  index=False)

etl/customers/ncha/portfolio.py

@@ -0,0 +1,14 @@
+import pandas as pd
+
+cavity = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Downloads/Energy Information MASTER June 2025 - Standardised.xlsx",
+    sheet_name="Cavity Properties (for review)",
+)
+solar = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Downloads/Energy Information MASTER June 2025 - Standardised.xlsx",
+    sheet_name="Solar Properties",
+)
+
+cavity_al = cavity[["domna_address_1", "domna_postcode", "epc_os_uprn"]].rename(
+    columns={"domna_address_1": "address", "domna_postcode": "postcode", "epc_os_uprn": "uprn"}
+)

etl/customers/plus dane/prepare_asset_list.py

@@ -0,0 +1,48 @@
+"""
+July 2025, this script prepares the asset list for Plus Dane
+"""
+import pandas as pd
+
+oldest_asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/PLUS DANE Asset List.xlsx"
+)
+solar_asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/Plus Dane - potential "
+    "PV List 04.03.2025.xlsx"
+)
+newest_asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/Sava Intelligent Energy "
+    "- Property List - March 2025.xlsx"
+)
+
+old_missed = oldest_asset_list[~oldest_asset_list["UPRN"].isin(newest_asset_list["UPRN"])]
+solar_missed = solar_asset_list[~solar_asset_list["UPRN"].isin(newest_asset_list["UPRN"])]  # Empty
+
+# Build new asset list
+# NEWEST
+# 'UPRN', 'Address', 'Postcode', 'Town', 'EPC SAP Band', 'SAP Rating',
+#        'CO₂ Emissions', 'EPC EI Band', 'Data Quality Indicator',
+#        'Results Calculated', 'Property Age', 'Property Type', 'Built Form',
+#        'Wall Construction', 'Wall Insulation', 'Roof Construction',
+#        'Joist Insulation', 'Space Heating System', 'Space Heating Fuel'
+#
+# SOlAR
+
+df = newest_asset_list.merge(
+    solar_asset_list, how="left", on="UPRN", suffixes=("", "_solar"),
+).merge(
+    oldest_asset_list, how="left", on="UPRN", suffixes=("", "_old")
+)
+df["asset_list_versiion"] = "July 2025"
+old_missed["asset_list_versiion"] = "Historic"
+
+# Append on the old missed?
+df = pd.concat(
+    [df, old_missed], ignore_index=True, sort=False
+)
+# Store excel
+df.to_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/Plus Dane Asset List "
+    "July 2025.xlsx",
+    index=False,
+)

etl/customers/remote_assessments/app.py

@@ -4,7 +4,7 @@ from dotenv import load_dotenv
 from utils.s3 import save_csv_to_s3
 from etl.find_my_epc.AssetListEpcData import AssetListEpcData
 
-PORTFOLIO_ID = 141
+PORTFOLIO_ID = 212
 USER_ID = 8
 
 load_dotenv(dotenv_path="backend/.env")
@@ -17,25 +17,15 @@ def app():
     :return:
     """
 
-    asset_list = [
-        {
-            "address": "196 Merrow Street",
-            "postcode": "SE17 2NP",
-            "uprn": 200003423454,
-            "patch": True
-        },
-        {
-            "address": "65 Liverpool Grove",
-            "postcode": "SE17 2HP",
-            "uprn": 200003423194
-        },
-        {
-            "address": "2 Brettell Street",
-            "postcode": "SE17 2NZ",
-            "uprn": 200003423607
-        },
-    ]
-    asset_list = pd.DataFrame(asset_list)
+    asset_list = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Downloads/Energy Information MASTER June 2025 - Standardised.xlsx",
+        sheet_name="Solar Properties",
+    )
+    asset_list = asset_list[~asset_list["estimated"]]
+    asset_list["domna_address_1"] = asset_list["domna_address_1"].astype(str)
+    asset_list = asset_list[["domna_address_1", "domna_postcode", "epc_os_uprn"]].rename(
+        columns={"domna_address_1": "address", "domna_postcode": "postcode", "epc_os_uprn": "uprn"}
+    )
 
     # Store the asset list in s3
     filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
@@ -98,14 +88,15 @@ def app():
         "portfolio_id": str(PORTFOLIO_ID),
         "housing_type": "Private",
         "goal": "Increasing EPC",
-        "goal_value": "C",
+        "goal_value": "A",
         "trigger_file_path": filename,
         "already_installed_file_path": "",
         "patches_file_path": patches_filename,
         "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
-        "valuation_file_path": valuation_filename,
+        "valuation_file_path": "",
         "scenario_name": "Full package remote assessment",
         "multi_plan": True,
         "budget": None,
+        "inclusions": ["cavity_wall_insulation", "ventilation"]
     }
     print(body)

etl/find_my_epc/RetrieveFindMyEpc.py

@@ -676,6 +676,9 @@ class RetrieveFindMyEpc:
                 "roomstat_programmer_trvs", "time_temperature_zone_control"
             ],
             "Internal wall insulation": ["internal_wall_insulation"],
+            "High heat retention storage heaters and dual immersion cylinder and dual rate meter": [
+                "high_heat_retention_storage_heater"
+            ]
         }
 
         survey = True
@@ -709,8 +712,13 @@ class RetrieveFindMyEpc:
             find_epc_data = searcher.retrieve_newest_find_my_epc_data()
         except Exception as e:
             logger.error(f"Error retrieving find my epc data: {e}")
+            if epc["address1"] == epc["address"]:
+                # There's no benefit of using the same address, so we split on comma
+                address1 = epc["address"].split(",")[0]
+            else:
+                address1 = epc["address1"]
             # We attempt with the backup add
-            searcher = cls(address=epc["address1"], postcode=epc["postcode"])
+            searcher = cls(address=address1, postcode=epc["postcode"])
             find_epc_data = searcher.retrieve_newest_find_my_epc_data()
 
         non_invasive_recommendations = {

recommendations/Recommendations.py

@@ -679,7 +679,7 @@ class Recommendations:
 
         # Handle the case of community schemes
         if (heating_description == "Community scheme") or (hotwater_description == "Community scheme"):
-            if main_fuel_description == "mains gas (community)":
+            if main_fuel_description in ["mains gas (community)", "UNKNOWN"]:
                 return {
                     "heating_fuel_type": "Natural Gas (Community Scheme)",
                     "hotwater_fuel_type": "Natural Gas (Community Scheme)",