Merge pull request #612 from Hestia-Homes/eco-eligiblity-bug

minor debugging

asset_list/app.py

@@ -59,25 +59,26 @@ def app():
     Property UPRN
     """
 
-    # Lambeth:
+    # Peabody data for cleaning
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lambeth/December 10th"
+    data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
-    data_filename = "lambeth_sw2_leigham court estate.xlsx"
+                   "Project/data_validation")
+    data_filename = "to_standardise_uprns.xlsx"
     sheet_name = "Sheet1"
     postcode_column = 'Postcode'
-    address1_column = "Address"
+    address1_column = "Address 1"
     address1_method = None
     fulladdress_column = None
-    address_cols_to_concat = ["Address"]
+    address_cols_to_concat = ["Address 1", "Address 2", "Address 3"]
     missing_postcodes_method = None
     landlord_year_built = None
     landlord_os_uprn = None
-    landlord_property_type = None
+    landlord_property_type = "Type"
-    landlord_built_form = None
+    landlord_built_form = "Attachment"
     landlord_wall_construction = None
     landlord_roof_construction = None
     landlord_heating_system = None
     landlord_existing_pv = None
-    landlord_property_id = "row_id"
+    landlord_property_id = "Org Ref"
     landlord_sap = None
     outcomes_filename = None
     outcomes_sheetname = None
@@ -93,6 +94,40 @@ def app():
     asset_list_header = 0
     landlord_block_reference = None
 
+    # Lambeth:
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lambeth/December 10th"
+    # data_filename = "lambeth_sw2_leigham court estate.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'Postcode'
+    # address1_column = "Address"
+    # address1_method = None
+    # fulladdress_column = None
+    # address_cols_to_concat = ["Address"]
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = None
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_roof_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "row_id"
+    # landlord_sap = None
+    # outcomes_filename = None
+    # outcomes_sheetname = None
+    # outcomes_postcode = None
+    # outcomes_houseno = None
+    # outcomes_id = None
+    # outcomes_address = None
+    # master_filepaths = []
+    # master_id_colnames = []
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+
     # Maps addresses to uprn in problematic cases
     manual_uprn_map = {}
 
@@ -230,22 +265,22 @@ def app():
         )
 
         # We now retrieve any failed properties
-        chunk_failed = chunk[chunk[asset_list.DOMNA_PROPERTY_ID].isin(errors_chunk)]
+        # chunk_failed = chunk[chunk[asset_list.DOMNA_PROPERTY_ID].isin(errors_chunk)]
-        epc_data_failed, _, _ = get_data(
+        # epc_data_failed, _, _ = get_data(
-            df=chunk_failed,
+        #     df=chunk_failed,
-            row_id_name=asset_list.DOMNA_PROPERTY_ID,
+        #     row_id_name=asset_list.DOMNA_PROPERTY_ID,
-            uprn_column=AssetList.STANDARD_UPRN,
+        #     uprn_column=AssetList.STANDARD_UPRN,
-            fulladdress_column=AssetList.STANDARD_FULL_ADDRESS,
+        #     fulladdress_column=AssetList.STANDARD_FULL_ADDRESS,
-            address1_column=AssetList.STANDARD_ADDRESS_1,
+        #     address1_column=AssetList.STANDARD_ADDRESS_1,
-            postcode_column=AssetList.STANDARD_POSTCODE,
+        #     postcode_column=AssetList.STANDARD_POSTCODE,
-            property_type_column=AssetList.STANDARD_PROPERTY_TYPE,
+        #     property_type_column=AssetList.STANDARD_PROPERTY_TYPE,
-            built_form_column=AssetList.STANDARD_BUILT_FORM,
+        #     built_form_column=AssetList.STANDARD_BUILT_FORM,
-            manual_uprn_map=manual_uprn_map,
+        #     manual_uprn_map=manual_uprn_map,
-            epc_api_only=epc_api_only,
+        #     epc_api_only=epc_api_only,
-            epc_auth_token=EPC_AUTH_TOKEN
+        #     epc_auth_token=EPC_AUTH_TOKEN
-        )
+        # )
-
+        #
-        epc_data_chunk.extend(epc_data_failed)
+        # epc_data_chunk.extend(epc_data_failed)
 
         # Append the failed data to the main data
         # Store the chunk locally as a csv
@@ -422,3 +457,7 @@ def app():
 
         if not asset_list.geographical_areas.empty:
             asset_list.geographical_areas.to_excel(writer, sheet_name="Geographical Areas", index=False)
+
+        # Store dupes
+        if not asset_list.duplicated_addresses.empty:
+            asset_list.duplicated_addresses.to_excel(writer, sheet_name="Duplicate Properties", index=False)

asset_list/mappings/built_form.py

@@ -458,6 +458,12 @@ BUILT_FORM_MAPPINGS = {
     'Maisonette: Detached: Mid Floor': 'detached',
     'Bungalow: EnclosedMidTerrace': 'enclosed mid-terrace',
 
-    'House: EnclosedMidTerrace': 'enclosed mid-terrace'
+    'House: EnclosedMidTerrace': 'enclosed mid-terrace',
+
+    'EnclosedMidTerrace': 'enclosed mid-terrace',
+    'EnclosedEndTerrace': 'enclosed end-terrace',
+    'EndTerrace': 'end-terrace',
+    'SemiDetached': 'semi-detached',
+    'MidTerrace': 'mid-terrace'
 
 }

backend/app/db/functions/recommendations_functions.py

@@ -1,4 +1,4 @@
-from sqlalchemy import insert, delete, text
+from sqlalchemy import insert, delete, select
 from sqlalchemy.orm import Session
 from sqlalchemy.exc import SQLAlchemyError
 from backend.app.db.models.recommendations import (
@@ -242,20 +242,26 @@ def chunked(iterable, size=100):
         yield iterable[i:i + size]
 
 
+# def fast_delete_recommendations(session, chunk):
+#     placeholders = ",".join(["(:p{})".format(i) for i in range(len(chunk))])
+#     params = {f"p{i}": chunk[i] for i in range(len(chunk))}
+#
+#     sql = text(f"""
+#         WITH ids(property_id) AS (
+#             VALUES {placeholders}
+#         )
+#         DELETE FROM recommendation r
+#         USING ids
+#         WHERE r.property_id = ids.property_id;
+#     """)
+#
+#     session.execute(sql, params, execution_options={"synchronize_session": False})
+
 def fast_delete_recommendations(session, chunk):
-    placeholders = ",".join(["(:p{})".format(i) for i in range(len(chunk))])
+    session.execute(
-    params = {f"p{i}": chunk[i] for i in range(len(chunk))}
+        delete(Recommendation)
-
+        .where(Recommendation.property_id.in_(chunk))
-    sql = text(f"""
+    )
-        WITH ids(property_id) AS (
-            VALUES {placeholders}
-        )
-        DELETE FROM recommendation r
-        USING ids
-        WHERE r.property_id = ids.property_id;
-    """)
-
-    session.execute(sql, params, execution_options={"synchronize_session": False})
 
 
 def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
@@ -362,11 +368,19 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
     # --------------------------
     # Recommendations (fast delete)
     # --------------------------
-    rec_chunks = list(chunked(property_ids, batch_size))
+    # rec_chunks = list(chunked(property_ids, batch_size * 5))  # larger chunks for fast delete
+    # total = len(rec_chunks)
+    # for i, chunk in enumerate(rec_chunks, start=1):
+    #     print_progress("Deleting Recommendations", i, total)
+    #     fast_delete_recommendations(session, chunk)
+    rec_chunks = list(chunked(recommendation_ids, batch_size))
     total = len(rec_chunks)
     for i, chunk in enumerate(rec_chunks, start=1):
         print_progress("Deleting Recommendations", i, total)
-        fast_delete_recommendations(session, chunk)
+        session.execute(
+            delete(Recommendation)
+            .where(Recommendation.id.in_(chunk))
+        )
 
     # --------------------------
     # Inspections
@@ -412,3 +426,114 @@ def clear_portfolio(session: Session, portfolio_id: int, batch_size=100):
 
     session.commit()
     print("Portfolio cleared.")
+
+
+def clear_portfolio_in_batches(
+    session: Session,
+    portfolio_id: int,
+    property_batch_size: int = 10
+):
+    # Fetch all property IDs once
+    property_ids = [
+        pid for (pid,) in
+        session.query(PropertyModel.id)
+        .filter(PropertyModel.portfolio_id == portfolio_id)
+        .all()
+    ]
+
+    def delete_for_property_batch(prop_ids):
+        # ----------------------------
+        # Recommendations → PlanRecommendations
+        # ----------------------------
+        rec_subq = (
+            select(Recommendation.id)
+            .where(Recommendation.property_id.in_(prop_ids))
+        )
+
+        session.execute(
+            delete(PlanRecommendations)
+            .where(PlanRecommendations.recommendation_id.in_(rec_subq))
+        )
+
+        session.execute(
+            delete(RecommendationMaterials)
+            .where(RecommendationMaterials.recommendation_id.in_(rec_subq))
+        )
+
+        session.execute(
+            delete(Recommendation)
+            .where(Recommendation.property_id.in_(prop_ids))
+        )
+
+        # ----------------------------
+        # Inspections
+        # ----------------------------
+        session.execute(
+            delete(InspectionModel)
+            .where(InspectionModel.property_id.in_(prop_ids))
+        )
+
+        # ----------------------------
+        # Plans (scoped to these properties)
+        # ----------------------------
+        plan_subq = (
+            select(Plan.id)
+            .where(Plan.property_id.in_(prop_ids))
+        )
+
+        session.execute(
+            delete(PlanRecommendations)
+            .where(PlanRecommendations.plan_id.in_(plan_subq))
+        )
+
+        session.execute(
+            delete(FundingPackageMeasures)
+            .where(
+                FundingPackageMeasures.funding_package_id.in_(
+                    select(FundingPackage.id)
+                    .where(FundingPackage.plan_id.in_(plan_subq))
+                )
+            )
+        )
+
+        session.execute(
+            delete(FundingPackage)
+            .where(FundingPackage.plan_id.in_(plan_subq))
+        )
+
+        session.execute(
+            delete(Plan)
+            .where(Plan.id.in_(plan_subq))
+        )
+
+        # ----------------------------
+        # Property-scoped auxiliary tables
+        # ----------------------------
+        session.execute(
+            delete(PropertyDetailsEpcModel)
+            .where(PropertyDetailsEpcModel.property_id.in_(prop_ids))
+        )
+
+        session.execute(
+            delete(PropertyTargetsModel)
+            .where(PropertyTargetsModel.property_id.in_(prop_ids))
+        )
+
+        # ----------------------------
+        # Properties (last)
+        # ----------------------------
+        session.execute(
+            delete(PropertyModel)
+            .where(PropertyModel.id.in_(prop_ids))
+        )
+
+    # -------- BATCH DELETE LOOP --------
+    property_chunks = list(chunked(property_ids, property_batch_size))
+    total_batches = len(property_chunks)
+
+    for i, prop_ids in enumerate(property_chunks, start=1):
+        print(f"Deleting batch {i}/{total_batches} ({len(prop_ids)} properties)")
+        delete_for_property_batch(prop_ids)
+        session.commit()
+
+    print("Portfolio cleared in batches.")

backend/engine/engine.py

@@ -662,7 +662,9 @@ async def model_engine(body: PlanTriggerRequest):
                 address1 = config.get("domna_address_1", None)
 
             address1 = str(int(address1)) if isinstance(address1, float) else str(address1)
-            full_address = config.get("domna_full_address") if body.file_format == "domna_asset_list" else None
+            full_address = config.get("domna_full_address", "") if body.file_format == "domna_asset_list" else None
+            if not isinstance(full_address, str):  # Catch for when the full address is nan
+                full_address = None
             heating_system = parse_heating_system(config)
 
             associated_uprns = []

backend/ml_models/AnnualBillSavings.py

@@ -290,6 +290,14 @@ class AnnualBillSavings:
             # The solar thermal covers a % of the heating kwh, so we need to adjust the cost
             return (kwh / cop) * assumptions.SOLAR_CONSUMPTION_PROPORTION * cls.ELECTRICITY_PRICE_CAP
 
+        if fuel in ['Oil + Solar Thermal']:
+            # The solar thermal covers a % of the heating kwh, so we need to adjust the cost
+            price_data = cls.FUEL_DATA[cls.FUEL_DATA["Fuel"] == "Kerosene"].squeeze()
+            cost_per_kwh = cls.cost_per_kwh(
+                price_data["Price (p)"], price_data["Energy Content, Net Calorific value (kWh/unit)"]
+            )
+            return (kwh / cop) * cost_per_kwh * assumptions.SOLAR_CONSUMPTION_PROPORTION
+
         if fuel == "LPG + Solar Thermal":
             # The solar thermal covers a % of the heating kwh, so we need to adjust the cost
             price_data = cls.FUEL_DATA[cls.FUEL_DATA["Fuel"] == "LPG"].squeeze()

backend/tests/test_integration.py

@@ -82,6 +82,12 @@ costs_by_floor_area = epc_data[
     ][["TOTAL_FLOOR_AREA", "CURRENT_ENERGY_EFFICIENCY", "LIGHTING_COST_CURRENT", "HEATING_COST_CURRENT",
        "HOT_WATER_COST_CURRENT"]].copy()
 
+epc_data = epc_data[
+    (epc_data["MAINHEAT_DESCRIPTION"].str.contains("SAP05:") == False) &
+    (~epc_data["LIGHTING_COST_CURRENT"].isin([None, ""])) &
+    (~pd.isnull(epc_data["LIGHTING_COST_CURRENT"]))
+    ]
+
 costs_by_floor_area.columns = [c.lower().replace("_", "-") for c in costs_by_floor_area.columns]
 for c in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]:
     costs_by_floor_area[c + "_scaled"] = costs_by_floor_area[c] / costs_by_floor_area["total-floor-area"]
@@ -92,8 +98,8 @@ costs_by_floor_area = costs_by_floor_area.groupby("current-energy-efficiency")[
 
 epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
 
-sample_epc_data = epc_data[pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2015-01-01"].drop_duplicates("UPRN").sample(
+sample_epc_data = epc_data[pd.to_datetime(epc_data["LODGEMENT_DATE"]) >= "2008-01-01"].drop_duplicates("UPRN").sample(
-    10000).reset_index(drop=True)
+    50000).reset_index(drop=True)
 
 # TODO: In Property find_energy_sources, sort out biomass community heating - what fuel type
 # TODO: We might be able to remove find_energy_sources entirely and remove estimate_electrical_consumption. It's used
@@ -163,6 +169,8 @@ mocked_kwh_predictions["heating_kwh_predictions"] = pd.DataFrame(mocked_kwh_pred
 mocked_kwh_predictions["hotwater_kwh_predictions"] = pd.DataFrame(mocked_kwh_predictions["hotwater_kwh_predictions"])
 
 # TODO: We might want to implement this generally, via an ETL process
+for x in cleaned["mainheat-description"]:
+    x["has_wood_chips"] = False
 for p in input_properties:
     for col in ["lighting-cost-current", "heating-cost-current", "hot-water-cost-current"]:
         if pd.isnull(p.data[col]):
@@ -313,6 +321,10 @@ for p in tqdm(input_properties):
     if not recommendations.get(p.id):
         continue
 
+    # Temp allow to skip
+    if not isinstance(recommendations.get(p.id)[0], list):
+        continue
+
     # we need to double unlist because we have a list of lists
     property_measure_types = {rec["type"] for recs in recommendations[p.id] for rec in recs}
     property_required_measures = [m for m in recommendations[p.id] if m[0]["type"] in body.required_measures]
@@ -336,32 +348,32 @@ for p in tqdm(input_properties):
     )
     gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain, eco_packages=eco_packages)
 
-    funding = Funding(
+    # funding = Funding(
-        tenure=body.housing_type,
+    #     tenure=body.housing_type,
-        project_scores_matrix=project_scores_matrix,
+    #     project_scores_matrix=project_scores_matrix,
-        partial_project_scores_matrix=partial_project_scores_matrix,
+    #     partial_project_scores_matrix=partial_project_scores_matrix,
-        whlg_eligible_postcodes=whlg_eligible_postcodes,
+    #     whlg_eligible_postcodes=whlg_eligible_postcodes,
-        eco4_social_cavity_abs_rate=13,
+    #     eco4_social_cavity_abs_rate=13,
-        eco4_social_solid_abs_rate=17,
+    #     eco4_social_solid_abs_rate=17,
-        eco4_private_cavity_abs_rate=13,
+    #     eco4_private_cavity_abs_rate=13,
-        eco4_private_solid_abs_rate=17,
+    #     eco4_private_solid_abs_rate=17,
-        gbis_social_cavity_abs_rate=21,
+    #     gbis_social_cavity_abs_rate=21,
-        gbis_social_solid_abs_rate=25,
+    #     gbis_social_solid_abs_rate=25,
-        gbis_private_cavity_abs_rate=21,
+    #     gbis_private_cavity_abs_rate=21,
-        gbis_private_solid_abs_rate=28,
+    #     gbis_private_solid_abs_rate=28,
-    )
+    # )
-
+    #
-    li_thickness = convert_thickness_to_numeric(
+    # li_thickness = convert_thickness_to_numeric(
-        p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"]
+    #     p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"]
-    )
+    # )
-    current_wall_u_value = p.walls["thermal_transmittance"]
+    # current_wall_u_value = p.walls["thermal_transmittance"]
-    if current_wall_u_value is None:
+    # if current_wall_u_value is None:
-        current_wall_u_value = get_wall_u_value(
+    #     current_wall_u_value = get_wall_u_value(
-            clean_description=p.walls["clean_description"],
+    #         clean_description=p.walls["clean_description"],
-            age_band=p.age_band,
+    #         age_band=p.age_band,
-            is_granite_or_whinstone=p.walls["is_granite_or_whinstone"],
+    #         is_granite_or_whinstone=p.walls["is_granite_or_whinstone"],
-            is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"],
+    #         is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"],
-        )
+    #     )
 
     # We insert the innovation uplift
     measures_to_optimise_with_uplift = deepcopy(measures_to_optimise)
@@ -369,35 +381,39 @@ for p in tqdm(input_properties):
     # TODO: Turn this into a function and store the innovaiton uplift
     for group in measures_to_optimise_with_uplift:
         for r in group:
-
+            (r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"],
-            if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating",
+             r["uplift_project_score"]) = (
-                             "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]:
+                0, 0, 0, 0
-                (
-                    r["partial_project_score"],
-                    r["partial_project_funding"],
-                    r["innovation_uplift"],
-                    r["uplift_project_score"],
-                ) = (
-                    0, 0, 0, 0
-                )
-                continue
-
-            (
-                r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"],
-                r["uplift_project_score"]
-            ) = funding.get_innovation_uplift(
-                measure=r,
-                starting_sap=int(p.data["current-energy-efficiency"]),
-                floor_area=p.floor_area,
-                is_cavity=p.walls["is_cavity_wall"],
-                current_wall_uvalue=current_wall_u_value,
-                is_partial="partial" in p.walls["clean_description"].lower(),
-                existing_li_thickness=li_thickness,
-                mainheating=p.main_heating,
-                main_fuel=p.main_fuel,
-                mainheat_energy_eff=p.data["mainheat-energy-eff"],
             )
 
+            # if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating",
+            #                  "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]:
+            #     (
+            #         r["partial_project_score"],
+            #         r["partial_project_funding"],
+            #         r["innovation_uplift"],
+            #         r["uplift_project_score"],
+            #     ) = (
+            #         0, 0, 0, 0
+            #     )
+            #     continue
+            #
+            # (
+            #     r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"],
+            #     r["uplift_project_score"]
+            # ) = funding.get_innovation_uplift(
+            #     measure=r,
+            #     starting_sap=int(p.data["current-energy-efficiency"]),
+            #     floor_area=p.floor_area,
+            #     is_cavity=p.walls["is_cavity_wall"],
+            #     current_wall_uvalue=current_wall_u_value,
+            #     is_partial="partial" in p.walls["clean_description"].lower(),
+            #     existing_li_thickness=li_thickness,
+            #     mainheating=p.main_heating,
+            #     main_fuel=p.main_fuel,
+            #     mainheat_energy_eff=p.data["mainheat-energy-eff"],
+            # )
+
             if r["already_installed"]:
                 # if already installed, we zero out the uplift and funding
                 (r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"],
@@ -411,7 +427,7 @@ for p in tqdm(input_properties):
     )
 
     # When the goal is Increasing EPC, we can run the funding optimiser
-    if body.goal == "Increasing EPC":
+    if body.goal == "Switch off":
 
         solutions = optimise_with_funding_paths(
             p=p,
@@ -481,37 +497,43 @@ for p in tqdm(input_properties):
             ROOF_INSULATION_MEASURES
         )
 
-        funding.check_funding(
+        # funding.check_funding(
-            measures=solution,
+        #     measures=solution,
-            starting_sap=int(p.data["current-energy-efficiency"]),
+        #     starting_sap=int(p.data["current-energy-efficiency"]),
-            ending_sap=int(p.data["current-energy-efficiency"]) + sum([x["gain"] for x in solution]),
+        #     ending_sap=int(p.data["current-energy-efficiency"]) + sum([x["gain"] for x in solution]),
-            floor_area=p.floor_area,
+        #     floor_area=p.floor_area,
-            mainheat_description=p.main_heating["clean_description"],
+        #     mainheat_description=p.main_heating["clean_description"],
-            heating_control_description=p.main_heating_controls["clean_description"],
+        #     heating_control_description=p.main_heating_controls["clean_description"],
-            is_cavity=p.walls["is_cavity_wall"],
+        #     is_cavity=p.walls["is_cavity_wall"],
-            current_wall_uvalue=current_wall_u_value,
+        #     current_wall_uvalue=current_wall_u_value,
-            is_partial="partial" in p.walls["clean_description"].lower(),
+        #     is_partial="partial" in p.walls["clean_description"].lower(),
-            existing_li_thickness=li_thickness,
+        #     existing_li_thickness=li_thickness,
-            mainheating=p.main_heating,
+        #     mainheating=p.main_heating,
-            main_fuel=p.main_fuel,
+        #     main_fuel=p.main_fuel,
-            mainheat_energy_eff=p.data["mainheat-energy-eff"],
+        #     mainheat_energy_eff=p.data["mainheat-energy-eff"],
-            has_wall_insulation_recommendation=has_wall_insulation_recommendation,
+        #     has_wall_insulation_recommendation=has_wall_insulation_recommendation,
-            has_roof_insulation_recommendation=has_roof_insulation_recommendation,
+        #     has_roof_insulation_recommendation=has_roof_insulation_recommendation,
-        )
+        # )
 
         # Determine the scheme
         scheme = "none"
-        if funding.eco4_eligible:
+        # if funding.eco4_eligible:
-            scheme = "eco4"
+        #     scheme = "eco4"
-        if scheme == "none" and funding.gbis_eligible:
+        # if scheme == "none" and funding.gbis_eligible:
-            scheme = "gbis"
+        #     scheme = "gbis"
 
-        funded_measures = solution if scheme in ["gbis", "eco4"] else []
+        funded_measures = []
-        project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs
+        # funded_measures = solution if scheme in ["gbis", "eco4"] else []
-        total_uplift = funding.eco4_uplift
+        # project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs
-        full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs
+        project_funding = 0
-        partial_project_score = funding.partial_project_abs
+        # total_uplift = funding.eco4_uplift
-        uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift
+        total_uplift = 0
+        # full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs
+        full_project_score = 0
+        # partial_project_score = funding.partial_project_abs
+        partial_project_score = 0
+        # uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift
+        uplift_project_score = 0
 
     selected = {r["id"] for r in solution}
 

etl/customers/lincs_rural/get_missed.py

@@ -0,0 +1,47 @@
+# After going back to Lincs rural, they gave us some additional data that we can use to try to fetch missed UPRNs again
+import pandas as pd
+
+# missed = pd.read_excel(
+#     "/Users/khalimconn-kowlessar/Downloads/lincs_rural_missed_nov_2025.xlsx",
+#     sheet_name="Missed Properties"
+# )
+# missed = missed[~pd.isnull(missed["rrn"])]
+
+prepared = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Downloads/lincs_rural_standardised_ara_nov_2025.xlsx",
+    sheet_name="Standardised Asset List"
+)
+
+updated_data = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Downloads/MASTER LIST EPCS UPDATED November 2025 Domna Homes - Copy.xlsx",
+    sheet_name="PROPERTY EPC RATINGS"
+)
+updated_data = updated_data[~pd.isnull(updated_data["Property Ref."])]
+
+missed = updated_data[~updated_data["Property Ref."].isin(prepared["landlord_property_id"].values.tolist())].copy()
+# missed.to_csv("/Users/khalimconn-kowlessar/Downloads/lincs_rural_missed_uprn.csv")
+# We'll grab the UPRNs manually and then pull them in, and prepare for ARA
+
+missing_uprns = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/lincs_rural_missed_uprn.csv")
+
+missing_uprns["landlord_property_id"] = missing_uprns["Property Ref."].copy()
+missing_uprns["domna_property_id"] = missing_uprns["Property Ref."].copy()
+missing_uprns["domna_address_1"] = missing_uprns['Unnamed: 1'].str.split(",").str[0].str.strip()
+missing_uprns["postcode"] = missing_uprns['Unnamed: 1'].str.split(",").str[-1].str.strip()
+missing_uprns["landlord_property_type"] = "unknown"
+missing_uprns["landlord_built_form"] = "unknown"
+missing_uprns["domna_full_address"] = missing_uprns['Unnamed: 1'].copy()
+
+missed_standardised_for_ara = missing_uprns[
+    ['landlord_property_id', 'domna_address_1', 'landlord_property_type', 'landlord_built_form', 'postcode',
+     'domna_property_id', 'UPRN']
+].rename(
+    columns={"UPRN": "epc_os_uprn"}
+)
+
+# Store
+missed_standardised_for_ara.to_excel(
+    "/Users/khalimconn-kowlessar/Downloads/lincs_rural_missed_standardised_ara_nov_2025.xlsx",
+    index=False,
+    sheet_name="Standardised Asset List"
+)

etl/customers/peabody/Nov 2025 Consulting Project/b_data_cleanse.py

@@ -0,0 +1,147 @@
+"""
+We have found, within the Peabody data, a large volume of properties with missing and incorrects
+UPRNS and incorrect address data. We want to flag these records and also find missings where we can
+
+We also have duplicate UPRNS that should be flagged
+"""
+import json
+import time
+import os
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+from dotenv import load_dotenv
+from asset_list.utils import get_data_for_property
+from utils.logger import setup_logger
+from utils.s3 import read_io_from_s3, save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet
+
+logger = setup_logger()
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+sustainability_data = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/2025_11_11 - Peabody "
+    "- Data Extracts for Domna.xlsx",
+    sheet_name="Sustainability"
+)
+property_data = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/2025_11_11 - Peabody "
+    "- Data Extracts for Domna.xlsx",
+    sheet_name="Properties"
+)
+
+missing_uprns = sustainability_data[pd.isnull(sustainability_data['UPRN'])].copy()
+
+# Any non-numeric UPRNS or leading with 0s are invalid
+non_numeric_uprns = sustainability_data[
+    ~sustainability_data['UPRN'].astype(str).str.match(r'^[1-9][0-9]*$') & ~pd.isnull(sustainability_data['UPRN'])
+    ].copy()
+# 70 properties
+leading_zero_uprns = sustainability_data[
+    sustainability_data['UPRN'].astype(str).str.startswith('0')
+].copy()
+
+# Flag duplicates
+duplicate_uprns = sustainability_data[
+    sustainability_data.duplicated(subset=['UPRN'], keep=False) & ~pd.isnull(sustainability_data['UPRN'])
+    ].copy()
+
+# Store this data
+# missing_uprns.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting
+# Project/data_validation/missing_uprns.csv", index=False)
+# non_numeric_uprns.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting
+# Project/data_validation/non_numeric_uprns.csv", index=False)
+# leading_zero_uprns.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting
+# Project/data_validation/leading_zero_uprns.csv", index=False)
+# duplicate_uprns.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting
+# Project/data_validation/duplicate_uprns.csv", index=False)
+
+# Take everything remaining
+data_needing_validation = sustainability_data[
+    ~sustainability_data["Org Ref"].isin(
+        missing_uprns["Org Ref"].values.tolist() + non_numeric_uprns["Org Ref"].values.tolist() +
+        leading_zero_uprns["Org Ref"].values.tolist() + duplicate_uprns["Org Ref"].values.tolist()
+    )
+].copy()
+
+# TODO: We should build a SAL for UPRNS that are missing, invalid or duplicated
+
+# We check UPRN validity against our OS data
+uprn_filenames = read_dataframe_from_s3_parquet(
+    bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
+)
+
+# We're going to:
+# 1) Grab a filename
+# 2) Read it in
+# 3) Check which UPRNS from our data are in that file
+# 4) Keep a record of which UPRNS were found where
+
+for uprn_file in tqdm(uprn_filenames['filenames'].values, total=len(uprn_filenames)):
+    spatial_data = read_dataframe_from_s3_parquet(
+        bucket_name="retrofit-data-dev", file_key=f"spatial/{uprn_file}"
+    )
+
+    uprns_in_file = data_needing_validation[
+        data_needing_validation['UPRN'].astype('Int64').isin(spatial_data['UPRN'].astype('Int64').values)
+    ].copy()
+
+    print("Found {} UPRNS in file {}".format(len(uprns_in_file), uprn_file))
+    if len(uprns_in_file) > 0:
+        # Store the found UPRNS in the validation cache
+        data_to_store = uprns_in_file[["Org Ref", "UPRN"]].copy()
+        data_to_store["Source File"] = uprn_file
+        # Store
+        data_to_store.to_csv(
+            "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+            f"Project/data_validation/validation_cache/{uprn_file.split('.parquet')[0]}_found_uprns.csv",
+            index=False
+        )
+
+# Get all of the files:
+storage_locations = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+                     "Project/data_validation/validation_cache")
+# List contents
+folder_contents = os.listdir(storage_locations)
+# Grab files and concatenate
+all_found_uprns = []
+for file in folder_contents:
+    if file.endswith("_found_uprns.csv"):
+        df = pd.read_csv(os.path.join(storage_locations, file))
+        all_found_uprns.append(df)
+
+all_found_uprns = pd.concat(all_found_uprns)
+
+# We now flag any UPRNS that were not found in any of the OS datasets
+os_missed_uprns = data_needing_validation[
+    ~data_needing_validation['Org Ref'].isin(all_found_uprns['Org Ref'].values.tolist())
+].copy()
+
+# store
+os_missed_uprns.to_csv(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+    "Project/data_validation/os_missed_uprns.csv",
+    index=False
+)
+
+# Now build a larger table for standardisation
+to_standardised = pd.concat(
+    [missing_uprns, non_numeric_uprns, leading_zero_uprns, duplicate_uprns, os_missed_uprns]
+)
+
+to_standardised.to_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+    "Project/data_validation/to_standardise_uprns.xlsx",
+    index=False)
+
+# We prepare a finalised dataset to work with, that excludes all problematic properties and leaves us with
+# properties for which we have the data we need
+
+finalised_data = sustainability_data[
+    ~sustainability_data["Org Ref"].isin(
+        to_standardised["Org Ref"].values.tolist()
+    )
+].copy()
+
+# Prepare with the column formats we need, as analogous to a_data_prep where we defined an initial working sample

etl/customers/peabody/Nov 2025 Consulting Project/c_finalised_modelling_data.py

@@ -0,0 +1,114 @@
+import pandas as pd
+
+# import pandas as pd
+#
+# sal = pd.read_excel(
+#     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+#     "Project/data_validation/to_standardise_uprns - Standardised.xlsx",
+#     sheet_name="Standardised Asset List"
+# )
+#
+# # Quick breadown of missingness
+# missing = sal[
+#     pd.isnull(sal["estimated"]) | (sal["estimated"] == True) | pd.isnull(sal["epc_os_uprn"])
+#     ]
+#
+# fetched = sal[(sal["estimated"] == False) | ~pd.isnull(sal["epc_os_uprn"])].copy()
+# fetched = fetched[
+#     ["landlord_property_id", "domna_address_1", "domna_postcode", "domna_full_address", "epc_address1",
+#      "epc_postcode", "epc_address", "landlord_property_type", "epc_property_type"]
+# ]
+#
+# known_issues = [
+#
+# ]
+#
+# # Missed postcodes
+# missed_postcode_agg = missing.groupby("domna_postcode").size().reset_index(name="count")
+# missed_postcode_agg = missed_postcode_agg.sort_values("count", ascending=False)
+#
+# multi_missed_postcode = missed_postcode_agg[missed_postcode_agg["count"] > 1]
+
+### Prepare
+sustainability_data = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/2025_11_11 - Peabody "
+    "- Data Extracts for Domna.xlsx",
+    sheet_name="Sustainability"
+)
+
+# Data we want to remove:
+missing_uprns = sustainability_data[pd.isnull(sustainability_data['UPRN'])].copy()
+
+# Any non-numeric UPRNS or leading with 0s are invalid
+non_numeric_uprns = sustainability_data[
+    ~sustainability_data['UPRN'].astype(str).str.match(r'^[1-9][0-9]*$') & ~pd.isnull(sustainability_data['UPRN'])
+    ].copy()
+# 70 properties
+leading_zero_uprns = sustainability_data[
+    sustainability_data['UPRN'].astype(str).str.startswith('0')
+].copy()
+
+# Flag duplicates
+duplicate_uprns = sustainability_data[
+    sustainability_data.duplicated(subset=['UPRN'], keep=False) & ~pd.isnull(sustainability_data['UPRN'])
+    ].copy()
+
+# Read in the UPRNs that were not valid based on the OS data
+os_missed_uprns = pd.read_csv(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+    "Project/data_validation/os_missed_uprns.csv",
+)
+
+modelling_data = sustainability_data[
+    ~sustainability_data["Org Ref"].isin(
+        missing_uprns["Org Ref"].unique().tolist() + non_numeric_uprns["Org Ref"].unique().tolist() +
+        leading_zero_uprns["Org Ref"].unique().tolist() + duplicate_uprns["Org Ref"].unique().tolist() +
+        os_missed_uprns["Org Ref"].unique().tolist()
+    )
+].copy()
+
+# Need to prepare for upload
+# Variables:
+
+
+modelling_data["landlord_property_id"] = sustainability_data["Org Ref"].copy()
+modelling_data["domna_property_id"] = sustainability_data["Org Ref"].copy()
+
+modelling_data = modelling_data.rename(
+    {
+        "Address 1": "domna_address_1",
+        "Postcode": "postcode",
+        "Type": "landlord_property_type",
+        "Attachment": "landlord_built_form",
+        "Heating": "landlord_heating_system",
+        "UPRN": "epc_os_uprn"
+    }
+)
+
+modelling_data = modelling_data[
+    [
+        "domna_address_1", "Address 2", "Address 3", "postcode", "landlord_property_type",
+        "landlord_built_form", "landlord_heating_system", "epc_os_uprn", "Total Floor Area (m2)",
+        "domna_property_id", "domna_full_address"
+    ]
+]
+
+modelling_data["landlord_built_form"] = modelling_data["landlord_built_form"].map(
+    {
+        "MidTerrace": "Mid-Terrace",
+        "EndTerrace": "End-Terrace",
+        "SemiDetached": "Semi-Detached",
+        "Detached": "Detached",
+        "EnclosedEndTerrace": "Enclosed End-Terrace",
+        "EnclosedMidTerrace": "Enclosed Mid-Terrace",
+    }
+)
+
+
+def make_full_address(x):
+    to_join = [x['domna_address_1'], x['Address 2'], x['Address 3']]
+    to_join = [x for x in to_join if not pd.isnull(x) and x != '']
+    return ", ".join(to_join)
+
+
+modelling_data["domna_full_address"] = modelling_data.apply(lambda x: make_full_address(x), axis=1)

etl/customers/peabody/Nov 2025 Consulting Project/data_cleanse.py

@@ -1,6 +0,0 @@
-"""
-We have found, within the Peabody data, a large volume of properties with missing and incorrects
-UPRNS and incorrect address data. We want to flag these records and also find missings where we can
-
-We also have duplicate UPRNS that should be flagged
-"""

etl/epc/Dataset.py

@@ -844,7 +844,7 @@ class TrainingDataset(BaseDataset):
 
         # Make sure they are all efficiency columns
         if any(~missings.index.str.contains("energy_eff")):
-            raise ValueError("Non efficiency columns are missing")
+            raise ValueError(f"Non efficiency columns are missing {missings.index}")
 
         for m in missings.index:
             self.df[m] = self.df[m].fillna("NO_RATING")

etl/webscrape/Zoopla.py

@@ -15,25 +15,10 @@ os.makedirs(CACHE_DIR, exist_ok=True)
 
 
 def random_delay():
-    """Pause randomly between requests (0.5–2 s)."""
     time.sleep(random.uniform(0.5, 2))
 
 
-def extract_feature(soup, icon_id):
-    tag = soup.find("use", href=f"#{icon_id}")
-    if tag:
-        parent = tag.find_parent("div", class_="_1pbf8i53")
-        if parent:
-            text = parent.get_text(strip=True)
-            return text
-    return None
-
-
 def extract_embedded_json(text):
-    """
-    Extract embedded property JSON containing attributes, energy, estimates, and sales history.
-    """
-    # Try to grab everything after "attributes"
     match = re.search(
         r'"attributes"\s*:\s*\{.*?\}\s*,.*?"historicSales".*?\]',
         text,
@@ -48,13 +33,16 @@ def extract_embedded_json(text):
         except json.JSONDecodeError:
             pass
 
-    # fallback for independent keys
     result = {}
     for key in [
         "attributes", "energy", "rentEstimate",
         "saleEstimate", "saleHistory", "historicSales"
     ]:
-        key_match = re.search(rf'"{key}"\s*:\s*(\{{.*?\}}|\[.*?\])', text, re.DOTALL)
+        key_match = re.search(
+            rf'"{key}"\s*:\s*(\{{.*?\}}|\[.*?\])',
+            text,
+            re.DOTALL
+        )
         if key_match:
             try:
                 result[key] = json.loads(key_match.group(1))
@@ -64,28 +52,23 @@ def extract_embedded_json(text):
 
 
 def scrape_all_estimates(session, url):
-    """Scrape valuation estimates for one Zoopla property URL."""
     resp = session.get(url, impersonate=random.choice(ENGINES))
     html = resp.text
-    page_source = BeautifulSoup(resp.text, "html.parser")
+    soup = BeautifulSoup(html, "html.parser")
-    estimates = page_source.find_all("div", {"data-testid": "sale-estimate"})
+    estimates = soup.find_all("div", {"data-testid": "sale-estimate"})
-
     data = extract_embedded_json(html)
 
-    is_blocked = len(estimates) == 0
-
     return {
         "estimates": estimates,
-        "is_blocked": is_blocked,
+        "is_blocked": len(estimates) == 0,
         "response_html": html,
-        "attributes": data.get("attributes"),
+        "attributes": data.get("attributes", {}),
-        "rent": data.get("rentEstimate"),
+        "rentEstimate": data.get("rentEstimate", {}),
-        "historicSales": data.get("historicSales"),
+        "historicSales": data.get("historicSales", []),
     }
 
 
 def extract_estimates(estimates):
-    """Extract low, mid, and high estimates from parsed HTML."""
     est = estimates[0]
     low = est.find("span", {"data-testid": "low-estimate-blurred"}).text
     mid = est.find("p", {"data-testid": "estimate-blurred"}).text
@@ -94,110 +77,123 @@ def extract_estimates(estimates):
 
 
 def cache_path_for_url(url):
-    """Return a deterministic local cache path for a URL."""
     uprn = url.split("/")[-2]
     return os.path.join(CACHE_DIR, f"{uprn}.html")
 
 
+def parse_cached_html(url, html):
+    soup = BeautifulSoup(html, "html.parser")
+    estimates = soup.find_all("div", {"data-testid": "sale-estimate"})
+    data = extract_embedded_json(html)
+    history = data.get("historicSales") or [{}]
+
+    if not estimates:
+        return None
+
+    low, mid, high = extract_estimates(estimates)
+
+    return {
+        "URL": url,
+        "Low Estimate": low,
+        "Middle Estimate": mid,
+        "High Estimate": high,
+        **data.get("attributes", {}),
+        **data.get("rentEstimate", {}),
+        **history[0],
+    }
+
+
 def parallel_task(url):
-    """Main worker function executed in each process."""
     cache_path = cache_path_for_url(url)
 
-    # Use cached file if it exists
     if os.path.exists(cache_path):
-        html = open(cache_path, "r").read()
+        with open(cache_path, "r", encoding="utf-8") as f:
-        page_source = BeautifulSoup(html, "html.parser")
+            html = f.read()
-        estimates = page_source.find_all("div", {"data-testid": "sale-estimate"})
+        cached = parse_cached_html(url, html)
-        data = extract_embedded_json(html)
+        if cached:
-        history_sales = data.get("historicSales", [{}])
+            return cached
-        if len(history_sales) == 0:
-            history_sales = [{}]
 
-        if estimates:
-            low, mid, high = extract_estimates(estimates)
-            return {
-                "URL": url, "Low Estimate": low, "Middle Estimate": mid, "High Estimate": high,
-                **data.get("attributes", {}), **data.get("rentEstimate", {}),
-                **history_sales[0]
-            }
-
-    # Otherwise scrape live
     with StealthSession() as session:
-        attempts = 0
+        for attempt in range(5):
-        while attempts < 5:
             output = scrape_all_estimates(session, url)
+
             if not output["is_blocked"] and output["estimates"]:
-                open(cache_path, "w").write(output["html"])
+                html = output.get("response_html")
+                if html:
+                    with open(cache_path, "w", encoding="utf-8") as f:
+                        f.write(html)
+
+                history = output.get("historicSales") or [{}]
                 low, mid, high = extract_estimates(output["estimates"])
-                history_sales = output.get("historicSales", [{}])
+
-                if len(history_sales) == 0:
-                    history_sales = [{}]
                 return {
-                    "URL": url, "Low Estimate": low, "Middle Estimate": mid, "High Estimate": high,
+                    "URL": url,
+                    "Low Estimate": low,
+                    "Middle Estimate": mid,
+                    "High Estimate": high,
                     **output.get("attributes", {}),
-                    **output.get("rent", {}),
+                    **output.get("rentEstimate", {}),
-                    **history_sales[0]
+                    **history[0],
                 }
-            attempts += 1
+
-            print(f"[Attempt {attempts}] Blocked or empty for {url}")
             random_delay()
 
-        # If still blocked, return placeholders
+    return {
-        return {"URL": url, "Low Estimate": None, "Middle Estimate": None, "High Estimate": None}
+        "URL": url,
+        "Low Estimate": None,
+        "Middle Estimate": None,
+        "High Estimate": None,
+    }
 
 
 def parse_price(p):
-    if p is None:
+    if not p:
         return None
 
     p = p.replace("£", "").strip().lower()
-    if not p:
-        return None
     if p.endswith("k"):
         return float(p[:-1]) * 1_000
-    elif p.endswith("m"):
+    if p.endswith("m"):
         return float(p[:-1]) * 1_000_000
-    else:
+
-        try:
+    try:
-            return float(p.replace(",", ""))
+        return float(p.replace(",", ""))
-        except ValueError:
+    except ValueError:
-            return None
+        return None
 
 
 if __name__ == "__main__":
-    # Load portfolio
     asset_list = pd.read_excel(
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/October 2025 AL portfolio/22.10 AL Portfolio - "
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
-        "Standardised - partial UPRN fill.xlsx",
+        "Project/modelling_sample.xlsx",
         sheet_name="Standardised Asset List"
     )
+
     asset_list = asset_list[~pd.isnull(asset_list["epc_os_uprn"])]
+    asset_list = asset_list.drop_duplicates("epc_os_uprn")
     asset_list["epc_os_uprn"] = asset_list["epc_os_uprn"].astype(int).astype(str)
+
     uprns = asset_list["epc_os_uprn"].tolist()
     urls = [f"https://www.zoopla.co.uk/property/uprn/{uprn}/" for uprn in uprns]
 
-    # Limit concurrency to avoid blocks
+    with Pool(processes=2) as pool:
-    with Pool(processes=2) as pool:  # fewer processes = fewer fingerprints
         estimates_list = list(
             tqdm(pool.imap(parallel_task, urls), total=len(urls))
         )
 
     df = pd.DataFrame(estimates_list)
-
-    print(df.head())
-
     df["uprn"] = df["URL"].str.extract(r"uprn/(\d+)/")
     df["valuation"] = df["Middle Estimate"].apply(parse_price)
 
     df.to_csv("zoopla_estimates.csv", index=False)
 
-    # Merge with asset list
     merged = asset_list.merge(
         df[["uprn", "valuation"]],
         left_on="epc_os_uprn",
         right_on="uprn",
         how="left"
     )
+
     merged.to_excel(
         "20251029 AL Portfolio - Standardised - with valuations.xlsx",
         index=False

sfr/principal_pitch/2_export_data.py

@@ -11,8 +11,8 @@ from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcMod
 
 # PORTFOLIO_ID = 206
 # SCENARIOS = [389]
-PORTFOLIO_ID = 388
+PORTFOLIO_ID = 404
-SCENARIOS = [803]
+SCENARIOS = [829]
 
 
 def get_data(portfolio_id, scenario_ids):
@@ -121,7 +121,8 @@ recommendations_measures_pivot["total_retrofit_cost"] = recommendations_measures
 
 df = properties_df[
     [
-        "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof", "heating", "windows",
+        "landlord_property_id", "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof",
+        "heating", "windows",
         "current_epc_rating",
         "current_sap_points", "total_floor_area", "number_of_rooms",
     ]
@@ -143,7 +144,7 @@ from utils.s3 import read_csv_from_s3, read_excel_from_s3
 
 # asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath='8/206/asset_list.csv')
 asset_list = read_excel_from_s3(
-    bucket_name="retrofit-plan-inputs-dev", file_key='2/388/20251208T203603925Z/asset_list.xlsx',
+    bucket_name="retrofit-plan-inputs-dev", file_key="2/404/20251211T163200754Z/asset_list.xlsx",
     header_row=0, sheet_name="Standardised Asset List"
 )
 asset_list = pd.DataFrame(asset_list)
@@ -201,11 +202,15 @@ asset_list = asset_list.merge(
 )
 
 # For exporting
-asset_list.to_excel(
+df.to_excel(
-    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lincs Rural/EPC C -without floors proposed measures - "
-    "Project/20251209_sample_package_data.xlsx",
+    "with ID.xlsx",
     index=False
 )
+# asset_list.to_excel(
+#     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lincs Rural/epc_measures.xlsx",
+#     index=False
+# )
 
 condition_costs = pd.read_excel(
     "/Users/khalimconn-kowlessar/Documents/hestia/sfr/Spring JV/Condition costs.xlsx",