Merge pull request #681 from Hestia-Homes/main

Dev deployment - condition etl, refactor of extraction and adjustments of recommendation impact

.devcontainer/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.12-bullseye
+FROM python:3.11.10-bullseye
+
 
 ARG USER=vscode
 ARG DEBIAN_FRONTEND=noninteractive
@@ -24,12 +25,17 @@ RUN useradd -m -s /usr/bin/bash ${USER} \
  && echo "${USER} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/${USER} \
  && chmod 0440 /etc/sudoers.d/${USER}
 
-# 4) Python deps
-ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
-# Model
+# # 4) Python deps - if you want to run assest list
+# ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
 # ADD asset_list/requirements.txt requirements.txt
-# FASTAPI backend
-ADD .devcontainer/requirements.txt requirements.txt
+# RUN pip install -r requirements.txt
+
+# 
+ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
+ADD backend/engine/requirements.txt requirements1.txt
+ADD backend/app/requirements/requirements.txt requirements2.txt
+ADD .devcontainer/requirements.txt requirements3.txt
+RUN cat requirements1.txt requirements2.txt requirements3.txt > requirements.txt
 RUN pip install -r requirements.txt
 
 # 5) Workdir
@@ -37,4 +43,4 @@ WORKDIR /workspaces/model
 
 # 6) Make Python find your package
 # Add project root to PYTHONPATH for all processes
-ENV PYTHONPATH=/workspaces/model:${PYTHONPATH}
+ENV PYTHONPATH=/workspaces/model:${PYTHONPATH}

.devcontainer/devcontainer.json

@@ -27,5 +27,8 @@
         "ms-python.vscode-python-envs"
       ]
     }
+  },
+  "containerEnv": {
+    "PYTHONFLAGS": "-Xfrozen_modules=off"
   }
 }

.devcontainer/requirements.txt

@@ -14,4 +14,7 @@ openpyxl==3.1.2
 pytz
 uvicorn[standard]
 sqlmodel
-
+# Testing
+pytest==9.0.2
+pytest-cov==7.0.0
+ipykernel>=6.25,<7

.gitignore

@@ -242,6 +242,8 @@ fabric.properties
 local_data/*
 /local_data/*
 etl/epc/local_data/*
+/backend/condition/sample_data/lbwf/*
+/backend/condition/sample_data/peabody/*
 
 *.DS_Store
 infrastructure/terraform/.terraform*

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

.vscode/settings.json

@@ -9,6 +9,9 @@
             "path": "/bin/bash"
         }
     },
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.testing.pytestArgs": ["-s", "-q", "--no-cov"]
 
     // Hot reload setting that needs to be in user settings
     // "jupyter.runStartupCommands": [

backend/condition/file_type.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+class FileType(Enum):
+    LBWF = "lbwf"
+
+def detect_file_type(filepath: str) -> FileType:
+    path = filepath.lower()
+
+    if "lbwf" in path:
+        return FileType.LBWF
+    
+    raise ValueError("Unrecognised file path")

backend/condition/handler.py

@@ -0,0 +1,16 @@
+from typing import Mapping, Any
+from io import BytesIO
+
+from utils.logger import setup_logger
+from backend.condition.processor import process_file
+
+
+logger = setup_logger()
+
+def handler(event: Mapping[str, Any], context: Any) -> None:
+    # Temporary stub for PoC wiring
+    dummy_stream = BytesIO(b"")
+
+    source_key = event.get("source_key", "unknown-source")
+
+    process_file(dummy_stream, source_key)

backend/condition/local_runner.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from backend.condition.processor import process_file
+
+def main() -> None:
+    try:
+        # Works in scripts / debugger / pytest
+        ROOT_DIR = Path(__file__).resolve().parents[1]
+    except NameError:
+        # __file__ is not defined in notebooks
+        ROOT_DIR = Path.cwd()
+
+    path: Path = ROOT_DIR / "condition" / "sample_data"
+
+    lbwf_path: Path = path / "lbwf" / "LBWF - Example Asset Data September 2025.xlsx"  # TODO: get this from s3 as part of devcontainer init
+
+    with lbwf_path.open("rb") as f:
+        process_file(
+            file_stream=f,
+            source_key=lbwf_path.as_posix(),
+        )
+
+if __name__ == "__main__":
+    main()
+

backend/condition/parsing/factory.py

@@ -0,0 +1,9 @@
+from backend.condition.file_type import FileType
+from backend.condition.parsing.parser import Parser
+from backend.condition.parsing.lbwf_parser import LbwfParser
+
+def select_parser(file_type: FileType) -> Parser:
+    if file_type is FileType.LBWF:
+        return LbwfParser()
+
+    raise ValueError("Unrecognised file type, unable to instantiate Parser")

backend/condition/parsing/lbwf_parser.py

@@ -0,0 +1,180 @@
+from typing import BinaryIO, Any, Dict, Iterator, List, Tuple
+from openpyxl import Workbook, load_workbook
+from collections import defaultdict
+
+from backend.condition.parsing.parser import Parser
+from backend.condition.parsing.records.lbwf.lbwf_asset_condition import LbwfAssetCondition
+from backend.condition.parsing.records.lbwf.lbwf_house import LbwfHouse
+from backend.condition.utils.date_utils import normalise_date
+from utils.logger import setup_logger
+
+logger = setup_logger
+
+class LbwfParser(Parser):
+
+    def parse(self, file_stream: BinaryIO) -> Any:
+        wb: Workbook = load_workbook(file_stream)
+        address_to_uprn_map: Dict[str, int] = self._generate_address_to_uprn_dict(wb)
+
+        assets = self._parse_assets(wb)
+        houses = self._parse_houses(wb, address_to_uprn_map)
+
+        self._merge_assets_into_houses(assets, houses)
+
+        return houses
+
+    @staticmethod
+    def _parse_assets(wb: Workbook) -> List[LbwfAssetCondition]:
+        assets_sheet = wb["Houses Asset Data"]
+        asset_rows = assets_sheet.iter_rows(values_only=True)
+
+        asset_headers = next(asset_rows)
+        asset_header_indexes = LbwfParser._get_column_indexes_by_name(asset_headers)
+
+        assets: List[LbwfAssetCondition] = []
+        for row in asset_rows:
+            try:
+                assets.append(
+                    LbwfParser._map_row_to_asset_record(row, asset_header_indexes)
+                )
+            except Exception as e:
+                logger.error(f"Error mapping LBWF row to asset record: {e}")
+                continue
+
+        return assets
+
+    @staticmethod
+    def _parse_houses(
+        wb: Workbook,
+        address_to_uprn_map: Dict[str, int],
+    ) -> List[LbwfHouse]:
+        houses_sheet = wb["Houses"]
+        house_rows = houses_sheet.iter_rows(values_only=True)
+
+        house_headers = next(house_rows)
+        house_header_indexes = LbwfParser._get_column_indexes_by_name(house_headers)
+
+        houses: List[LbwfHouse] = []
+        for row in house_rows:
+            try:
+                houses.append(
+                    LbwfParser._map_row_to_house_record(
+                        row,
+                        house_header_indexes,
+                        address_to_uprn_map,
+                    )
+                )
+            except Exception as e:
+                logger.error(f"Error mapping LBWF row to house record: {e}")
+                continue
+
+        return houses
+
+    @staticmethod
+    def _merge_assets_into_houses(
+        assets: List[LbwfAssetCondition],
+        houses: List[LbwfHouse],
+    ) -> None:
+        assets_by_ref: Dict[int, List[LbwfAssetCondition]] = defaultdict(list)
+        for asset in assets:
+            assets_by_ref[asset.prop_ref].append(asset)
+
+        for house in houses:
+            house.assets = assets_by_ref.get(house.reference, [])
+
+
+    @staticmethod
+    def _map_row_to_house_record(
+        row: Any | Tuple[object | None, ...],
+        header_indexes: Dict[str, int],
+        address_to_uprn_map: Dict[str, int],
+    ) -> LbwfHouse:
+        address: str = row[header_indexes["Address"]]
+
+        return LbwfHouse(
+            uprn=LbwfParser._get_uprn_from_address(address, address_to_uprn_map),
+            reference=row[header_indexes["Reference"]],
+            address=address,
+            epc=row[header_indexes["EPC "]],
+            shdf=row[header_indexes["SHDF"]],
+            house=row[header_indexes["HOSUE"]],
+            fail_decency=row[header_indexes["Fail Decency"]],
+            assets=[],
+        ) 
+    
+    @staticmethod
+    def _map_row_to_asset_record(
+        row: Any | Tuple[object | None, ...],
+        header_indexes: Dict[str, int],
+    ) -> LbwfAssetCondition:
+        return LbwfAssetCondition(
+            prop_ref=row[header_indexes["PROP REF"]],
+            domna=row[header_indexes["Domna"]],
+            address=row[header_indexes["ADDRESS"]],
+            ownership=row[header_indexes["OWNERSHIP"]],
+            prop_status=row[header_indexes["PROP STATUS"]],
+            prop_type=row[header_indexes["PROP TYPE"]],
+            prop_sub_type=row[header_indexes["PROP SUB TYPE"]],
+            element_group=row[header_indexes["ELEMENT GROUP"]],
+            element_code=row[header_indexes["ELEMENT CODE"]],
+            element_code_description=row[header_indexes["ELEMENT CODE DESCRIPTION"]],
+            attribute_code=row[header_indexes["ATTRIBUTE CODE"]],
+            attribute_code_description=row[header_indexes["ATTRIBUTE CODE DESCRIPTION"]],
+            element_date_value=row[header_indexes["ELEMENT DATE VALUE"]],
+            element_numerical_value=row[header_indexes["ELEMENT NUMERIC VALUE"]],
+            element_text_value=row[header_indexes["ELEMENT TEXT VALUE"]],
+            quantity=row[header_indexes["QUANTITY"]],
+            install_date=normalise_date(row[header_indexes["INSTALL DATE"]]),
+            remaining_life=row[header_indexes["REMAINING LIFE"]],
+            element_comments=row[header_indexes["ELEMENT COMMENTS"]],
+        )
+    
+
+    @staticmethod
+    def _generate_address_to_uprn_dict(wb: Workbook) -> Dict[str, int | None]:
+        sheet: Workbook = wb["All Energy Breakdown "]
+
+        rows: Iterator[Tuple[object | None, ...]] = sheet.iter_rows(values_only=True)
+
+        headers = next(rows)
+        header_indexes: Dict[str, int] = LbwfParser._get_column_indexes_by_name(headers)
+
+        address_idx = header_indexes["Address"]
+        uprn_idx = header_indexes["UPRN"]
+
+        mapping: Dict[str, int | None] = {}
+
+        for row in rows:
+            address = row[address_idx]
+            uprn = row[uprn_idx]
+
+            if not isinstance(address, str):
+                continue
+
+            if uprn is not None and not isinstance(uprn, int):
+                raise ValueError(f"Unexpected UPRN value: {uprn!r}")
+
+            mapping[address] = uprn
+
+        return mapping
+
+
+    def _get_column_indexes_by_name(
+        headers: Tuple[object | None, ...]
+    ) -> Dict[str, int]:
+        index: Dict[str, int] = {}
+
+        for i, header in enumerate(headers):
+            if isinstance(header, str):
+                index[header] = i
+
+        return index
+    
+    def _get_uprn_from_address(address: str, address_to_uprn_map: Dict[str, int]) -> int | None:
+        pseudo_name = address.split(",")[0]
+
+        if pseudo_name.lower() in (k.lower() for k in address_to_uprn_map.keys()):
+            return address_to_uprn_map[pseudo_name.upper()]
+        
+        return None
+        

backend/condition/parsing/parser.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+from typing import BinaryIO, Any
+
+class Parser(ABC):
+
+    @abstractmethod
+    def parse(self, file_stream: BinaryIO) -> Any:
+        pass

backend/condition/parsing/records/lbwf/lbwf_asset_condition.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from datetime import date
+
+
+@dataclass
+class LbwfAssetCondition:
+    prop_ref: int
+    domna: int
+    address: str
+    ownership: str
+    prop_status: str
+    prop_type: str # TODO: make this enum?
+    prop_sub_type: str  # TODO: make this enum?
+    element_group: str
+    element_code: str
+    element_code_description: str
+    attribute_code: str
+    attribute_code_description: str
+    element_date_value: str | None = None
+    element_numerical_value: int | None = None
+    element_text_value: str | None = None
+    quantity: int | None = None
+    install_date: date | None = None
+    remaining_life: int | None = None
+    element_comments: str | None = None
+

backend/condition/parsing/records/lbwf/lbwf_house.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+from typing import List
+
+from backend.condition.parsing.records.lbwf.lbwf_asset_condition import LbwfAssetCondition
+
+@dataclass
+class LbwfHouse:
+    uprn: int
+    reference: int
+    address: str
+    epc: str # TODO: make enum
+    shdf: bool
+    house: str
+    fail_decency: int
+    assets: List[LbwfAssetCondition]

backend/condition/processor.py

@@ -0,0 +1,18 @@
+from typing import Any, BinaryIO, List
+
+from backend.condition.parsing.parser import Parser
+from utils.logger import setup_logger
+from backend.condition.file_type import FileType, detect_file_type
+from backend.condition.parsing.factory import select_parser
+
+def process_file(file_stream: BinaryIO, source_key: str) -> None:
+    print(f"[processor] Received file: {source_key}")
+
+    # Instantiation
+    file_type: FileType = detect_file_type(source_key)
+    parser: Parser = select_parser(file_type)
+
+    # Orchestration
+    records: List[Any] = parser.parse(file_stream)
+
+    print(records) # temp

backend/condition/tests/parsing/test_lbwf_parser.py

@@ -0,0 +1,134 @@
+from typing import Any
+import pytest
+from io import BytesIO
+from openpyxl import Workbook
+from datetime import datetime
+
+from backend.condition.parsing.lbwf_parser import LbwfParser
+from backend.condition.parsing.records.lbwf.lbwf_asset_condition import LbwfAssetCondition
+from backend.condition.parsing.records.lbwf.lbwf_house import LbwfHouse
+
+@pytest.fixture
+def lbwf_homes_xlsx_bytes() -> BytesIO:
+    wb = Workbook()
+    houses_asset_data = wb.active
+    houses_asset_data.title = "Houses Asset Data"
+    houses_asset_data.append([
+        "PROP REF",
+        "Domna",
+        "ADDRESS",
+        "OWNERSHIP",
+        "PROP STATUS",
+        "PROP TYPE",
+        "PROP SUB TYPE",
+        "ELEMENT GROUP",
+        "ELEMENT CODE",
+        "ELEMENT CODE DESCRIPTION",
+        "ATTRIBUTE CODE",
+        "ATTRIBUTE CODE DESCRIPTION",
+        "ELEMENT DATE VALUE",
+        "ELEMENT NUMERIC VALUE",
+        "ELEMENT TEXT VALUE",
+        "QUANTITY",
+        "INSTALL DATE",
+        "REMAINING LIFE",
+        "ELEMENT COMMENTS"
+        ]
+    )
+    houses_asset_data.append([
+        12345,
+        12345,
+        "123 Fake Street, London, A10 1AB",
+        "LBWF_OWNED",
+        "OCCP",
+        "HOU",
+        "TERRACED",
+        "ASSETS",
+        "AHR_CAT",
+        "Accessible Housing Register Category",
+        "F",
+        "General Needs",
+        None,
+        None,
+        None,
+        1,
+        None,
+        None,
+        None,
+    ])
+    houses_asset_data.append([
+        54321,
+        54321,
+        "100 Random Road, London, A10 1AB",
+        "LBWF_OWNED",
+        "OCCP",
+        "HOU",
+        "EOT",
+        "ASSETS",
+        "INTSMKDET",
+        "Smoke Detectors in Property",
+        "HARDWRDMNS",
+        "Hard Wired Mains Smoke Alarm in Property",
+        None,
+        None,
+        None,
+        2,
+        datetime(2019,4,1),
+        4,
+        "Source of Data = Joe Bloggs",
+    ])
+    
+    houses = wb.create_sheet("Houses")
+    houses.append(["Reference", "Address", "EPC ", "SHDF", "HOSUE", "Fail Decency"])
+    houses.append([12345, "123 Fake Street, London, A10 1AB", "E", "NO", "HOUSE", 2025])
+    houses.append([54321, "100 Random Road, London, A10 1AB", "F", "NO", "HOUSE", 2025])
+
+    all_energy_breakdown = wb.create_sheet("All Energy Breakdown ") # Trailing space is intentional; matches source
+    all_energy_breakdown.append([
+        "UPRN",
+        "Organisation Reference",
+        "Alternate Organisation Reference",
+        "Address",
+        "Postcode"
+    ])
+    all_energy_breakdown.append([
+        1,
+        200,
+        None,
+        "123 FAKE STREET",
+        "A10 1AB"
+    ])
+    all_energy_breakdown.append([
+        2,
+        100,
+        101,
+        "100 RANDOM ROAD",
+        "A10 1AB"
+    ])
+
+    stream = BytesIO()
+    wb.save(stream)
+    stream.seek(0)
+
+    return stream
+
+def test_lbwf_parser_passes_houses(lbwf_homes_xlsx_bytes):
+    # arrange
+    parser = LbwfParser()
+
+    # act
+    result: Any = parser.parse(lbwf_homes_xlsx_bytes)
+
+    # assert
+    # TODO: Improve these asserts
+    assert len(result) == 2
+
+    assert isinstance(result[0], LbwfHouse)
+    assert result[0].uprn == 1
+    assert len(result[0].assets) == 1
+    assert isinstance(result[0].assets[0], LbwfAssetCondition)
+
+    assert isinstance(result[1], LbwfHouse)
+    assert result[1].uprn == 2
+    assert len(result[1].assets) == 1
+    assert isinstance(result[1].assets[0], LbwfAssetCondition)

backend/condition/tests/parsing/test_parsing_factory.py

@@ -0,0 +1,15 @@
+import pytest
+
+from backend.condition.parsing.factory import select_parser
+from backend.condition.file_type import FileType
+
+def test_selects_lbwf_parser():
+    # arrange
+    file_type = FileType.LBWF
+    expected_class_name = "LbwfParser"
+
+    # act
+    actual_class_name = select_parser(file_type).__class__.__name__
+
+    # assert
+    assert expected_class_name == actual_class_name

backend/condition/tests/test_detect_file_type.py

@@ -0,0 +1,22 @@
+import pytest
+
+from backend.condition.file_type import FileType, detect_file_type
+
+def test_detects_lbwf_file_type():
+    # arrange
+    file_path_str = "uploads/lbwf/Exaple Asset Data.xlsx"
+    expected_file_type = FileType.LBWF
+
+    # act
+    actual_file_type: FileType = detect_file_type(file_path_str)
+
+    # assert
+    assert expected_file_type == actual_file_type
+
+def test_unknown_filepath_raises_value_error():
+    # arrange
+    file_path_str = "unknown/Example Asset Data.xlsx"
+
+    # act + assert
+    with pytest.raises(ValueError):
+        detect_file_type(file_path_str)

backend/condition/utils/date_utils.py

@@ -0,0 +1,18 @@
+from datetime import datetime, date
+from typing import Any
+
+
+def normalise_date(value: Any, allow_none: bool = True) -> date | None:
+    if value is None and allow_none:
+        return None
+    
+    if isinstance(value, datetime):
+        return value.date()
+    
+    if isinstance(value, str):
+        try:
+            return datetime.strptime(value.strip(), "%d/%m/%Y").date()
+        except ValueError as exc:
+            raise ValueError(f"Invalid date string: {value!r}") from exc
+        
+    raise ValueError(f"Unexpected date value: {value!r}")

backend/engine/engine.py

@@ -1118,7 +1118,7 @@ async def model_engine(body: PlanTriggerRequest):
                     p=p,
                     input_measures=input_measures,
                     budget=body.budget,
-                    target_gain=gain - already_installed_sap,
+                    target_gain=gain,
                     enforce_heat_pump_insulation=True,
                     enforce_fabric_first=body.enforce_fabric_first,
                     already_installed_sap=already_installed_sap,  # To be passed to output

backend/onboarders/parity.py

@@ -65,7 +65,7 @@ data["Wall Insulation"].value_counts()
 data["Wall Construction"].value_counts()
 
 as_built_map = {
-    "Cavity": {"insulated_age_bands":[], "partial_insulated_age_bands": []},
+    "Cavity": {"insulated_age_bands": [], "partial_insulated_age_bands": []},
     "Solid Brick": {"insulated_age_bands": [], "partial_insulated_age_bands": []},
     "System": {"insulated_age_bands": [], "partial_insulated_age_bands": []},
     "Timber Frame": {"insulated_age_bands": [], "partial_insulated_age_bands": []},
@@ -74,6 +74,7 @@ as_built_map = {
     "Cob": {"insulated_age_bands": [], "partial_insulated_age_bands": []},
 }
 
+
 def map_wall_construction(wall_constuction, wall_insulation, construction_age_band):
     if wall_insulation == "AsBuilt":
         # Deduce based on wall construction and age band
@@ -83,13 +84,10 @@ def map_wall_construction(wall_constuction, wall_insulation, construction_age_ba
 
         # We check if the age band is in insulated or partial insulated, and if neither, we assume uninsulated
 
-
-
-
 # Variables we want to map
-'Org Ref', 'Address 1', 'Address 2', 'Address 3', 'Postcode', 'Type',
-       'Attachment', 'Construction Years', 'Wall Construction',
-       'Wall Insulation', 'Roof Construction', 'Roof Insulation',
-       'Floor Construction', 'Floor Insulation', 'Glazing', 'Heating',
-       'Boiler Efficiency', 'Main Fuel', 'Controls Adequacy', 'UPRN',
-       'Total Floor Area (m2)'
+# 'Org Ref', 'Address 1', 'Address 2', 'Address 3', 'Postcode', 'Type',
+#        'Attachment', 'Construction Years', 'Wall Construction',
+#        'Wall Insulation', 'Roof Construction', 'Roof Insulation',
+#        'Floor Construction', 'Floor Insulation', 'Glazing', 'Heating',
+#        'Boiler Efficiency', 'Main Fuel', 'Controls Adequacy', 'UPRN',
+#        'Total Floor Area (m2)'

backend/tests/test_funding.py

@@ -1395,7 +1395,7 @@ def test_private_epc_e_solar_with_heating_and_minimum_insulation_produces_uplift
     assert funding.eco4_funding and funding.eco4_funding > 0
 
 
-def test_existing_gshp_to_ashp():
+def test_existing_gshp_to_ashp(mock_project_scores_matrix, mock_partial_scores_matrix, mock_whlg_postcodes):
     r = {'phase': 3, 'parts': [], 'type': 'heating', 'measure_type': 'air_source_heat_pump',
          'description': 'Install a 5KW air source heat pump, and upgrade heating controls to Smart Thermostats, '
                         'room sensors and smart radiator valves (time & temperature zone control). Ensure you have a '

etl/customers/peabody/Nov 2025 Consulting Project/k_deck_stats.py

@@ -114,14 +114,16 @@ from backend.app.db.models.recommendations import Recommendation, Plan, PlanReco
 from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel
 from collections import defaultdict
 
-PORTFOLIO_ID = 434  # Peabody
+PORTFOLIO_ID = 435  # Peabody
 SCENARIOS = [
-    904,
-    905
+    908,
+    909,
+    910,
 ]
 scenario_names = {
-    904: "EPC C - no solid floor, ashp 3.0",
-    905: "EPC B - no solid floor, ashp 3.0",
+    908: "EPC C - no solid floor, ashp 3.0",
+    909: "EPC C - no solid floor, no EWI or IWI, ashp 3.0",
+    910: "EPC B - no solid floor, no EWI, ashp 3.0"
 }
 
 
@@ -232,9 +234,58 @@ properties_data, plans_data, recommendations_data = get_data(
 
 recommendations_df = pd.DataFrame(recommendations_data)
 properties_df = pd.DataFrame(properties_data)
+plans_df = pd.DataFrame(plans_data)
 
-solar_pv_recommendations = recommendations_df[recommendations_df["measure_type"] == "solar_pv"]
+s_id = 910
+ps_w_a_plan = plans_df[plans_df["scenario_id"] == s_id].copy()
+# Take the newest by scenario id
+ps_w_a_plan = ps_w_a_plan.sort_values("created_at", ascending=False).drop_duplicates(
+    subset=["property_id"]
+)
+z = ps_w_a_plan[
+    ps_w_a_plan["cost_of_works"] > 0
+    ].copy()
+z2 = properties_df[properties_df["property_id"].isin(z["property_id"].values)]
+# '', 'hot_water_cost_current',
+#        'lighting_cost_current', 'appliances_cost_current',
+#        'gas_standing_charge', 'electricity_standing_charge'
+z2["total_bills"] = z2["heating_cost_current"] + z2["hot_water_cost_current"] + z2["lighting_cost_current"] + z2[
+    "appliances_cost_current"
+] + z2["gas_standing_charge"] + z2["electricity_standing_charge"]
+
+from tqdm import tqdm
+
+# For a property ID, find a property where the no EWI/IWI approach is more expensive than the EWI approach
+pids = properties_df["property_id"].unique()
+for pid in tqdm(pids):
+
+    if pid in [603272, 550550, 574493]:
+        continue
+
+    # get the plans
+    property_plan = plans_df[plans_df["property_id"] == int(pid)]
+    # Take the newest plan by scenario id
+    property_plan = property_plan.sort_values("created_at", ascending=False).drop_duplicates(
+        subset=["scenario_id"]
+    )
+    a = property_plan[property_plan["scenario_id"] == 909].squeeze()  # no EWI/IWI
+    b = property_plan[property_plan["scenario_id"] == 908].squeeze()  # EWI
+    if (a["cost_of_works"] > b["cost_of_works"]) and (
+        a["post_epc_rating"].value == "C") and (b["cost_of_works"] > 5000):
+        bah
+
+solar_pv_recommendations = recommendations_df[
+    recommendations_df["measure_type"] == "solar_pv"
+    ]
+
+solid_wall_recommendation = recommendations_df[
+    recommendations_df["scenario_id"].isin([908]) &
+    recommendations_df["measure_type"].isin(["internal_wall_insulation"]) &
+    recommendations_df["default"]
+    ]
 average_savings = solar_pv_recommendations.groupby("scenario_id")["energy_cost_savings"].mean().reset_index()
+# Add on scenarion names
+average_savings["scenario_name"] = average_savings["scenario_id"].map(scenario_names)
 
 # Check tenures
 initial_asset_data = pd.read_excel(

etl/customers/peabody/Nov 2025 Consulting Project/n_fixing_already_installed_bug.py

@@ -11,14 +11,12 @@ from etl.customers.cambridge.surveys import current_epc
 with db_session() as session:
     # We need installed measures, where the measure type is ewi or iwi
     installed_measures = session.query(InstalledMeasure).filter(
-        InstalledMeasure.measure_type.in_(["external_wall_insulation", "internal_wall_insulation"])
     ).all()
     # Get the uprns
     installed_uprns = [x.uprn for x in installed_measures]
 
 installed_uprns = list(set(installed_uprns))
 
-# This is 21425 properties.
 # We then create a portfolio of properties we need to re-run
 import pandas as pd
 
@@ -33,7 +31,7 @@ needing_retry = sal[sal["epc_os_uprn"].isin(installed_uprns)]
 # Store
 needing_retry.to_excel(
     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final "
-    "SAL/properties_needing_retry_20260115.xlsx",
+    "SAL/properties_needing_retry_20260115 - all already installed.xlsx",
     sheet_name="Standardised Asset List",
     index=False
 )

etl/customers/peabody/Nov 2025 Consulting Project/o_rerunning_iwi_jobs.py

@@ -0,0 +1,41 @@
+# get all properties that have an IWI recommendation
+import pandas as pd
+
+r1 = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/EPC B - no "
+    "solid floor, no EWI, ashp 3.0 - 20250113 final.xlsx"
+)
+
+r2 = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/EPC C - no "
+    "solid floor, ashp 3.0 - 20250113 final.xlsx"
+)
+
+r3 = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/EPC C - no "
+    "solid floor, no EWI or IWI, ashp 3.0 - 20250113 final.xlsx"
+)
+
+s1 = r1[~pd.isnull(r1["internal_wall_insulation"])]
+s2 = r2[~pd.isnull(r2["internal_wall_insulation"])]
+
+# Combined uprns
+uprns = s1["uprn"].tolist() + s2["uprn"].tolist()
+uprns = list(set(uprns))
+
+# Create SAL of these uprns
+sal = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260113 - "
+    "final asset list.xlsx",
+    sheet_name="Standardised Asset List"
+)
+
+needing_retry = sal[sal["epc_os_uprn"].isin(uprns)]
+
+# Store
+needing_retry.to_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final "
+    "SAL/properties_needing_retry_20260115 - internal wall insulation.xlsx",
+    sheet_name="Standardised Asset List",
+    index=False
+)

etl/epc_clean/tests/test_data/test_floor_attributes_cases.py

@@ -378,7 +378,7 @@ clean_floor_cases = [
     },
     {
         # This example gets remapped to another dwelling below
-        "description": "Above unheated space or full exposed",
+        "original_description": "Above unheated space or full exposed",
         'thermal_transmittance': 0, 'thermal_transmittance_unit': 'w/m-¦k', 'is_assumed': False,
         'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False,
         'another_property_below': True, 'insulation_thickness': None

etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py

@@ -226,7 +226,7 @@ hotwater_cases = [
     {'original_description': 'Single-point gas water heater, standard tariff',
      'heater_type': 'single-point gas', 'system_type': "water heater", 'thermostat_characteristics': None,
      'heating_scope': None, 'energy_recovery': None, 'tariff_type': 'standard tariff', 'extra_features': None,
-     'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'appliance': None
+     'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'appliance': None, "assumed": False
      }
 
 ]

etl/epc_clean/tests/test_floor_attributes.py

@@ -15,7 +15,12 @@ class TestCleanFloor:
         empty = FloorAttributes('')
         assert empty.nodata
         output = empty.process()
-        assert output == {"no_data": True}
+        assert output == {
+            'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': True,
+            'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': True,
+            'is_solid': False, 'another_property_below': False, 'insulation_thickness': 'none',
+            'no_data': True
+        }
 
         # Test initialization with a description that contains none of the keywords
         with pytest.raises(ValueError):

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 pythonpath = .
 addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial
-testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests
+testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests

recommendations/Recommendations.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import numpy as np
 from backend.Property import Property
-from typing import List
+from typing import List, Mapping, Any
 from itertools import groupby
 from recommendations.FloorRecommendations import FloorRecommendations
 from recommendations.WallRecommendations import WallRecommendations
@@ -31,6 +31,18 @@ class Recommendations:
     High level recommendations class, which sits above the measure specific recommendation classes
     """
 
+    # Used in calculation of recommendation impact - increasing variables are features where
+    # a higher value indicates an improvement. Decreasing is the opposite
+    INCREASING_VARIABLES = ["sap"]
+    DECREASING_VARIABLES = ["carbon", "heat_demand"]
+
+    # If the recommendation is mechanical ventilation, we don't apply the rule that the new value should be higher
+    MV_INCREASING_VARIABLES = ["carbon", "heat_demand"]
+    MV_DECREASING_VARIABLES = ["sap"]
+
+    # List of models we expect predictions for, when calculation recommendation impact
+    PREDICTION_PREFIXES = ["sap_change", "heat_demand", "carbon_change"]
+
     def __init__(
         self,
         property_instance: Property,
@@ -486,19 +498,354 @@ class Recommendations:
 
         return predicted_appliances_cost_reduction, predicted_appliances_kwh_reduction
 
+    @staticmethod
+    def _check_ventilation_out_of_bounds(sap_impact, ventilation_sap_limit):
+        return (sap_impact < ventilation_sap_limit) or (sap_impact >= 0)
+
+    @staticmethod
+    def _adjust_ventilation_sap(sap_impact, ventilation_sap_limit):
+
+        if sap_impact >= 0:
+            return -1
+        if sap_impact < ventilation_sap_limit:
+            return ventilation_sap_limit
+
+        return sap_impact
+
+    @staticmethod
+    def _filter_phase_adjustment(phase_adjustments):
+        """
+        Utility function to select the entry from the dictionary, by phase, with the largest
+        phase adjustment
+        :param phase_adjustments: List of phase adjustments, in the form
+        [{"recommendation_id": str, "phase": int, "sap_adjustment": float}]
+        :return:
+        """
+        filtered_adjustments = []
+        phase_adjustments = sorted(phase_adjustments, key=lambda x: x["phase"])
+        for phase, adjustments in groupby(phase_adjustments, key=lambda x: x["phase"]):
+            adjustments = list(adjustments)
+            adjustments.sort(key=lambda x: x["sap_adjustment"], reverse=True)
+            filtered_adjustments.append(adjustments[0])
+        return filtered_adjustments
+
+    @classmethod
+    def _filter_predictions_for_property(
+        cls,
+        all_predictions: Mapping[str, pd.DataFrame],
+        property_id: str,
+    ) -> dict:
+        """
+        Utility function to filter predictions for a specific property
+        :param all_predictions: Dictionary of all predictions from the model apis
+        :param property_id: The property id to filter for
+        :return:
+        """
+
+        return {
+            f"{prefix}_predictions": (
+                all_predictions[f"{prefix}_predictions"]
+                .loc[
+                    all_predictions[f"{prefix}_predictions"]["property_id"] == property_id
+                    ]
+                .copy()
+            )
+            for prefix in cls.PREDICTION_PREFIXES
+        }
+
+    @classmethod
+    def get_monotonic_variables(cls, rec_type: str) -> tuple[List[str], List[str]]:
+        """
+        Utility function to get the monotonic variables for a specific recommendation type
+        :param rec_type: The recommendation type
+        :return:
+        """
+        if rec_type == "mechanical_ventilation":
+            return cls.MV_INCREASING_VARIABLES, cls.MV_DECREASING_VARIABLES
+        return cls.INCREASING_VARIABLES, cls.DECREASING_VARIABLES
+
+    @staticmethod
+    def _get_previous_phase_values(
+        rec_phase: int,
+        starting_phase: int,
+        impact_summary: list[dict],
+        property_instance: Property,
+    ) -> dict:
+        if rec_phase == starting_phase:
+            return {
+                "sap": float(property_instance.data["current-energy-efficiency"]),
+                "carbon": float(property_instance.data["co2-emissions-current"]),
+                "heat_demand": float(property_instance.data["energy-consumption-current"]),
+            }
+
+        previous_phase_reps = [
+            x for x in impact_summary
+            if x["phase"] == rec_phase - 1 and x["representative"]
+        ]
+
+        if len(previous_phase_reps) == 1:
+            return previous_phase_reps[0]
+
+        # It's unlikely that this will occur but this fallback will ensure that we don't
+        # run the next step and run a median of nothing, which will return None
+        if not previous_phase_reps:
+            return {
+                "sap": float(property_instance.data["current-energy-efficiency"]),
+                "carbon": float(property_instance.data["co2-emissions-current"]),
+                "heat_demand": float(property_instance.data["energy-consumption-current"]),
+            }
+
+        # Median fallback (including zero-length case)
+        keys = ("sap", "carbon", "heat_demand")
+        return {
+            key: np.median([item[key] for item in previous_phase_reps])
+            for key in keys
+        }
+
+    @classmethod
+    def _get_phase_predictions(
+        cls,
+        property_predictions: dict,
+        recommendation_id: str,
+    ) -> dict:
+        return {
+            prefix: (
+                property_predictions[f"{prefix}_predictions"]
+                .loc[
+                    property_predictions[f"{prefix}_predictions"]["recommendation_id"]
+                    == str(recommendation_id),
+                    "predictions",
+                ]
+                .values[0]
+            )
+            for prefix in cls.PREDICTION_PREFIXES
+        }
+
+    @classmethod
+    def _resolve_current_phase_sap(
+        cls,
+        rec: Mapping[str, Any],
+        previous_phase_values: Mapping[str, Any],
+        phase_energy_efficiency_metrics: Mapping[str, Any],
+        adjustments: list[dict],
+    ) -> float:
+        if rec.get("survey", False):
+            return rec["sap_points"] + previous_phase_values["sap"]
+
+        sap = phase_energy_efficiency_metrics["sap_change"]
+
+        prior_adjustments = [a for a in adjustments if a["phase"] < rec["phase"]]
+        if not prior_adjustments:
+            return sap
+
+        filtered = cls._filter_phase_adjustment(prior_adjustments)
+        return sap - sum(a["sap_adjustment"] for a in filtered)
+
+    @classmethod
+    def _compute_phase_impact(
+        cls,
+        rec_type: str,
+        previous_phase_values: dict,
+        current_phase_values: dict,
+    ) -> dict:
+        """
+        Utility function for computing the impact of a recommendation phase, enforcing monotonicity
+
+        :param rec_type: string, the recommendation type
+        :param previous_phase_values: dict, the previous phase values
+        :param current_phase_values: dict, the current phase values
+        :return: dict, the impact of the phase
+        """
+        phase_increasing, phase_decreasing = cls.get_monotonic_variables(rec_type)
+
+        # Enforce monotonicity
+        for v in phase_increasing:
+            current_phase_values[v] = max(current_phase_values[v], previous_phase_values[v])
+
+        for v in phase_decreasing:
+            current_phase_values[v] = min(current_phase_values[v], previous_phase_values[v])
+
+        # Compute impact
+        impact = {
+            "sap": current_phase_values["sap"] - previous_phase_values["sap"],
+            "carbon": previous_phase_values["carbon"] - current_phase_values["carbon"],
+            "heat_demand": previous_phase_values["heat_demand"] - current_phase_values["heat_demand"],
+        }
+
+        # Clamp values
+        for metric in impact:
+            if rec_type != "mechanical_ventilation":
+                impact[metric] = max(0, impact[metric])
+                if metric == "sap":
+                    impact[metric] = round(impact[metric], 2)
+            else:
+                impact[metric] = min(0, impact[metric])
+
+        return impact
+
+    @classmethod
+    def _apply_measure_specific_rules(
+        cls,
+        rec: dict,
+        property_phase_impact: dict,
+        previous_phase_values: dict,
+        current_phase_values: dict,
+        adjustments: list,
+        property_instance,
+    ):
+        # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2
+        if rec["type"] == "low_energy_lighting":
+            lighting_sap_limit = LightingRecommendations.get_sap_limit(
+                property_instance.data["lighting-energy-eff"],
+                property_instance.lighting["low_energy_proportion"]
+            )
+
+            # add an adjustment
+            proposed_sap_impact = min(property_phase_impact["sap"], lighting_sap_limit)
+            if proposed_sap_impact != property_phase_impact["sap"]:
+                # Store the sap adjustment. The proposed sap impact will always be less
+                # than the current sap impact, so the adjustment is always positive
+                # as we subtract it from the future phases
+                adjustments.append(
+                    {
+                        "recommendation_id": rec["recommendation_id"],
+                        "phase": rec["phase"],
+                        "sap_adjustment": property_phase_impact["sap"] - proposed_sap_impact,
+                    }
+                )
+
+            property_phase_impact["sap"] = proposed_sap_impact
+            property_phase_impact["carbon"] = min(
+                property_phase_impact["carbon"], rec["co2_equivalent_savings"]
+            )
+
+            # Update the current phase values
+            current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+            current_phase_values["carbon"] = previous_phase_values["carbon"] - property_phase_impact["carbon"]
+        elif rec["type"] == "mechanical_ventilation":
+            # ventilation is capped by having no greater and a -4 impact
+            ventilation_sap_limit = -4
+            ventilation_out_of_bounds = cls._check_ventilation_out_of_bounds(
+                property_phase_impact["sap"], ventilation_sap_limit
+            )
+
+            if ventilation_out_of_bounds:
+                previous_modelled_sap = previous_phase_values.get("sap_prediction", 0)
+                proposed_sap_impact = current_phase_values["sap"] - previous_modelled_sap
+                proposal_out_of_bounds = cls._check_ventilation_out_of_bounds(
+                    proposed_sap_impact, ventilation_sap_limit
+                )
+                if proposal_out_of_bounds:
+                    proposed_sap_impact = cls._adjust_ventilation_sap(
+                        proposed_sap_impact, ventilation_sap_limit
+                    )
+
+                # We keep track of the adjustment
+                # In this case, if the SAP impact has increased, then the adustment should be negative
+                # otherwise it should be positive
+                # When we add the total adjustment, it's an addition
+                # Example
+                # Before: 60, impact -2 => 58
+                # After: 60, impact -1 (So the impact is bigger) => 59
+                # So in this case, we need to make sure we add 1 to all future predictions so
+                # the adjustment should be positive
+                # Before: 60, impact 1 => 61
+                # After: 60, impact -1 => 59
+                # So in this case, we need to make sure we subtract 1 to all future predictions so
+                # the adjustment should be negative
+                # Both cases are reflected in sap adjustment
+                sap_adjustment = proposed_sap_impact - float(property_phase_impact["sap"])
+
+                adjustments.append(
+                    {
+                        "recommendation_id": rec["recommendation_id"],
+                        "phase": rec["phase"],
+                        "sap_adjustment": sap_adjustment,
+                    }
+                )
+
+                property_phase_impact["sap"] = proposed_sap_impact
+
+                # Update the current phase values
+                current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+        elif rec["type"] == "loft_insulation":
+            # When we have a loft insulation recommendation, where there is an extension and the existing
+            # amount of loft insulation is already good, we limit the SAP points
+            # By limiting here, we don't change the value in current_phase_values. This means that the
+            # future recommendations won't have an impact that is too large
+            li_sap_limit = RoofRecommendations.get_loft_insulation_sap_limit(
+                property_instance.data["roof-energy-eff"], property_instance.roof["insulation_thickness"]
+            )
+            if li_sap_limit is not None:
+                new_value = min(property_phase_impact["sap"], li_sap_limit)
+                # If we've made an adjustment, keep track of it
+                if new_value != property_phase_impact["sap"]:
+                    adjustments.append(
+                        {
+                            "recommendation_id": rec["recommendation_id"],
+                            "phase": rec["phase"],
+                            # If we've made an adjustment, it will be negative
+                            "sap_adjustment": property_phase_impact["sap"] - new_value,
+                        }
+                    )
+                property_phase_impact["sap"] = new_value
+                # Update the current phase values
+                current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+        elif rec["type"] == "solar_pv":
+            # We use the SAP points in the recommendation as a minimum
+            proposed_impact = (
+                rec["sap_points"] if property_phase_impact["sap"] < rec["sap_points"] else
+                property_phase_impact["sap"]
+            )
+
+            # SAP adjustments should be negative
+            if proposed_impact != property_phase_impact["sap"]:
+                adjustments.append(
+                    {
+                        "recommendation_id": rec["recommendation_id"],
+                        "phase": rec["phase"],
+                        # If we've made an adjustment, we will be increasing the number of SAP
+                        # points. Since, we subtract adjustments, this number should be negative
+                        "sap_adjustment": property_phase_impact["sap"] - proposed_impact,
+                    }
+                )
+            property_phase_impact["sap"] = proposed_impact
+
+            # Update the current phase values
+            current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+
+        return property_phase_impact, current_phase_values, adjustments
+
+    @staticmethod
+    def _validate_recommendation_updates(rec: Mapping[str, Any]):
+        """
+        Utility function to validate that the recommendation updates have been applied correctly
+        :param rec: updated recommendation
+        :return:
+        """
+        if (
+            (rec["sap_points"] is None) and (rec["co2_equivalent_savings"] is None) or
+            (rec["heat_demand"] is None)
+        ):
+            raise ValueError("sap points, co2 or heat demand is missing")
+
     @classmethod
     def calculate_recommendation_impact(
         cls,
-        property_instance,
-        all_predictions,
-        recommendations,
-        representative_recommendations,
-    ):
+        property_instance: Property,
+        all_predictions: Mapping[str, Any],
+        recommendations: Mapping[int, List],
+        representative_recommendations: Mapping[int, List],
+        debug: bool = False
+    ) -> (Mapping[int, List], List[Mapping[str, Any]]):
 
         """
         Given predictions from the model apis, with method will update the recommendations with the predicted
         impact of the recommendation on the property
 
+        This algorithm is structured as a large loop, but this is due to the fact that it's sequential in nature -
+        each phase depends on the previous, with adjustments and constraints being allied along the way
+
         This function will return two objects:
         1) Updated recommendations with the predicted impact of the recommendation
         2) A list of impacts by phase, which will be used for the kwh model scoring
@@ -507,49 +854,43 @@ class Recommendations:
         :param all_predictions: dictionary of predictions from the model apis
         :param recommendations: dictionary of recommendations for the property
         :param representative_recommendations: dictionary of representative recommendations for the property
-        :return:
+        :param debug: boolean, indicating if the function is running in debug mode. The only difference is that
+                      adjustments are returned for testing
+
+        :return: Updated recommendations with predicted impact, and a list of impacts by phase
         """
-
-        property_predictions = {
-            prefix + "_predictions": all_predictions[prefix + "_predictions"][
-                all_predictions[prefix + "_predictions"]["property_id"] == str(property_instance.id)
-                ].copy() for prefix in ["sap_change", "heat_demand", "carbon_change"]
-        }
-
-        property_recommendations = recommendations[property_instance.id].copy()
-
-        representative_recs = representative_recommendations[property_instance.id].copy()
-        representative_ids = [r["recommendation_id"] for r in representative_recs]
-
-        increasing_variables = ["sap"]
-        decreasing_variables = ["carbon", "heat_demand"]
-
-        # If the recommendation is mechanical ventilation, we don't apply the rule that the new value should be higher
-        mv_increasing_variables = ["carbon", "heat_demand"]
-        mv_decreasing_variables = ["sap"]
-
-        # We allow for negative phase
-        starting_phase = min(
-            rec["phase"] for recs in property_recommendations for rec in recs
+        property_predictions = cls._filter_predictions_for_property(
+            all_predictions, str(property_instance.id)
         )
 
-        impact_summary = []
+        # shallow copy intentional - we're going to modify the internals
+        property_recommendations = recommendations[property_instance.id].copy()
+
+        representative_ids = [
+            r["recommendation_id"] for r in representative_recommendations[property_instance.id]
+        ]
+
+        # We allow for negative phase
+        starting_phase = min(rec["phase"] for recs in property_recommendations for rec in recs)
+
+        # We keep a history of adjustments we have made, so that we ensure that we adjust future
+        # phases for SAP
+        impact_summary, adjustments = [], []
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
-                if rec["type"] in ["trickle_vents", "draught_proofing", "extension_cavity_wall_insulation"]:
-                    # We don't have a percieved sap impact of mechanical ventilation or trickle vents, and we don't
-                    # have the capacity to score draught proofing
+                # --- Special-case: non-modelled measures -------------------------
+                if rec["type"] in {
+                    "trickle_vents",
+                    "draught_proofing",
+                    "extension_cavity_wall_insulation",
+                }:
                     if rec["type"] == "extension_cavity_wall_insulation":
-
-                        previous_phase = [x for x in impact_summary if x["phase"] == (rec["phase"] - 1)]
-                        if previous_phase:
-                            sap = previous_phase[0]["sap"]
-                            carbon = previous_phase[0]["carbon"]
-                            heat_demand = previous_phase[0]["heat_demand"]
-                        else:
-                            sap = float(property_instance.data["current-energy-efficiency"])
-                            carbon = float(property_instance.data["co2-emissions-current"])
-                            heat_demand = float(property_instance.data["energy-consumption-current"])
+                        previous = cls._get_previous_phase_values(
+                            rec_phase=rec["phase"],
+                            starting_phase=starting_phase,
+                            impact_summary=impact_summary,
+                            property_instance=property_instance,
+                        )
 
                         impact_summary.append(
                             {
@@ -557,62 +898,29 @@ class Recommendations:
                                 "representative": rec["recommendation_id"] in representative_ids,
                                 "recommendation_id": rec["recommendation_id"],
                                 "measure_type": rec["measure_type"],
-                                "sap": sap + rec["sap_points"],
-                                "carbon": carbon - rec["co2_equivalent_savings"],
-                                "heat_demand": heat_demand - rec["heat_demand"],
+                                "sap": previous["sap"] + rec["sap_points"],
+                                "carbon": previous["carbon"] - rec["co2_equivalent_savings"],
+                                "heat_demand": previous["heat_demand"] - rec["heat_demand"],
                             }
                         )
                     continue
 
-                phase_energy_efficiency_metrics = {
-                    prefix: property_predictions[prefix + "_predictions"][
-                        property_predictions[prefix + "_predictions"]["recommendation_id"] == str(
-                            rec["recommendation_id"]
-                        )]["predictions"].values[0] for prefix in ["sap_change", "heat_demand", "carbon_change"]
-                }
+                phase_energy_efficiency_metrics = cls._get_phase_predictions(
+                    property_predictions=property_predictions,
+                    recommendation_id=rec["recommendation_id"],
+                )
 
-                # We structure this so that depending on the phase, we capture the previous phase impacts and
-                # then just have one piece of code to calculate the difference
-                if rec["phase"] == starting_phase:
-                    # These are just the starting values, from the EPC. When we score the ML models,
-                    # heating_cost_starting and heating_cost_ending are just the values in the EPC. However, with
-                    # heating_cost_ending, we expect that the EPC will predict a heating cost based on what would happen
-                    # if we implemented the recommendation today, so our starting value is the EPC
-
-                    previous_phase_values = {
-                        "sap": float(property_instance.data["current-energy-efficiency"]),
-                        # For carbon, even though we generally use the updated figure which includes the carbon
-                        # associated to appliances, for this scoring process we use the EPC carbon value. This means
-                        # that we don't overestimate the impact since the model uses the EPC carbon value
-                        "carbon": float(property_instance.data["co2-emissions-current"]),
-                        "heat_demand": float(property_instance.data["energy-consumption-current"]),
-                    }
-
-                else:
-
-                    previous_phase_values_multiple = [
-                        x for x in impact_summary if x["phase"] == (rec["phase"] - 1) and x["representative"]
-                    ]
-                    if len(previous_phase_values_multiple) != 1:
-                        # Take an average of each of the previous phases
-                        keys_to_median = ["sap", "carbon", "heat_demand"]
-
-                        previous_phase_values = {}
-                        for key in keys_to_median:
-                            values = [item[key] for item in previous_phase_values_multiple]
-                            previous_phase_values[key] = np.median(values)
-
-                    else:
-                        previous_phase_values = previous_phase_values_multiple[0]
-
-                # We extract the values for the current phase
-                if rec.get("survey", False):
-                    current_phase_sap = rec["sap_points"] + previous_phase_values["sap"]
-                else:
-                    current_phase_sap = phase_energy_efficiency_metrics["sap_change"]
+                previous_phase_values = cls._get_previous_phase_values(
+                    rec_phase=rec["phase"],
+                    starting_phase=starting_phase,
+                    impact_summary=impact_summary,
+                    property_instance=property_instance
+                )
 
                 current_phase_values = {
-                    "sap": current_phase_sap,
+                    "sap": cls._resolve_current_phase_sap(
+                        rec, previous_phase_values, phase_energy_efficiency_metrics, adjustments
+                    ),
                     "carbon": phase_energy_efficiency_metrics["carbon_change"],
                     "heat_demand": phase_energy_efficiency_metrics["heat_demand"],
                 }
@@ -625,113 +933,20 @@ class Recommendations:
                 # However, if the recommendation is mechanical ventilation, this can have a negative SAP impact so
                 # we don't apply this rule
 
-                if rec["type"] == "mechanical_ventilation":
-                    phase_increasing_variables = mv_increasing_variables
-                    phase_decreasing_variables = mv_decreasing_variables
-                else:
-                    phase_increasing_variables = increasing_variables
-                    phase_decreasing_variables = decreasing_variables
+                property_phase_impact = cls._compute_phase_impact(
+                    rec_type=rec["type"],
+                    previous_phase_values=previous_phase_values,
+                    current_phase_values=current_phase_values,
+                )
 
-                for v in phase_increasing_variables:
-                    current_phase_values[v] = (
-                        current_phase_values[v] if current_phase_values[v] > previous_phase_values[v] else
-                        previous_phase_values[v]
-                    )
-                for v in previous_phase_values:
-                    if v in phase_decreasing_variables:
-                        current_phase_values[v] = (
-                            current_phase_values[v] if current_phase_values[v] < previous_phase_values[v] else
-                            previous_phase_values[v]
-                        )
-
-                property_phase_impact = {
-                    # Increasing
-                    "sap": current_phase_values["sap"] - previous_phase_values["sap"],
-                    # Decreasing
-                    "carbon": previous_phase_values["carbon"] - current_phase_values["carbon"],
-                    # Decreasing
-                    "heat_demand": previous_phase_values["heat_demand"] - current_phase_values["heat_demand"],
-                }
-
-                # Prevent from being negative - apart from ventilation
-                for metric in ["sap", "carbon", "heat_demand"]:
-                    if rec["type"] != "mechanical_ventilation":
-                        property_phase_impact[metric] = (
-                            0 if property_phase_impact[metric] < 0 else property_phase_impact[metric]
-                        )
-                        if metric == "sap":
-                            property_phase_impact[metric] = round(property_phase_impact[metric], 2)
-                    else:
-                        # We prevent mechanical ventilation from being positive
-                        property_phase_impact[metric] = (
-                            0 if property_phase_impact[metric] > 0 else property_phase_impact[metric]
-                        )
-
-                # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2
-                if rec["type"] == "low_energy_lighting":
-                    lighting_sap_limit = LightingRecommendations.get_sap_limit(
-                        property_instance.data["lighting-energy-eff"],
-                        property_instance.lighting["low_energy_proportion"]
-                    )
-
-                    property_phase_impact["sap"] = min(property_phase_impact["sap"], lighting_sap_limit)
-                    property_phase_impact["carbon"] = min(
-                        property_phase_impact["carbon"], rec["co2_equivalent_savings"]
-                    )
-
-                    # Update the current phase values
-                    current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
-                    current_phase_values["carbon"] = previous_phase_values["carbon"] - property_phase_impact["carbon"]
-
-                # We also ensure that mechanical ventilation doesn't have an ovely strong negative SAP impact
-                if rec["type"] == "mechanical_ventilation":
-                    # ventilation is capped by having no greater and a -4 impact
-                    ventilation_sap_limit = -4
-
-                    def _check_veniltation_out_of_bounds(sap_impact):
-                        return (sap_impact < ventilation_sap_limit) or (sap_impact >= 0)
-
-                    def _adjust_ventilation_sap(sap_impact):
-                        if sap_impact >= 0:
-                            return -1
-                        if sap_impact < ventilation_sap_limit:
-                            return ventilation_sap_limit
-
-                    ventilation_out_of_bounds = _check_veniltation_out_of_bounds(property_phase_impact["sap"])
-
-                    if ventilation_out_of_bounds:
-                        previous_modelled_sap = previous_phase_values.get("sap_prediction", 0)
-                        proposed_sap_impact = current_phase_sap - previous_modelled_sap
-                        proposal_out_of_bounds = _check_veniltation_out_of_bounds(proposed_sap_impact)
-                        if proposal_out_of_bounds:
-                            property_phase_impact["sap"] = _adjust_ventilation_sap(proposed_sap_impact)
-                        else:
-                            property_phase_impact["sap"] = proposed_sap_impact
-
-                        # Update the current phase values
-                        current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
-
-                if rec["type"] == "loft_insulation":
-                    # When we have a loft insulation recommendation, where there is an extension and the existing
-                    # amount of loft insulation is already good, we limit the SAP points
-                    # By limiting here, we don't change the value in current_phase_values. This means that the
-                    # future recommendations won't have an impact that is too large
-                    li_sap_limit = RoofRecommendations.get_loft_insulation_sap_limit(
-                        property_instance.data["roof-energy-eff"], property_instance.roof["insulation_thickness"]
-                    )
-                    if li_sap_limit is not None:
-                        property_phase_impact["sap"] = min(property_phase_impact["sap"], li_sap_limit)
-                        # Update the current phase values
-                        current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
-
-                if rec["type"] == "solar_pv":
-                    # We use the SAP points in the recommendation as a minimum
-                    property_phase_impact["sap"] = (
-                        rec["sap_points"] if property_phase_impact["sap"] < rec["sap_points"] else
-                        property_phase_impact["sap"]
-                    )
-                    # Update the current phase values
-                    current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+                property_phase_impact, current_phase_values, adjustments = cls._apply_measure_specific_rules(
+                    rec=rec,
+                    property_phase_impact=property_phase_impact,
+                    previous_phase_values=previous_phase_values,
+                    current_phase_values=current_phase_values,
+                    adjustments=adjustments,
+                    property_instance=property_instance
+                )
 
                 # Insert this information into the recommendation.
                 if not rec.get("survey", False):
@@ -740,11 +955,7 @@ class Recommendations:
                 rec["co2_equivalent_savings"] = property_phase_impact["carbon"]
                 rec["heat_demand"] = property_phase_impact["heat_demand"]
 
-                if (
-                    (rec["sap_points"] is None) and (rec["co2_equivalent_savings"] is None) or
-                    (rec["heat_demand"] is None)
-                ):
-                    raise ValueError("sap points, co2 or heat demand is missing")
+                cls._validate_recommendation_updates(rec)
 
                 impact_summary.append(
                     {
@@ -757,6 +968,9 @@ class Recommendations:
                     }
                 )
 
+        if debug:
+            return property_recommendations, impact_summary, adjustments
+
         return property_recommendations, impact_summary
 
     @staticmethod

recommendations/optimiser/funding_optimiser.py

@@ -711,9 +711,12 @@ def optimise_with_scenarios(
             if kept:
                 remaining_measures.append(kept)
 
+        remaining_budget = budget - fabric_cost if budget is not None else None
+        remaining_budget = 0 if remaining_budget < 0 else remaining_budget
+
         picked_extra, extra_cost, extra_gain = run_optimizer(
             remaining_measures,
-            budget=budget - fabric_cost if budget is not None else None,
+            budget=remaining_budget,
             sub_target_gain=(
                 target_gain - fabric_gain
                 if target_gain is not None
@@ -769,6 +772,12 @@ def optimise_with_scenarios(
 
         fixed_cost, fixed_gain = sum_cost_gain(fixed_items)
 
+        if budget is not None:
+            # If we have a budget, we cannot exceed it via our fixed cost. If we do,
+            # this is not a viable solution
+            if fixed_cost > budget:
+                continue
+
         # Remaining measures (all other groups)
         remaining_measures = [
             grp for gi, grp in enumerate(optimisation_measures)

recommendations/tests/test_fireplace_recommendations.py

@@ -1,11 +1,28 @@
+import pytest
+import datetime
 from backend.Property import Property
 from recommendations.FireplaceRecommendations import FireplaceRecommendations
 from etl.epc.Record import EPCRecord
 
 
+@pytest.fixture
+def fireplace_materials():
+    return [
+        {'id': 3591, 'type': 'sealing_fireplace', 'description': 'Sealing of an open fireplace', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'Warm Front',
+         'created_at': datetime.datetime(2025, 8, 15, 16, 31, 52, 995292), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0,
+         'plant_cost': 0.0, 'total_cost': 185.0, 'notes': None, 'is_installer_quote': True, 'innovation_rate': 0.0,
+         'size': None, 'size_unit': None, 'includes_scaffolding': False, 'includes_battery': False,
+         'battery_size': None}
+    ]
+
+
 class TestFirepaceRecommendations:
 
-    def test_no_fireplaces(self):
+    def test_no_fireplaces(self, fireplace_materials):
         epc_record = EPCRecord()
         epc_record.prepared_epc = {
             "number-open-fireplaces": 0,
@@ -13,9 +30,7 @@ class TestFirepaceRecommendations:
 
         property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
 
-        recommender = FireplaceRecommendations(
-            property_instance=property_instance
-        )
+        recommender = FireplaceRecommendations(property_instance=property_instance, materials=fireplace_materials)
 
         assert recommender.recommendation is None
 
@@ -23,16 +38,15 @@ class TestFirepaceRecommendations:
 
         assert recommender.recommendation is None
 
-    def test_one_fireplace(self):
+    def test_one_fireplace(self, fireplace_materials):
         epc_record = EPCRecord()
         epc_record.prepared_epc = {
             "number-open-fireplaces": 1,
         }
         property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance.already_installed = []
 
-        recommender = FireplaceRecommendations(
-            property_instance=property_instance
-        )
+        recommender = FireplaceRecommendations(property_instance=property_instance, materials=fireplace_materials)
 
         assert recommender.recommendation is None
 
@@ -40,18 +54,17 @@ class TestFirepaceRecommendations:
 
         assert recommender.recommendation
         assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
-        assert recommender.recommendation[0]["total"] == 235
+        assert recommender.recommendation[0]["total"] == 185
 
-    def test_multiple_fireplaces(self):
+    def test_multiple_fireplaces(self, fireplace_materials):
         epc_record = EPCRecord()
         epc_record.prepared_epc = {
             "number-open-fireplaces": 3,
         }
         property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance.already_installed = []
 
-        recommender = FireplaceRecommendations(
-            property_instance=property_instance
-        )
+        recommender = FireplaceRecommendations(property_instance=property_instance, materials=fireplace_materials)
 
         assert recommender.recommendation is None
 
@@ -59,4 +72,4 @@ class TestFirepaceRecommendations:
 
         assert recommender.recommendation
         assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
-        assert recommender.recommendation[0]["total"] == 235 * 3
+        assert recommender.recommendation[0]["total"] == 185 * 3

recommendations/tests/test_window_recommendations.py

@@ -1,5 +1,6 @@
 import pytest
 import pickle
+import numpy as np
 from recommendations.WindowsRecommendations import WindowsRecommendations
 from backend.Property import Property
 from recommendations.tests.test_data.materials import materials
@@ -44,11 +45,13 @@ class TestWindowRecommendations:
             epc_record=epc_record
         )
         property_1.windows = {
-            'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': 'full',
+            'original_description': 'Single glazed', 'clean_description': 'Single glazed',
+            'has_glazing': False, 'glazing_coverage': 'full',
             'glazing_type': 'single',
             'no_data': False
         }
         property_1.number_of_windows = 7
+        property_1.already_installed = []
 
         recommender = WindowsRecommendations(property_instance=property_1, materials=materials)
 
@@ -58,25 +61,16 @@ class TestWindowRecommendations:
 
         # The home is going from single glazing (v poor energy eff) -> double glazing (average energy eff)
 
-        assert recommender.recommendation == [
-            {
-                'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing",
-                'description': 'Install double glazing to all windows',
-                'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False,
-                'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False,
-                'description_simulation': {
-                    'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good',
-                    'windows-description': 'Fully double glazed',
-                    'glazed-type': 'double glazing installed during or after 2002'
-                },
-                'simulation_config': {
-                    'has_glazing_ending': True, 'glazing_type_ending': 'double',
-                    'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good',
-                    'glazed_type_ending': 'double glazing installed during or after 2002'
-                },
-                "survey": None
-            }
-        ]
+        assert len(recommender.recommendation) == 1
+        assert recommender.recommendation[0]["total"] == np.float64(7980.0)
+        assert recommender.recommendation[0]["phase"] == 0
+        assert recommender.recommendation[0]["description"] == 'Install double glazing to all windows'
+        assert recommender.recommendation[0]["contingency"] == np.float64(1197.0)
+        assert recommender.recommendation[0]["simulation_config"] == {
+            'has_glazing_ending': True, 'glazing_type_ending': 'double',
+            'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good',
+            'glazed_type_ending': 'double glazing installed during or after 2002'
+        }
 
     def test_partial_double_glazed(self):
         """
@@ -97,10 +91,15 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_2.windows = {'original_description': 'Mostly double glazing', 'has_glazing': True,
-                              'glazing_coverage': 'most',
-                              'glazing_type': 'double', 'no_data': False}
+        property_2.windows = {
+            'original_description': 'Mostly double glazing',
+            'clean_description': 'Mostly double glazing',
+            'has_glazing': True,
+            'glazing_coverage': 'most',
+            'glazing_type': 'double', 'no_data': False
+        }
         property_2.number_of_windows = 7
+        property_2.already_installed = []
 
         recommender2 = WindowsRecommendations(property_instance=property_2, materials=materials)
 
@@ -108,26 +107,16 @@ class TestWindowRecommendations:
 
         recommender2.recommend()
 
-        assert recommender2.recommendation == [
-            {
-                'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing",
-                'description': 'Install double glazing to the remaining windows', 'starting_u_value': None,
-                'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0,
-                'labour_hours': 0.0,
-                'labour_days': 0.0, 'is_secondary_glazing': False,
-                'description_simulation': {
-                    'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good',
-                    'windows-description': 'Fully double glazed',
-                    'glazed-type': 'double glazing installed during or after 2002'
-                },
-                'simulation_config': {
-                    'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100,
-                    'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'double',
-                    'glazed_type_ending': 'double glazing installed during or after 2002'
-                },
-                "survey": None
-            }
-        ]
+        assert len(recommender2.recommendation) == 1
+        assert recommender2.recommendation[0]["total"] == np.float64(5700.0)
+        assert recommender2.recommendation[0]["phase"] == 0
+        assert recommender2.recommendation[0]["description"] == 'Install double glazing to all windows'
+        assert recommender2.recommendation[0]["contingency"] == np.float64(855.0)
+        assert recommender2.recommendation[0]["simulation_config"] == {
+            'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100,
+            'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'double',
+            'glazed_type_ending': 'double glazing installed during or after 2002'
+        }
 
     def test_fully_double_glazed(self):
         """
@@ -146,10 +135,14 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_3.windows = {'original_description': 'Fully double glazed', 'has_glazing': True,
-                              'glazing_coverage': 'full',
-                              'glazing_type': 'double', 'no_data': False}
+        property_3.windows = {
+            'original_description': 'Fully double glazed', 'clean_description': 'Fully double glazed',
+            'has_glazing': True,
+            'glazing_coverage': 'full',
+            'glazing_type': 'double', 'no_data': False
+        }
         property_3.number_of_windows = 7
+        property_3.already_installed = []
 
         recommender3 = WindowsRecommendations(property_instance=property_3, materials=materials)
 
@@ -172,10 +165,15 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_4.windows = {'original_description': 'Full secondary glazing', 'has_glazing': True,
-                              'glazing_coverage': 'full',
-                              'glazing_type': 'secondary', 'no_data': False}
+        property_4.windows = {
+            'original_description': 'Full secondary glazing',
+            'clean_description': 'Full secondary glazing',
+            'has_glazing': True,
+            'glazing_coverage': 'full',
+            'glazing_type': 'secondary', 'no_data': False
+        }
         property_4.number_of_windows = 7
+        property_4.already_installed = []
 
         recommender4 = WindowsRecommendations(property_instance=property_4, materials=materials)
 
@@ -199,10 +197,15 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_5.windows = {'original_description': 'Partial secondary glazing', 'has_glazing': True,
-                              'glazing_coverage': 'partial',
-                              'glazing_type': 'secondary', 'no_data': False}
+        property_5.windows = {
+            'original_description': 'Partial secondary glazing',
+            'clean_description': 'Partial secondary glazing',
+            'has_glazing': True,
+            'glazing_coverage': 'partial',
+            'glazing_type': 'secondary', 'no_data': False
+        }
         property_5.number_of_windows = 7
+        property_5.already_installed = []
 
         recommender5 = WindowsRecommendations(property_instance=property_5, materials=materials)
 
@@ -210,25 +213,15 @@ class TestWindowRecommendations:
 
         recommender5.recommend()
 
-        assert recommender5.recommendation == [
-            {
-                'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing',
-                'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None,
-                'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0,
-                'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True,
-                'description_simulation': {
-                    'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good',
-                    'windows-description': 'Full secondary glazing',
-                    'glazed-type': 'secondary glazing'
-                },
-                'simulation_config': {
-                    'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100,
-                    'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'secondary',
-                    'glazed_type_ending': 'secondary glazing'
-                },
-                "survey": None
-            }
-        ]
+        assert len(recommender5.recommendation) == 1
+        assert recommender5.recommendation[0]["total"] == np.float64(4560.0)
+        assert recommender5.recommendation[0]["phase"] == 0
+        assert recommender5.recommendation[0]["description"] == 'Install double glazing to all windows'
+        assert recommender5.recommendation[0]["contingency"] == np.float64(684.0)
+        assert recommender5.recommendation[0]["simulation_config"] == {
+            'glazing_coverage_ending': 'full', 'glazing_type_ending': 'multiple', 'multi_glaze_proportion_ending': 100,
+            'windows_energy_eff_ending': 'Average', 'glazed_type_ending': 'secondary glazing'
+        }
 
     def test_single_glazed_restricted_measures(self):
         epc_record = EPCRecord()
@@ -245,12 +238,16 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_6.windows = {'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': None,
-                              'glazing_type': 'single',
-                              'no_data': False}
+        property_6.windows = {
+            'original_description': 'Single glazed', 'clean_description': 'Single glazed',
+            'has_glazing': False, 'glazing_coverage': None,
+            'glazing_type': 'single',
+            'no_data': False
+        }
         property_6.number_of_windows = 7
         property_6.restricted_measures = True
         property_6.is_heritage = True
+        property_6.already_installed = []
 
         recommender6 = WindowsRecommendations(property_instance=property_6, materials=materials)
 
@@ -258,26 +255,18 @@ class TestWindowRecommendations:
 
         recommender6.recommend()
 
-        assert recommender6.recommendation == [
-            {
-                'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing',
-                'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to '
-                               'herigate building status',
-                'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False,
-                'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True,
-                'description_simulation': {
-                    'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good',
-                    'windows-description': 'Full secondary glazing',
-                    'glazed-type': 'secondary glazing'
-                },
-                'simulation_config': {
-                    'has_glazing_ending': True, 'glazing_coverage_ending': 'full',
-                    'glazing_type_ending': 'secondary', 'multi_glaze_proportion_ending': 100,
-                    'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'secondary glazing'
-                },
-                "survey": None
-            },
-        ]
+        assert len(recommender6.recommendation) == 1
+        assert recommender6.recommendation[0]["total"] == np.float64(7980.0)
+        assert recommender6.recommendation[0]["phase"] == 0
+        assert recommender6.recommendation[0]["contingency"] == np.float64(1197.0)
+        assert recommender6.recommendation[0]["description"] == (
+            'Install secondary glazing to all windows. Secondary glazing recommended due to herigate building status'
+        )
+        assert recommender6.recommendation[0]["simulation_config"] == {
+            'has_glazing_ending': True, 'glazing_coverage_ending': 'full', 'glazing_type_ending': 'secondary',
+            'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good',
+            'glazed_type_ending': 'secondary glazing'
+        }
 
     def test_full_triple_glazed(self):
         epc_record = EPCRecord()
@@ -292,10 +281,14 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_7.windows = {'original_description': 'Fully triple glazed', 'has_glazing': True,
-                              'glazing_coverage': 'full',
-                              'glazing_type': 'triple', 'no_data': False}
+        property_7.windows = {
+            'original_description': 'Fully triple glazed', 'clean_description': 'Fully triple glazed',
+            'has_glazing': True,
+            'glazing_coverage': 'full',
+            'glazing_type': 'triple', 'no_data': False
+        }
         property_7.number_of_windows = 7
+        property_7.already_installed = []
 
         recommender7 = WindowsRecommendations(property_instance=property_7, materials=materials)
 
@@ -321,10 +314,15 @@ class TestWindowRecommendations:
             address='1',
             epc_record=epc_record
         )
-        property_8.windows = {'original_description': 'Mostly triple glazing', 'has_glazing': True,
-                              'glazing_coverage': 'most',
-                              'glazing_type': 'triple', 'no_data': False}
+        property_8.windows = {
+            'original_description': 'Mostly triple glazing',
+            'clean_description': 'Mostly triple glazing',
+            'has_glazing': True,
+            'glazing_coverage': 'most',
+            'glazing_type': 'triple', 'no_data': False
+        }
         property_8.number_of_windows = 7
+        property_8.already_installed = []
 
         recommender8 = WindowsRecommendations(property_instance=property_8, materials=materials)
 
@@ -394,7 +392,9 @@ class TestWindowRecommendations:
             epc_record=epc_record
         )
         property_9.windows = {
-            'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': None,
+            'original_description': 'Single glazed',
+            'clean_description': 'Single glazed',
+            'has_glazing': False, 'glazing_coverage': None,
             'glazing_type': 'single',
             'no_data': False
         }
@@ -403,6 +403,7 @@ class TestWindowRecommendations:
         property_9.number_of_windows = 7
         property_9.restricted_measures = False
         property_9.is_heritage = False
+        property_9.already_installed = []
 
         recommender9 = WindowsRecommendations(property_instance=property_9, materials=materials)
 
@@ -410,26 +411,10 @@ class TestWindowRecommendations:
 
         recommender9.recommend()
 
-        assert recommender9.recommendation == [
-            {
-                'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'double_glazing',
-                'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None,
-                'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0,
-                'labour_days': 0.0, 'is_secondary_glazing': False,
-                'description_simulation': {
-                    'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good',
-                    'windows-description': 'Fully double glazed',
-                    'glazed-type': 'double glazing installed during or after 2002'
-                },
-                'simulation_config': {
-                    'has_glazing_ending': True, 'glazing_coverage_ending': 'full',
-                    'glazing_type_ending': 'double', 'multi_glaze_proportion_ending': 100,
-                    'windows_energy_eff_ending': 'Good',
-                    'glazed_type_ending': 'double glazing installed during or after 2002'
-                },
-                "survey": None
-            }
-        ]
+        assert recommender9.recommendation[0]["total"] == np.float64(7980.0)
+        assert recommender9.recommendation[0]["phase"] == 0
+        assert recommender9.recommendation[0]["description"] == 'Install double glazing to all windows'
+        assert recommender9.recommendation[0]["contingency"] == np.float64(1197.0)
 
         # We now simulate the outcome
         windows_rec = recommender9.recommendation.copy()
@@ -537,8 +522,10 @@ class TestWindowRecommendations:
             'mainheatc_energy_eff_ending': 'Average', 'lighting_energy_eff_starting': 'Very Good',
             'lighting_energy_eff_ending': 'Very Good', 'number_habitable_rooms_starting': 4.0,
             'number_habitable_rooms_ending': 4.0, 'number_heated_rooms_starting': 4.0,
-            'number_heated_rooms_ending': 4.0, 'days_to_starting': 3642, 'days_to_ending': 3642,
-            'estimated_perimeter_starting': 23.430749027719962, 'estimated_perimeter_ending': 23.430749027719962
+            'number_heated_rooms_ending': 4.0, 'is_post_sap10_starting': False, 'is_post_sap10_ending': False,
+            'lodgement_date_starting': '2024-07-21', 'lodgement_date_ending': '2024-07-21', 'days_to_starting': 3642,
+            'days_to_ending': 3642, 'estimated_perimeter_starting': 23.430749027719962,
+            'estimated_perimeter_ending': 23.430749027719962
         }
 
         assert starting_record == expected_base_difference_record
@@ -553,104 +540,168 @@ class TestWindowRecommendations:
         assert len(simulated_data) == 1
 
         expected_simulated_outcome = {
-            'uprn': 200001041444, 'rdsap_change': 0, 'heat_demand_change': 0, 'carbon_change': 0.0,
+            'uprn': 200001041444, 'rdsap_change': 0, 'heat_demand_change': 0,
+            'carbon_change': 0.0,
             'potential_energy_efficiency': 82.0, 'environment_impact_potential': 79.0,
-            'energy_consumption_potential': 155.0, 'co2_emissions_potential': 1.7, 'property_type': 'House',
-            'built_form': 'Semi-Detached', 'constituency': 'E14000909', 'number_habitable_rooms': 4.0,
-            'number_heated_rooms': 4.0, 'construction_age_band': 'England and Wales: before 1900',
+            'energy_consumption_potential': 155.0, 'co2_emissions_potential': 1.7,
+            'property_type': 'House',
+            'built_form': 'Semi-Detached', 'constituency': 'E14000909',
+            'number_habitable_rooms': 4.0,
+            'number_heated_rooms': 4.0,
+            'construction_age_band': 'England and Wales: before 1900',
             'fixed_lighting_outlets_count': 7.0, 'walls_thermal_transmittance': 1.7,
-            'walls_thermal_transmittance_unit': 'Unknown', 'is_cavity_wall': False, 'is_filled_cavity': False,
+            'walls_thermal_transmittance_unit': 'Unknown', 'is_cavity_wall': False,
+            'is_filled_cavity': False,
             'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False,
-            'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'walls_is_assumed': True,
-            'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none',
-            'external_insulation': False, 'internal_insulation': False, 'floor_thermal_transmittance': 0.96,
-            'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True,
-            'another_property_below': False, 'floor_insulation_thickness': 'none', 'roof_thermal_transmittance': 2.3,
-            'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
-            'is_at_rafters': False, 'has_dwelling_above': False, 'roof_insulation_thickness': 'none',
-            'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown',
-            'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown',
-            'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown',
-            'no_system_present': 'Unknown', 'appliance': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False,
-            'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
-            'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False,
-            'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False,
+            'is_granite_or_whinstone': False,
+            'is_as_built': True, 'is_cob': False, 'walls_is_assumed': True,
+            'is_sandstone_or_limestone': False,
+            'is_park_home': False, 'walls_insulation_thickness': 'none',
+            'external_insulation': False,
+            'internal_insulation': False, 'floor_thermal_transmittance': 0.96,
+            'is_to_unheated_space': False,
+            'is_to_external_air': False, 'is_suspended': False, 'is_solid': True,
+            'another_property_below': False,
+            'floor_insulation_thickness': 'none', 'roof_thermal_transmittance': 2.3,
+            'is_pitched': True,
+            'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
+            'is_at_rafters': False,
+            'has_dwelling_above': False, 'roof_insulation_thickness': 'none',
+            'heater_type': 'Unknown',
+            'system_type': 'from main system', 'thermostat_characteristics': 'Unknown',
+            'heating_scope': 'Unknown',
+            'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown',
+            'extra_features': 'Unknown',
+            'chp_systems': 'Unknown', 'distribution_system': 'Unknown',
+            'no_system_present': 'Unknown',
+            'appliance': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False,
+            'has_pipes_in_screed_above_insulation': False,
+            'has_pipes_in_insulated_timber_floor': False,
+            'has_pipes_in_concrete_slab': False, 'has_boiler': True,
+            'has_air_source_heat_pump': False,
+            'has_room_heaters': False, 'has_electric_storage_heaters': False,
+            'has_warm_air': False,
             'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
-            'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+            'has_community_scheme': False, 'has_ground_source_heat_pump': False,
+            'has_no_system_present': False,
             'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
-            'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
-            'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False,
-            'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
-            'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
-            'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
-            'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
-            'thermostatic_control': 'room thermostat', 'charging_system': 'Unknown', 'switch_system': 'programmer',
+            'has_electric_heat_pump': False,
+            'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
+            'has_exhaust_source_heat_pump': False,
+            'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': True,
+            'has_wood_logs': False,
+            'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
+            'has_anthracite': False,
+            'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False,
+            'has_lpg': False, 'has_b30k': False,
+            'has_electricaire': False, 'has_assumed_for_most_rooms': False,
+            'has_underfloor_heating': False,
+            'thermostatic_control': 'room thermostat', 'charging_system': 'Unknown',
+            'switch_system': 'programmer',
             'no_control': 'Unknown', 'dhw_control': 'Unknown', 'community_heating': 'Unknown',
-            'multiple_room_thermostats': False, 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown',
+            'multiple_room_thermostats': False, 'auxiliary_systems': 'Unknown',
+            'trvs': 'Unknown',
             'rate_control': 'Unknown', 'glazing_type': 'single', 'fuel_type': 'mains gas',
             'main-fuel_tariff_type': 'Unknown', 'is_community': False,
-            'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
-            'walls_thermal_transmittance_ending': 1.7, 'walls_thermal_transmittance_unit_ending': 'Unknown',
-            'is_filled_cavity_ending': False, 'is_as_built_ending': True, 'walls_is_assumed_ending': True,
+            'no_individual_heating_or_community_network': False,
+            'complex_fuel_type': 'Unknown',
+            'walls_thermal_transmittance_ending': 1.7,
+            'walls_thermal_transmittance_unit_ending': 'Unknown',
+            'is_filled_cavity_ending': False, 'is_as_built_ending': True,
+            'walls_is_assumed_ending': True,
             'is_park_home_ending': False, 'walls_insulation_thickness_ending': 'none',
             'external_insulation_ending': False, 'internal_insulation_ending': False,
-            'floor_thermal_transmittance_ending': 0.96, 'floor_insulation_thickness_ending': 'none',
+            'floor_thermal_transmittance_ending': 0.96,
+            'floor_insulation_thickness_ending': 'none',
             'roof_thermal_transmittance_ending': 2.3, 'is_at_rafters_ending': False,
             'roof_insulation_thickness_ending': 'none', 'heater_type_ending': 'Unknown',
-            'system_type_ending': 'from main system', 'thermostat_characteristics_ending': 'Unknown',
+            'system_type_ending': 'from main system',
+            'thermostat_characteristics_ending': 'Unknown',
             'heating_scope_ending': 'Unknown', 'energy_recovery_ending': 'Unknown',
             'hotwater_tariff_type_ending': 'Unknown', 'extra_features_ending': 'Unknown',
-            'chp_systems_ending': 'Unknown', 'distribution_system_ending': 'Unknown',
-            'no_system_present_ending': 'Unknown', 'appliance_ending': 'Unknown', 'has_radiators_ending': True,
-            'has_fan_coil_units_ending': False, 'has_pipes_in_screed_above_insulation_ending': False,
-            'has_pipes_in_insulated_timber_floor_ending': False, 'has_pipes_in_concrete_slab_ending': False,
-            'has_boiler_ending': True, 'has_air_source_heat_pump_ending': False, 'has_room_heaters_ending': False,
+            'chp_systems_ending': 'Unknown',
+            'distribution_system_ending': 'Unknown', 'no_system_present_ending': 'Unknown',
+            'appliance_ending': 'Unknown',
+            'has_radiators_ending': True, 'has_fan_coil_units_ending': False,
+            'has_pipes_in_screed_above_insulation_ending': False,
+            'has_pipes_in_insulated_timber_floor_ending': False,
+            'has_pipes_in_concrete_slab_ending': False, 'has_boiler_ending': True,
+            'has_air_source_heat_pump_ending': False, 'has_room_heaters_ending': False,
             'has_electric_storage_heaters_ending': False, 'has_warm_air_ending': False,
-            'has_electric_underfloor_heating_ending': False, 'has_electric_ceiling_heating_ending': False,
+            'has_electric_underfloor_heating_ending': False,
+            'has_electric_ceiling_heating_ending': False,
             'has_community_scheme_ending': False, 'has_ground_source_heat_pump_ending': False,
-            'has_no_system_present_ending': False, 'has_portable_electric_heaters_ending': False,
-            'has_water_source_heat_pump_ending': False, 'has_electric_heat_pump_ending': False,
-            'has_micro-cogeneration_ending': False, 'has_solar_assisted_heat_pump_ending': False,
-            'has_exhaust_source_heat_pump_ending': False, 'has_community_heat_pump_ending': False,
-            'has_electric_ending': False, 'has_mains_gas_ending': True, 'has_wood_logs_ending': False,
-            'has_coal_ending': False, 'has_oil_ending': False, 'has_wood_pellets_ending': False,
+            'has_no_system_present_ending': False,
+            'has_portable_electric_heaters_ending': False,
+            'has_water_source_heat_pump_ending': False,
+            'has_electric_heat_pump_ending': False,
+            'has_micro-cogeneration_ending': False,
+            'has_solar_assisted_heat_pump_ending': False,
+            'has_exhaust_source_heat_pump_ending': False,
+            'has_community_heat_pump_ending': False,
+            'has_electric_ending': False, 'has_mains_gas_ending': True,
+            'has_wood_logs_ending': False,
+            'has_coal_ending': False, 'has_oil_ending': False,
+            'has_wood_pellets_ending': False,
             'has_anthracite_ending': False, 'has_dual_fuel_mineral_and_wood_ending': False,
-            'has_smokeless_fuel_ending': False, 'has_lpg_ending': False, 'has_b30k_ending': False,
+            'has_smokeless_fuel_ending': False, 'has_lpg_ending': False,
+            'has_b30k_ending': False,
             'has_electricaire_ending': False, 'has_assumed_for_most_rooms_ending': False,
-            'has_underfloor_heating_ending': False, 'thermostatic_control_ending': 'room thermostat',
-            'charging_system_ending': 'Unknown', 'switch_system_ending': 'programmer', 'no_control_ending': 'Unknown',
+            'has_underfloor_heating_ending': False,
+            'thermostatic_control_ending': 'room thermostat',
+            'charging_system_ending': 'Unknown', 'switch_system_ending': 'programmer',
+            'no_control_ending': 'Unknown',
             'dhw_control_ending': 'Unknown', 'community_heating_ending': 'Unknown',
-            'multiple_room_thermostats_ending': False, 'auxiliary_systems_ending': 'Unknown', 'trvs_ending': 'Unknown',
-            'rate_control_ending': 'Unknown', 'glazing_type_ending': 'double', 'fuel_type_ending': 'mains gas',
+            'multiple_room_thermostats_ending': False, 'auxiliary_systems_ending': 'Unknown',
+            'trvs_ending': 'Unknown',
+            'rate_control_ending': 'Unknown', 'glazing_type_ending': 'double',
+            'fuel_type_ending': 'mains gas',
             'main-fuel_tariff_type_ending': 'Unknown', 'is_community_ending': False,
-            'no_individual_heating_or_community_network_ending': False, 'complex_fuel_type_ending': 'Unknown',
-            'sap_starting': 47, 'sap_ending': 47, 'heat_demand_starting': 478, 'heat_demand_ending': 478,
-            'carbon_starting': 5.1, 'carbon_ending': 5.1, 'lighting_cost_starting': 91.0, 'lighting_cost_ending': 91.0,
-            'heating_cost_starting': 1677.0, 'heating_cost_ending': 1677.0, 'hot_water_cost_starting': 161.0,
+            'no_individual_heating_or_community_network_ending': False,
+            'complex_fuel_type_ending': 'Unknown',
+            'sap_starting': 47, 'sap_ending': 47, 'heat_demand_starting': 478,
+            'heat_demand_ending': 478,
+            'carbon_starting': 5.1, 'carbon_ending': 5.1, 'lighting_cost_starting': 91.0,
+            'lighting_cost_ending': 91.0,
+            'heating_cost_starting': 1677.0, 'heating_cost_ending': 1677.0,
+            'hot_water_cost_starting': 161.0,
             'hot_water_cost_ending': 161.0, 'mechanical_ventilation_starting': 'natural',
-            'mechanical_ventilation_ending': 'natural', 'secondheat_description_starting': 'None',
+            'mechanical_ventilation_ending': 'natural',
+            'secondheat_description_starting': 'None',
             'secondheat_description_ending': 'None', 'glazed_type_starting': 'not defined',
             'glazed_type_ending': 'double glazing installed during or after 2002',
-            'multi_glaze_proportion_starting': 0.0, 'multi_glaze_proportion_ending': 100,
-            'low_energy_lighting_starting': 100.0, 'low_energy_lighting_ending': 100.0,
-            'number_open_fireplaces_starting': 0.0, 'number_open_fireplaces_ending': 0.0,
-            'solar_water_heating_flag_starting': 'N', 'solar_water_heating_flag_ending': 'N',
-            'photo_supply_starting': 0.0, 'photo_supply_ending': 0.0, 'transaction_type_starting': 'rental',
-            'transaction_type_ending': 'rental', 'energy_tariff_starting': 'dual', 'energy_tariff_ending': 'dual',
-            'extension_count_starting': 3.0, 'extension_count_ending': 3.0, 'total_floor_area_starting': 61.0,
-            'total_floor_area_ending': 61.0, 'floor_height_starting': 2.37, 'floor_height_ending': 2.37,
-            'hot_water_energy_eff_starting': 'Good', 'hot_water_energy_eff_ending': 'Good',
+            'multi_glaze_proportion_starting': 0.0,
+            'multi_glaze_proportion_ending': 100, 'low_energy_lighting_starting': 100.0,
+            'low_energy_lighting_ending': 100.0, 'number_open_fireplaces_starting': 0.0,
+            'number_open_fireplaces_ending': 0.0, 'solar_water_heating_flag_starting': 'N',
+            'solar_water_heating_flag_ending': 'N', 'photo_supply_starting': 0.0,
+            'photo_supply_ending': 0.0,
+            'transaction_type_starting': 'rental', 'transaction_type_ending': 'rental',
+            'energy_tariff_starting': 'dual',
+            'energy_tariff_ending': 'dual', 'extension_count_starting': 3.0,
+            'extension_count_ending': 3.0,
+            'total_floor_area_starting': 61.0, 'total_floor_area_ending': 61.0,
+            'floor_height_starting': 2.37,
+            'floor_height_ending': 2.37, 'hot_water_energy_eff_starting': 'Good',
+            'hot_water_energy_eff_ending': 'Good',
             'floor_energy_eff_starting': 'NO_RATING', 'floor_energy_eff_ending': 'NO_RATING',
             'windows_energy_eff_starting': 'Very Poor', 'windows_energy_eff_ending': 'Good',
             'walls_energy_eff_starting': 'Very Poor', 'walls_energy_eff_ending': 'Very Poor',
-            'sheating_energy_eff_starting': 'NO_RATING', 'sheating_energy_eff_ending': 'NO_RATING',
+            'sheating_energy_eff_starting': 'NO_RATING',
+            'sheating_energy_eff_ending': 'NO_RATING',
             'roof_energy_eff_starting': 'Very Poor', 'roof_energy_eff_ending': 'Very Poor',
             'mainheat_energy_eff_starting': 'Good', 'mainheat_energy_eff_ending': 'Good',
-            'mainheatc_energy_eff_starting': 'Average', 'mainheatc_energy_eff_ending': 'Average',
-            'lighting_energy_eff_starting': 'Very Good', 'lighting_energy_eff_ending': 'Very Good',
+            'mainheatc_energy_eff_starting': 'Average',
+            'mainheatc_energy_eff_ending': 'Average',
+            'lighting_energy_eff_starting': 'Very Good',
+            'lighting_energy_eff_ending': 'Very Good',
             'number_habitable_rooms_starting': 4.0, 'number_habitable_rooms_ending': 4.0,
-            'number_heated_rooms_starting': 4.0, 'number_heated_rooms_ending': 4.0, 'days_to_starting': 3642,
-            'days_to_ending': 3713, 'estimated_perimeter_starting': 23.430749027719962,
+            'number_heated_rooms_starting': 4.0, 'number_heated_rooms_ending': 4.0,
+            'is_post_sap10_starting': False,
+            'is_post_sap10_ending': False, 'lodgement_date_starting': '2024-07-21',
+            'lodgement_date_ending': '2024-07-21',
+            'days_to_starting': 3642, 'days_to_ending': 4189,
+            'estimated_perimeter_starting': 23.430749027719962,
             'estimated_perimeter_ending': 23.430749027719962, 'has_glazing_ending': True,
             'glazing_coverage_ending': 'full', 'id': '1+1'
         }

sfr/principal_pitch/2_export_data.py

@@ -11,6 +11,7 @@ from backend.app.db.models.recommendations import Recommendation, Plan, PlanReco
 from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel, PropertyDetailsSpatial
 from backend.app.db.functions.materials_functions import get_materials
 from collections import defaultdict
+from sqlalchemy import func
 
 # PORTFOLIO_ID = 206
 # SCENARIOS = [389]
@@ -57,9 +58,44 @@ def get_data(portfolio_id, scenario_ids):
     # --------------------
     # Plans
     # --------------------
-    plans_query = session.query(Plan).filter(
-        Plan.scenario_id.in_(scenario_ids)
-    ).all()
+    latest_plans_subq = (
+        session.query(
+            Plan.scenario_id,
+            Plan.property_id,
+            func.max(Plan.created_at).label("latest_created_at")
+        )
+        .filter(Plan.scenario_id.in_(scenario_ids))
+        .group_by(
+            Plan.scenario_id,
+            Plan.property_id
+        )
+        .subquery()
+    )
+
+    # plans_query = session.query(Plan).filter(
+    #     Plan.scenario_id.in_(scenario_ids)
+    # ).all()
+
+    plans_query = (
+        session.query(Plan)
+        .join(
+            latest_plans_subq,
+            (Plan.scenario_id == latest_plans_subq.c.scenario_id) &
+            (Plan.property_id == latest_plans_subq.c.property_id) &
+            (Plan.created_at == latest_plans_subq.c.latest_created_at)
+        )
+        .all()
+    )
+
+    # plans_query = (
+    #     session.query(Plan)
+    #     .join(
+    #         latest_plans_subq,
+    #         (Plan.scenario_id == latest_plans_subq.c.scenario_id) &
+    #         (Plan.created_at == latest_plans_subq.c.latest_created_at)
+    #     )
+    #     .all()
+    # )
 
     plans_data = [
         {col.name: getattr(plan, col.name) for col in Plan.__table__.columns}
@@ -73,7 +109,8 @@ def get_data(portfolio_id, scenario_ids):
     # --------------------
     recommendations_query = session.query(
         Recommendation,
-        Plan.scenario_id
+        Plan.scenario_id,
+        PlanRecommendations.plan_id
     ).join(
         PlanRecommendations,
         Recommendation.id == PlanRecommendations.recommendation_id
@@ -216,6 +253,7 @@ for scenario_id in SCENARIOS:
         [
             "landlord_property_id", "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof",
             "heating", "windows", "current_epc_rating", "current_sap_points", "total_floor_area", "number_of_rooms",
+            "id"
         ]
     ].merge(
         recommendations_measures_pivot, how="left", on="property_id"
@@ -223,17 +261,42 @@ for scenario_id in SCENARIOS:
         post_install_sap, how="left", on="property_id"
     )
 
-    df = df.drop(columns=["property_id"])
+    # df = df.drop(columns=["property_id"])
     df["sap_points"] = df["sap_points"].fillna(0)
 
     df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
-    df["predicted_post_works_sap"] = df["predicted_post_works_sap"].round()
+    df["predicted_post_works_sap"] = df["predicted_post_works_sap"]
     df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(lambda x: sap_to_epc(x))
     df["uprn"] = df["uprn"].astype(str)
 
+    relevant_plans = plans_df[plans_df["scenario_id"] == scenario_id]
+    df2 = df.merge(
+        relevant_plans[["property_id", "post_sap_points", "post_epc_rating"]], how="left", on="property_id",
+        suffixes=("", "_plan")
+    )
+    print(df2["predicted_post_works_epc"].value_counts())
+    print(df2["post_epc_rating"].value_counts())
+
+    z = df2[
+        (df2["predicted_post_works_epc"] != "D") &
+        (df2["post_epc_rating"].astype(str) == "Epc.D")
+        ]
+
+    df2["predicted_post_works_epc"].value_counts()
+    df2["post_epc_rating"].astype(str).value_counts()
+
+    df2[df2["total_retrofit_cost"] > 0].shape
+
+    getting_works = df[df["total_retrofit_cost"] > 0]
+    getting_works["predicted_post_works_epc"].value_counts()
+
+    32565 / getting_works.shape[0]
+
+    df[df["predicted_post_works_sap"] == ""]
+
     # Create excel to store to
     filename = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
-                f"Project/Final SAL/{scenario_names[scenario_id]} - 20250113 final.xlsx")
+                f"Project/Final SAL/scenarios/{scenario_names[scenario_id]} - 20250114 final.xlsx")
     with pd.ExcelWriter(filename) as writer:
         df.to_excel(writer, sheet_name="properties", index=False)
 
@@ -388,3 +451,27 @@ asset_list.to_excel(
 condition_cost_comparison = asset_list[
     ["condition_score", "decoration_sum_min ", "decoration_sum_max", "domna_condition_cost"]
 ]
+
+# Testing
+plans_df.head()
+
+example = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final "
+    "SAL/scenarios/EPC C - no solid floor, no EWI or IWI, ashp 3.0 - 20250114 final.xlsx"
+)
+
+plans_df2 = plans_df.merge(
+    properties_df[["property_id", "landlord_property_id"]],
+    left_on="property_id",
+    right_on="property_id",
+    how="left"
+)
+
+plans_df2 = plans_df2[plans_df2["scenario_id"] == 909]
+
+dupes = plans_df2[plans_df2["property_id"].duplicated()]
+
+# merge on plans
+example = example.merge(
+    plans_df, how="left",
+)

tox.ini

@@ -7,6 +7,7 @@ passenv = EPC_AUTH_TOKEN
 description = Install dependencies and run tests
 deps =
     -rbackend/engine/requirements.txt
+    -rbackend/app/requirements/requirements.txt
     -rtest.requirements.txt
-commands = pytest
+commands = pytest {posargs}