Merge pull request #294 from Hestia-Homes/immo-pilot

Immo pilot

backend/Property.py

@@ -61,7 +61,8 @@ class Property:
     n_bedrooms = None
 
     def __init__(
-        self, id, postcode, address, epc_record, already_installed=None, **kwargs
+        self, id, postcode, address, epc_record, already_installed=None, non_invasive_recommendations=None,
+        **kwargs
     ):
 
         self.epc_record = epc_record
@@ -80,6 +81,10 @@ class Property:
         # cost and instead, provide a message that the measure has already been installed
 
         self.already_installed = ast.literal_eval(already_installed['already_installed']) if already_installed else []
+        self.non_invasive_recommendations = (
+            ast.literal_eval(non_invasive_recommendations['recommendations']) if
+            non_invasive_recommendations else []
+        )
 
         self.uprn = epc_record.get("uprn")
         self.full_sap_epc = epc_record.get("full_sap_epc")
@@ -142,6 +147,8 @@ class Property:
 
         self.current_adjusted_energy = None
         self.expected_adjusted_energy = None
+        self.current_energy_bill = None
+        self.expected_energy_bill = None
 
         self.recommendations_scoring_data = []
 
@@ -277,6 +284,7 @@ class Property:
                     recommendation_record=recommendation_record,
                     recommendations=previous_phase_representatives + [rec],
                     primary_recommendation_id=rec["recommendation_id"],
+                    non_invasive_recommendations=self.non_invasive_recommendations,
                 )
                 self.recommendations_scoring_data.append(scoring_dict)
 
@@ -286,6 +294,7 @@ class Property:
         recommendation_record,
         recommendations: list,
         primary_recommendation_id: int,
+        non_invasive_recommendations: list = None,
     ):
         """
         This function will iterate through a list of recommendations and apply a simulation for each recommendation
@@ -294,10 +303,12 @@ class Property:
         :param recommendation_record: The record of the property, which will be updated
         :param recommendations: The list of recommendations to apply
         :param primary_recommendation_id: The id of the primary recommendation, which is used to identify the record
+        :param non_invasive_recommendations: The list of non-invasive recommendations
         :return: The updated recommendation record
         """
 
         output = recommendation_record.copy()
+        non_invasive_recommendations = [] if non_invasive_recommendations is None else non_invasive_recommendations
 
         for col in [
             "walls_insulation_thickness",
@@ -316,6 +327,13 @@ class Property:
                 "external_wall_insulation",
                 "cavity_wall_insulation",
             ]:
+
+                # # If we have a non-incasive recommendation that the cavity wall is partially filled, we skip the
+                # # cavity wall insulation recommendation (since on the EPC, the property will look like how it did
+                # # before any works)
+                # if "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations:
+                #     continue
+
                 # The upgrade made here is to the u-value of the walls and the description of the
                 # insulation thickness
                 output["walls_thermal_transmittance_ending"] = recommendation[
@@ -892,12 +910,16 @@ class Property:
 
         return component_data
 
-    def set_adjusted_energy(self, current_adjusted_energy, expected_adjusted_energy):
+    def set_adjusted_energy(
+        self, current_adjusted_energy, expected_adjusted_energy, current_energy_bill, expected_energy_bill
+    ):
         """
         Stores these values for usage later
         """
         self.current_adjusted_energy = current_adjusted_energy
         self.expected_adjusted_energy = expected_adjusted_energy
+        self.current_energy_bill = current_energy_bill
+        self.expected_energy_bill = expected_energy_bill
 
     def set_windows_count(self):
         """

backend/app/db/functions/portfolio_functions.py

@@ -4,7 +4,7 @@ from backend.app.db.models.portfolio import Portfolio
 
 
 def aggregate_portfolio_recommendations(
-    session, portfolio_id: int, total_valuation_increase: float, labour_days: float
+    session, portfolio_id: int, total_valuation_increase: float, labour_days: float, aggregated_data: dict
 ):
     # Aggregate multiple fields
     aggregates = (
@@ -27,6 +27,7 @@ def aggregate_portfolio_recommendations(
         "energy_savings": aggregates.energy_savings or 0,
         "co2_equivalent_savings": aggregates.co2_equivalent_savings or 0,
         "energy_cost_savings": aggregates.energy_cost_savings or 0,
+        **aggregated_data
     }
 
     # Get the portfolio and update the fields

backend/app/db/models/portfolio.py

@@ -45,6 +45,21 @@ class Portfolio(Base):
     labour_days = Column(Float)
     created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
     updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
+    # Aggregations for summary
+    epc_breakdown_pre_retrofit = Column(Text)
+    epc_breakdown_post_retrofit = Column(Text)
+    n_units_to_retrofit = Column(Integer)
+    co2_per_unit_pre_retrofit = Column(Text)
+    co2_per_unit_post_retrofit = Column(Text)
+    energy_bill_per_unit_pre_retrofit = Column(Text)
+    energy_bill_per_unit_post_retrofit = Column(Text)
+    energy_consumption_per_unit_pre_retrofit = Column(Text)
+    energy_consumption_per_unit_post_retrofit = Column(Text)
+    valuation_improvement_per_unit = Column(Text)
+    cost_per_unit = Column(Text)
+    cost_per_co2_saved = Column(Text)
+    cost_per_sap_point = Column(Text)
+    valuation_return_on_investment = Column(Text)
 
 
 class PropertyCreationStatus(enum.Enum):

backend/app/plan/router.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 from tqdm import tqdm
@@ -51,12 +52,166 @@ def patch_epc(patch, epc_records):
     """
 
     for patch_variable, patch_value in patch.items():
+        if patch_value == "":
+            continue
         if patch_variable in epc_records["original_epc"]:
             epc_records["original_epc"][patch_variable] = patch_value
 
     return epc_records
 
 
+def extract_portfolio_aggregation_data(
+    input_properties, total_valuation_increase, recommendations, new_epc_bands, property_value_increase_ranges
+):
+    # We aggregate a number of metrics for the portfolio:
+    # 1) A breakdown of the number of properties in each EPC band
+    #    a) before retrofit
+    #    b) after retrofit
+    # 2) Number of units
+    # 3) Co2/unit
+    #    a) before retrofit
+    #    b) after retrofit
+    # 4) Energy bill/unit
+    #    a) before retrofit
+    #    b) after retrofit
+    # 5) Average valuation improvement/unit
+    # 6) Total cost
+    # 7) Cost per unit
+    # 8) £ per CO2 saved
+    # 9) £ per SAP point
+
+    # We need to construct the underlyind data for this
+
+    # Helper function to reformat the EPC data
+    def reformat_epc_data(epc_counts):
+        # Define all possible EPC bands in the required order
+        epc_bands = ["G", "F", "E", "D", "C", "B", "A"]
+
+        # Create the formatted data list by checking each band in the order
+        formatted_data = []
+        for band in epc_bands:
+            # Get the count from the dictionary, defaulting to 0 if not present
+            count = epc_counts.get(band, 0)
+            # Append the formatted dictionary to the list
+            formatted_data.append({"name": band, band: count})
+
+        return formatted_data
+
+    n_units = len(input_properties)
+
+    agg_data = []
+    for p in input_properties:
+        # Get the recommendations for the property - we include all properties, even ones without recommendations
+        property_recommendations = recommendations.get(p.id, [])
+
+        # Get just the default recommendations
+        default_recommendations = [r for r in property_recommendations if r["default"]]
+
+        has_recommendations = len(default_recommendations) > 0
+
+        # We can now calculate multiple outputs based on default recommendations
+        carbon_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations])
+
+        pre_retrofit_co2 = p.data["co2-emissions-current"]
+        post_retrofit_co2 = pre_retrofit_co2 - carbon_savings
+
+        pre_retrofit_energy_bill = p.current_energy_bill
+        post_retrofit_energy_bill = p.current_energy_bill - sum(
+            [r["energy_cost_savings"] for r in default_recommendations]
+        )
+
+        pre_retrofit_energy_consumption = p.current_adjusted_energy
+        post_retrofit_energy_consumption = p.current_adjusted_energy - sum(
+            [r["adjusted_heat_demand"] for r in default_recommendations]
+        )
+
+        # Add up energy savings
+        cost = sum([r["total"] for r in default_recommendations])
+        sap_point_improvement = sum([r["sap_points"] for r in default_recommendations])
+
+        lower_bound_valuation_uplift = (
+            property_value_increase_ranges[p.id]["lower_bound_increased_value"] -
+            property_value_increase_ranges[p.id]["current_value"]
+        )
+        upper_bound_valuation_uplift = (
+            property_value_increase_ranges[p.id]["upper_bound_increased_value"] -
+            property_value_increase_ranges[p.id]["current_value"]
+        )
+
+        agg_data.append({
+            "pre_retrofit_epc": p.data["current-energy-rating"],
+            "post_retrofit_epc": new_epc_bands[p.id],
+            "pre_retrofit_co2": pre_retrofit_co2,
+            "post_retrofit_co2": post_retrofit_co2,
+            "pre_retrofit_energy_bill": pre_retrofit_energy_bill,
+            "post_retrofit_energy_bill": post_retrofit_energy_bill,
+            "pre_retrofit_energy_consumption": pre_retrofit_energy_consumption,
+            "post_retrofit_energy_consumption": post_retrofit_energy_consumption,
+            "cost": cost,
+            "sap_point_improvement": sap_point_improvement,
+            "lower_bound_valuation_uplift": lower_bound_valuation_uplift,
+            "upper_bound_valuation_uplift": upper_bound_valuation_uplift,
+            "has_recommendations": has_recommendations
+        })
+
+    agg_data = pd.DataFrame(agg_data)
+
+    n_units_to_retrofit = agg_data["has_recommendations"].sum()
+
+    valuation_improvement_lower_bound_per_unit = (
+        agg_data["lower_bound_valuation_uplift"].mean()
+    )
+    valuation_improvement_upper_bound_per_unit = (
+        agg_data["upper_bound_valuation_uplift"].mean()
+    )
+
+    total_carbon_saved = agg_data["pre_retrofit_co2"].sum() - agg_data["post_retrofit_co2"].sum()
+    total_sap_points = agg_data["sap_point_improvement"].sum()
+
+    def format_money(amount):
+        return f"£{amount:,.0f}"
+
+    valuation_improvment_per_unit = str(
+        format_money(
+            total_valuation_increase / n_units) + (f" ({format_money(valuation_improvement_lower_bound_per_unit)} - "
+                                                   f"{format_money(valuation_improvement_upper_bound_per_unit)})")
+    )
+
+    valuation_return_on_investment = str(
+        str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) +
+        f" ("
+        f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - "
+        f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})"
+    )
+
+    aggregation_data = {
+        "epc_breakdown_pre_retrofit": json.dumps(
+            reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict())
+        ),
+        "epc_breakdown_post_retrofit": json.dumps(
+            reformat_epc_data(agg_data["post_retrofit_epc"].value_counts().to_dict())
+        ),
+        "number_of_properties": int(n_units),
+        "n_units_to_retrofit": int(n_units_to_retrofit),
+        "co2_per_unit_pre_retrofit": str(round(agg_data["pre_retrofit_co2"].mean(), 2)) + "t",
+        "co2_per_unit_post_retrofit": str(round(agg_data["post_retrofit_co2"].mean(), 2)) + "t",
+        "energy_bill_per_unit_pre_retrofit": format_money(agg_data["pre_retrofit_energy_bill"].mean()),
+        "energy_bill_per_unit_post_retrofit": format_money(agg_data["post_retrofit_energy_bill"].mean()),
+        "energy_consumption_per_unit_pre_retrofit": str(
+            round(agg_data["pre_retrofit_energy_consumption"].mean())) + "kWh",
+        "energy_consumption_per_unit_post_retrofit": str(
+            round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh",
+        "valuation_improvement_per_unit": valuation_improvment_per_unit,
+        "cost_per_unit": format_money(agg_data["cost"].mean()),
+        "cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved),
+        "cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points),
+        "valuation_return_on_investment": valuation_return_on_investment,
+        # TODO: Could we add 10yr carbon credits value?
+    }
+
+    return aggregation_data
+
+
 router = APIRouter(
     prefix="/plan",
     tags=["plan"],
@@ -91,6 +246,12 @@ async def trigger_plan(body: PlanTriggerRequest):
                 bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.already_installed_file_path
             )
 
+        non_invasive_recommendations = []
+        if body.non_invasive_recommendations_file_path:
+            non_invasive_recommendations = read_csv_from_s3(
+                bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.non_invasive_recommendations_file_path
+            )
+
         cleaning_data = read_dataframe_from_s3_parquet(
             bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet",
         )
@@ -146,6 +307,12 @@ async def trigger_plan(body: PlanTriggerRequest):
                 x for x in already_installed if
                 (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
             ), {})
+
+            property_non_invasive_recommendations = next((
+                x for x in non_invasive_recommendations if
+                (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
+            ), {})
+
             input_properties.append(
                 Property(
                     id=property_id,
@@ -153,6 +320,7 @@ async def trigger_plan(body: PlanTriggerRequest):
                     postcode=epc_searcher.postcode_clean,
                     epc_record=prepared_epc,
                     already_installed=property_already_installed,
+                    non_invasive_recommendations=property_non_invasive_recommendations,
                     **Property.extract_kwargs(config)
                 )
             )
@@ -243,7 +411,13 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             property_instance = [p for p in input_properties if p.id == property_id][0]
 
-            recommendations_with_impact, current_adjusted_energy, expected_adjusted_energy = (
+            (
+                recommendations_with_impact,
+                current_adjusted_energy,
+                expected_adjusted_energy,
+                current_energy_bill,
+                expected_energy_bill
+            ) = (
                 Recommendations.calculate_recommendation_impact(
                     property_instance=property_instance,
                     all_predictions=all_predictions,
@@ -254,7 +428,9 @@ async def trigger_plan(body: PlanTriggerRequest):
             # Store the resulting adjusted energy in the property instance
             property_instance.set_adjusted_energy(
                 current_adjusted_energy=current_adjusted_energy,
-                expected_adjusted_energy=expected_adjusted_energy
+                expected_adjusted_energy=expected_adjusted_energy,
+                current_energy_bill=current_energy_bill,
+                expected_energy_bill=expected_energy_bill
             )
 
             input_measures = prepare_input_measures(recommendations_with_impact, body.goal)
@@ -316,6 +492,8 @@ async def trigger_plan(body: PlanTriggerRequest):
         logger.info("Uploading recommendations to the database")
         property_valuation_increases = []
         session.commit()
+        new_epc_bands = {}
+        property_value_increase_ranges = {}
         for i in range(0, len(input_properties), BATCH_SIZE):
             try:
                 # Take a slice of the input_properties list to make a batch
@@ -327,8 +505,10 @@ async def trigger_plan(body: PlanTriggerRequest):
                     total_sap_points = sum([r["sap_points"] for r in default_recommendations])
                     new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points
                     new_epc = sap_to_epc(new_sap_points)
+                    new_epc_bands[p.id] = new_epc
 
                     valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc)
+                    property_value_increase_ranges[p.id] = valuations
 
                     # Your existing operations
                     property_details_epc = p.get_property_details_epc(
@@ -392,11 +572,20 @@ async def trigger_plan(body: PlanTriggerRequest):
             [sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()]
         ))
 
+        aggregated_data = extract_portfolio_aggregation_data(
+            input_properties=input_properties,
+            total_valuation_increase=total_valuation_increase,
+            recommendations=recommendations,
+            new_epc_bands=new_epc_bands,
+            property_value_increase_ranges=property_value_increase_ranges
+        )
+
         aggregate_portfolio_recommendations(
             session,
             portfolio_id=body.portfolio_id,
             total_valuation_increase=total_valuation_increase,
-            labour_days=labour_days
+            labour_days=labour_days,
+            aggregated_data=aggregated_data
         )
 
         # Commit final changes

backend/app/plan/schemas.py

@@ -11,6 +11,7 @@ class PlanTriggerRequest(BaseModel):
     trigger_file_path: str
     already_installed_file_path: Optional[str] = None
     patches_file_path: Optional[str] = None
+    non_invasive_recommendations_file_path: Optional[str] = None
     exclusions: Optional[conlist(str, min_items=1)] = None
 
     # Pre-defined list of possibilities for exclusions

etl/customers/immo/pilot/asset_list.py

@@ -21,6 +21,7 @@ council_tax_bands = pd.DataFrame(council_tax_bands)
 
 # This is information we need to override on the EPC itself, for instance if a new survey has been conducted and
 # that has not reached the API
+# For 53 Bromley, the non-invasives found the walls to be partially filled
 patches = [
     {
         'address': '6 Beech Road', 'postcode': 'DY1 4BP',
@@ -42,7 +43,11 @@ patches = [
         'energy-consumption-current': '491',
         'co2-emissions-current': '5.0',
         'potential-energy-efficiency': '87'
-    }
+    },
+    {
+        'address': '53 Bromley', 'postcode': 'DY5 4PJ',
+        'walls-description': 'Cavity wall, partial insulation (assumed)',
+    },
 ]
 
 # This is information that is found as a result of the non-invasives, that mean that certain measures
@@ -56,6 +61,19 @@ already_installed = [
     }
 ]
 
+non_invasive_recommendations = [
+    {'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'recommendations': []},
+    {'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'recommendations': ['cavity_surveyed_as_filled_is_partial']},
+    {'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'recommendations': []},
+    {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'recommendations': ['cavity_extract_and_refill']},
+]
+
 
 def app():
     raw_asset_list = read_excel_from_s3(
@@ -102,6 +120,14 @@ def app():
         file_name=patches_filename
     )
 
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
     # EPC C portoflio
     body = {
         "portfolio_id": str(PORTFOLIO_ID),
@@ -111,6 +137,7 @@ def app():
         "trigger_file_path": filename,
         "already_installed_file_path": already_installed_filename,
         "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
         "budget": None,
     }
     print(body)
@@ -124,6 +151,7 @@ def app():
         "trigger_file_path": filename,
         "already_installed_file_path": already_installed_filename,
         "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
         "budget": None,
     }
     print(body)

etl/non_intrusive_surveys/photos/README.md

@@ -0,0 +1,19 @@
+# Non Intrusive Surveys - photo upload
+
+This folder contains photos taken during non-intrusive surveys. Photos are stored in folders named after the survey ID.
+
+## Getting started
+
+Install the required packages by running the following command:
+
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+
+The main application is found in the app.py file. To run the application, use the following command:
+
+```bash
+python app.py
+```

etl/non_intrusive_surveys/photos/app.py

@@ -0,0 +1,149 @@
+import boto3
+import os
+from PIL import Image
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Inputs
+ENV_FILEPATH = "etl/non_intrusive_surveys/photos/.env"
+PHOTO_DIRECTORY = "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data"
+FOLDER_UPRN_LOOKUP = {
+    "91 Osprey Drive DY1 2JS": 90048026,
+    "195 Ashenhurst Rd DY1 2JB": 90051858,
+    "6 Beech Rd DY1 4BP": 90055152,
+    "53 Bromley DY5 4PJ": 90060989,
+    "5 Oaklands B62 0JA": 90028499,
+    "47 Fairfield Rd DY8 5UJ": 90077535,
+    "150 Huntingtree Rd B63 4HP": 90093693,
+    "27 Milton Rd DY1 2JB": 90106884,
+    "21 Wells Rd DY5 3TB": 90022227,
+    "8 Corporation Rd DY2 7PX": 90070461
+}
+
+load_dotenv(ENV_FILEPATH)
+CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME = os.getenv("CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME", None)
+CDN_BUCKET_NAME = os.getenv("CDN_BUCKET_NAME", None)
+
+
+def list_subdirectories(directory_path):
+    """
+    List all subdirectories within a given directory.
+
+    :param directory_path: Path to the directory.
+    :return: A list of paths to the subdirectories.
+    """
+    directory = Path(directory_path)
+    subdirectories = [subdir for subdir in directory.iterdir() if subdir.is_dir()]
+    return subdirectories
+
+
+def list_files_in_directory(directory_path, file_extension=".jpg"):
+    """
+    List all files with a specific extension within a given directory and its subdirectories.
+
+    :param directory_path: Path to the directory to scan.
+    :param file_extension: File extension to filter by.
+    :return: A list of paths to the files.
+    """
+    # Convert the directory path to a Path object if it's not already one
+    directory = Path(directory_path) if not isinstance(directory_path, Path) else directory_path
+
+    # List all files of the specified type in the directory and subdirectories
+    file_list = [file for file in directory.rglob(f'*{file_extension}')]
+
+    return file_list
+
+
+def create_images(input_path, uprn):
+    # Define the base directory path
+    base_directory = f"non_intrusive_photos/{uprn}"
+    print(f"Creating directory: {base_directory}")  # Debug: print the directory to be created
+
+    # Need to create local directory if it doesn't exist
+    os.makedirs(base_directory, exist_ok=True)
+
+    # Define output paths
+    thumbnail_path = os.path.join(base_directory, "thumbnail.jpg")
+    full_hd_path = os.path.join(base_directory, "1080p.jpg")
+    webp_path = os.path.join(base_directory, "webp.webp")  # Save as WebP format
+
+    # Load the image
+    with Image.open(input_path) as img:
+        # Create a thumbnail
+        thumbnail = img.copy()
+        thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS)
+        thumbnail.save(thumbnail_path, 'JPEG', quality=85)
+
+        # Create a 1080p version
+        full_hd = img.copy()
+        full_hd.thumbnail((1920, 1080), Image.Resampling.LANCZOS)
+        full_hd.save(full_hd_path, 'JPEG', quality=90)
+
+        # Convert to WebP for better compression
+        webp = img.copy()
+        webp.save(webp_path, 'WEBP', quality=90)
+
+    # Return paths to the processed images
+    return thumbnail_path, full_hd_path, webp_path
+
+
+def upload_to_s3(bucket_name, file_path, object_name):
+    s3_client = boto3.client('s3')
+    s3_client.upload_file(file_path, bucket_name, object_name)
+    print(f"Uploaded {object_name} to S3 bucket {bucket_name}")
+
+
+def upload_photos_to_s3(bucket_name, photo_paths):
+    # Upload each photo
+    for path in photo_paths:
+        object_name = path.split('/')[-1]  # Assuming the path format is folder/filename
+        upload_to_s3(bucket_name, path, object_name)
+
+
+def generate_cdn_url(distribution_domain, object_name):
+    return f"https://{distribution_domain}/{object_name}"
+
+
+def process_and_upload_images(uprn, input_image_path, bucket_name, distribution_domain):
+    # Create images
+    thumbnail, full_hd, original = create_images(input_image_path, uprn=str(uprn))
+
+    # Upload images
+    upload_photos_to_s3(bucket_name, photo_paths=[thumbnail, full_hd, original])
+
+    # Generate CDN links
+    cdn_links = [generate_cdn_url(distribution_domain, path.split('/')[-1]) for path in [thumbnail, full_hd, original]]
+
+    # Delete local files
+    for path in [thumbnail, full_hd, original]:
+        os.remove(path)
+
+    return cdn_links
+
+
+def app():
+    """
+    This application is tasked with uploading the photos, recorded during the non-invasive surveys, to s3 and the
+    database.
+    To begin with, this app will simply read the files from the local machine, however we will come up with a more
+    efficient way to do this in the future.
+
+    :return:
+    """
+
+    # List all files in the directory using pathlib
+    property_directories = list_subdirectories(PHOTO_DIRECTORY)
+
+    # For each property, we want to list all of the photos in the directory
+    for property_dir in property_directories:
+        photo_files = list_files_in_directory(property_dir)
+        uprn = FOLDER_UPRN_LOOKUP[property_dir.name]
+
+        # We now want to convert each file, and upload it to s3
+        for photo_filepath in photo_files:
+            process_and_upload_images(
+                uprn=uprn,
+                input_image_path=photo_filepath,
+                bucket_name=CDN_BUCKET_NAME,
+                distribution_domain=CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME
+            )

etl/non_intrusive_surveys/photos/requirements.txt

@@ -0,0 +1,3 @@
+Pillow
+boto3
+python-dotenv

recommendations/Costs.py

@@ -91,6 +91,10 @@ DOUBLE_RADIATOR_COST = 300
 FLUE_COST = 600
 PIPEWORK_COST = 750  # Min cost is £500
 
+# This is the cost per meter squared for cavity extraction
+# https://www.checkatrade.com/blog/cost-guides/cavity-wall-insulation-removal-cost/
+CAVITY_EXTRACTION_COST = 21.5
+
 
 class Costs:
     """
@@ -173,7 +177,7 @@ class Costs:
         if not self.labour_adjustment_factor:
             raise ValueError("Labour adjustment factor not found")
 
-    def cavity_wall_insulation(self, wall_area, material):
+    def cavity_wall_insulation(self, wall_area, material, is_extraction_and_refill=False):
         """
         Calculates the total cost for cavity wall insulation based on material and labor costs,
         including contingency, preliminaries, profit, and VAT.
@@ -208,6 +212,13 @@ class Costs:
         # Assume a team of 2
         labour_days = (labour_hours / 8) / 2
 
+        if is_extraction_and_refill:
+            # bump up the cost of the work
+            total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area
+            # Additional 2 days work
+            labour_hours = labour_hours + (2 * 8)
+            labour_days = labour_days + 2
+
         return {
             "total": total_cost,
             "subtotal": subtotal_before_vat,

recommendations/Recommendations.py

@@ -149,12 +149,14 @@ class Recommendations:
         property_recommendations = self.insert_temp_recommendation_id(property_recommendations)
 
         # We also need to create the representative recommendations for each recommendation type
-        property_representative_recommendations = self.create_representative_recommendations(property_recommendations)
+        property_representative_recommendations = self.create_representative_recommendations(
+            property_recommendations, non_invasive_recommendations=self.property_instance.non_invasive_recommendations
+        )
 
         return property_recommendations, property_representative_recommendations
 
     @staticmethod
-    def create_representative_recommendations(property_recommendations):
+    def create_representative_recommendations(property_recommendations, non_invasive_recommendations):
         """
         This method will create a representative recommendation for each recommendation type
         In order to create a representative recommendation, we choose the recommendation that has:
@@ -169,6 +171,13 @@ class Recommendations:
 
         for recommendations_by_type in property_recommendations:
 
+            # If the property was initially surveyed as filled, but the cavity was only partially filled, we don't
+            # want to include the cavity wall insulation recommendation in the defaults
+            # if (recommendations_by_type[0].get("type") == "cavity_wall_insulation") and (
+            #     "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations
+            # ):
+            #     continue
+
             if recommendations_by_type[0].get("type") == "mechanical_ventilation":
                 continue
 
@@ -238,13 +247,13 @@ class Recommendations:
 
         property_sap_predictions = all_predictions["sap_change_predictions"][
             all_predictions["sap_change_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
         property_heat_predictions = all_predictions["heat_demand_predictions"][
             all_predictions["heat_demand_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
         property_carbon_predictions = all_predictions["carbon_change_predictions"][
             all_predictions["carbon_change_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
 
         property_recommendations = recommendations[property_instance.id].copy()
 
@@ -272,6 +281,8 @@ class Recommendations:
             current_epc_rating=property_instance.data["current-energy-rating"],
         )
 
+        # TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are
+        #       actually implemented
         expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
             epc_energy_consumption=expected_heat_demand,
             current_epc_rating=property_instance.data["current-energy-rating"],
@@ -281,6 +292,10 @@ class Recommendations:
             current_adjusted_energy - expected_adjusted_energy
         )
 
+        # TODO: We should determine if the home is gas & electricity or just electricity
+        current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy)
+        expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy)
+
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
 
@@ -355,4 +370,10 @@ class Recommendations:
                     rec["heat_demand"] is None) or (rec["energy_cost_savings"] is None):
                     raise ValueError("sap points, co2 or heat demand is missing")
 
-        return property_recommendations, current_adjusted_energy, expected_adjusted_energy
+        return (
+            property_recommendations,
+            current_adjusted_energy,
+            expected_adjusted_energy,
+            current_energy_bill,
+            expected_energy_bill
+        )

recommendations/WallRecommendations.py

@@ -113,7 +113,9 @@ class WallRecommendations(Definitions):
         insulation_thickness = self.property.walls["insulation_thickness"]
 
         # We check if the wall is already insulated and if so, we exit
-        if (insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]:
+        if ((insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]) and (
+            "cavity_extract_and_refill" not in self.property.non_invasive_recommendations
+        ):
             return
 
         if u_value:
@@ -216,15 +218,26 @@ class WallRecommendations(Definitions):
             if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE:
                 lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
 
+                is_extraction_and_refill = "cavity_extract_and_refill" in self.property.non_invasive_recommendations
+
                 cost_result = self.costs.cavity_wall_insulation(
                     wall_area=self.property.insulation_wall_area,
                     material=material.to_dict(),
+                    is_extraction_and_refill=is_extraction_and_refill
                 )
 
                 already_installed = "cavity_wall_insulation" in self.property.already_installed
                 if already_installed:
                     cost_result = override_costs(cost_result)
 
+                if is_extraction_and_refill:
+                    description = f"Extract and refill cavity wall insulation with {material['description']}"
+                else:
+                    description = self._make_description(material)
+
+                # updated the new u-value with the best possible our installers have
+                new_u_value = max(0.31, new_u_value)
+
                 recommendations.append(
                     {
                         "phase": phase,
@@ -237,7 +250,7 @@ class WallRecommendations(Definitions):
                             )
                         ],
                         "type": "cavity_wall_insulation",
-                        "description": self._make_description(material),
+                        "description": description,
                         "starting_u_value": u_value,
                         "new_u_value": new_u_value,
                         "sap_points": None,