diff --git a/backend/Property.py b/backend/Property.py index a8ed9129..2e6cbbb6 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -61,7 +61,8 @@ class Property: n_bedrooms = None def __init__( - self, id, postcode, address, epc_record, already_installed=None, **kwargs + self, id, postcode, address, epc_record, already_installed=None, non_invasive_recommendations=None, + **kwargs ): self.epc_record = epc_record @@ -80,6 +81,10 @@ class Property: # cost and instead, provide a message that the measure has already been installed self.already_installed = ast.literal_eval(already_installed['already_installed']) if already_installed else [] + self.non_invasive_recommendations = ( + ast.literal_eval(non_invasive_recommendations['recommendations']) if + non_invasive_recommendations else [] + ) self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") @@ -142,6 +147,8 @@ class Property: self.current_adjusted_energy = None self.expected_adjusted_energy = None + self.current_energy_bill = None + self.expected_energy_bill = None self.recommendations_scoring_data = [] @@ -277,6 +284,7 @@ class Property: recommendation_record=recommendation_record, recommendations=previous_phase_representatives + [rec], primary_recommendation_id=rec["recommendation_id"], + non_invasive_recommendations=self.non_invasive_recommendations, ) self.recommendations_scoring_data.append(scoring_dict) @@ -286,6 +294,7 @@ class Property: recommendation_record, recommendations: list, primary_recommendation_id: int, + non_invasive_recommendations: list = None, ): """ This function will iterate through a list of recommendations and apply a simulation for each recommendation @@ -294,10 +303,12 @@ class Property: :param recommendation_record: The record of the property, which will be updated :param recommendations: The list of recommendations to apply :param primary_recommendation_id: The id of the primary recommendation, which is used to identify the record + :param non_invasive_recommendations: The list of non-invasive recommendations :return: The updated recommendation record """ output = recommendation_record.copy() + non_invasive_recommendations = [] if non_invasive_recommendations is None else non_invasive_recommendations for col in [ "walls_insulation_thickness", @@ -316,6 +327,13 @@ class Property: "external_wall_insulation", "cavity_wall_insulation", ]: + + # # If we have a non-incasive recommendation that the cavity wall is partially filled, we skip the + # # cavity wall insulation recommendation (since on the EPC, the property will look like how it did + # # before any works) + # if "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations: + # continue + # The upgrade made here is to the u-value of the walls and the description of the # insulation thickness output["walls_thermal_transmittance_ending"] = recommendation[ @@ -892,12 +910,16 @@ class Property: return component_data - def set_adjusted_energy(self, current_adjusted_energy, expected_adjusted_energy): + def set_adjusted_energy( + self, current_adjusted_energy, expected_adjusted_energy, current_energy_bill, expected_energy_bill + ): """ Stores these values for usage later """ self.current_adjusted_energy = current_adjusted_energy self.expected_adjusted_energy = expected_adjusted_energy + self.current_energy_bill = current_energy_bill + self.expected_energy_bill = expected_energy_bill def set_windows_count(self): """ diff --git a/backend/app/db/functions/portfolio_functions.py b/backend/app/db/functions/portfolio_functions.py index ead8280f..69203368 100644 --- a/backend/app/db/functions/portfolio_functions.py +++ b/backend/app/db/functions/portfolio_functions.py @@ -4,7 +4,7 @@ from backend.app.db.models.portfolio import Portfolio def aggregate_portfolio_recommendations( - session, portfolio_id: int, total_valuation_increase: float, labour_days: float + session, portfolio_id: int, total_valuation_increase: float, labour_days: float, aggregated_data: dict ): # Aggregate multiple fields aggregates = ( @@ -27,6 +27,7 @@ def aggregate_portfolio_recommendations( "energy_savings": aggregates.energy_savings or 0, "co2_equivalent_savings": aggregates.co2_equivalent_savings or 0, "energy_cost_savings": aggregates.energy_cost_savings or 0, + **aggregated_data } # Get the portfolio and update the fields diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index 830866e6..aa0146c0 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -45,6 +45,21 @@ class Portfolio(Base): labour_days = Column(Float) created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc)) + # Aggregations for summary + epc_breakdown_pre_retrofit = Column(Text) + epc_breakdown_post_retrofit = Column(Text) + n_units_to_retrofit = Column(Integer) + co2_per_unit_pre_retrofit = Column(Text) + co2_per_unit_post_retrofit = Column(Text) + energy_bill_per_unit_pre_retrofit = Column(Text) + energy_bill_per_unit_post_retrofit = Column(Text) + energy_consumption_per_unit_pre_retrofit = Column(Text) + energy_consumption_per_unit_post_retrofit = Column(Text) + valuation_improvement_per_unit = Column(Text) + cost_per_unit = Column(Text) + cost_per_co2_saved = Column(Text) + cost_per_sap_point = Column(Text) + valuation_return_on_investment = Column(Text) class PropertyCreationStatus(enum.Enum): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 49e14872..9854abe8 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from tqdm import tqdm @@ -51,12 +52,166 @@ def patch_epc(patch, epc_records): """ for patch_variable, patch_value in patch.items(): + if patch_value == "": + continue if patch_variable in epc_records["original_epc"]: epc_records["original_epc"][patch_variable] = patch_value return epc_records +def extract_portfolio_aggregation_data( + input_properties, total_valuation_increase, recommendations, new_epc_bands, property_value_increase_ranges +): + # We aggregate a number of metrics for the portfolio: + # 1) A breakdown of the number of properties in each EPC band + # a) before retrofit + # b) after retrofit + # 2) Number of units + # 3) Co2/unit + # a) before retrofit + # b) after retrofit + # 4) Energy bill/unit + # a) before retrofit + # b) after retrofit + # 5) Average valuation improvement/unit + # 6) Total cost + # 7) Cost per unit + # 8) £ per CO2 saved + # 9) £ per SAP point + + # We need to construct the underlyind data for this + + # Helper function to reformat the EPC data + def reformat_epc_data(epc_counts): + # Define all possible EPC bands in the required order + epc_bands = ["G", "F", "E", "D", "C", "B", "A"] + + # Create the formatted data list by checking each band in the order + formatted_data = [] + for band in epc_bands: + # Get the count from the dictionary, defaulting to 0 if not present + count = epc_counts.get(band, 0) + # Append the formatted dictionary to the list + formatted_data.append({"name": band, band: count}) + + return formatted_data + + n_units = len(input_properties) + + agg_data = [] + for p in input_properties: + # Get the recommendations for the property - we include all properties, even ones without recommendations + property_recommendations = recommendations.get(p.id, []) + + # Get just the default recommendations + default_recommendations = [r for r in property_recommendations if r["default"]] + + has_recommendations = len(default_recommendations) > 0 + + # We can now calculate multiple outputs based on default recommendations + carbon_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations]) + + pre_retrofit_co2 = p.data["co2-emissions-current"] + post_retrofit_co2 = pre_retrofit_co2 - carbon_savings + + pre_retrofit_energy_bill = p.current_energy_bill + post_retrofit_energy_bill = p.current_energy_bill - sum( + [r["energy_cost_savings"] for r in default_recommendations] + ) + + pre_retrofit_energy_consumption = p.current_adjusted_energy + post_retrofit_energy_consumption = p.current_adjusted_energy - sum( + [r["adjusted_heat_demand"] for r in default_recommendations] + ) + + # Add up energy savings + cost = sum([r["total"] for r in default_recommendations]) + sap_point_improvement = sum([r["sap_points"] for r in default_recommendations]) + + lower_bound_valuation_uplift = ( + property_value_increase_ranges[p.id]["lower_bound_increased_value"] - + property_value_increase_ranges[p.id]["current_value"] + ) + upper_bound_valuation_uplift = ( + property_value_increase_ranges[p.id]["upper_bound_increased_value"] - + property_value_increase_ranges[p.id]["current_value"] + ) + + agg_data.append({ + "pre_retrofit_epc": p.data["current-energy-rating"], + "post_retrofit_epc": new_epc_bands[p.id], + "pre_retrofit_co2": pre_retrofit_co2, + "post_retrofit_co2": post_retrofit_co2, + "pre_retrofit_energy_bill": pre_retrofit_energy_bill, + "post_retrofit_energy_bill": post_retrofit_energy_bill, + "pre_retrofit_energy_consumption": pre_retrofit_energy_consumption, + "post_retrofit_energy_consumption": post_retrofit_energy_consumption, + "cost": cost, + "sap_point_improvement": sap_point_improvement, + "lower_bound_valuation_uplift": lower_bound_valuation_uplift, + "upper_bound_valuation_uplift": upper_bound_valuation_uplift, + "has_recommendations": has_recommendations + }) + + agg_data = pd.DataFrame(agg_data) + + n_units_to_retrofit = agg_data["has_recommendations"].sum() + + valuation_improvement_lower_bound_per_unit = ( + agg_data["lower_bound_valuation_uplift"].mean() + ) + valuation_improvement_upper_bound_per_unit = ( + agg_data["upper_bound_valuation_uplift"].mean() + ) + + total_carbon_saved = agg_data["pre_retrofit_co2"].sum() - agg_data["post_retrofit_co2"].sum() + total_sap_points = agg_data["sap_point_improvement"].sum() + + def format_money(amount): + return f"£{amount:,.0f}" + + valuation_improvment_per_unit = str( + format_money( + total_valuation_increase / n_units) + (f" ({format_money(valuation_improvement_lower_bound_per_unit)} - " + f"{format_money(valuation_improvement_upper_bound_per_unit)})") + ) + + valuation_return_on_investment = str( + str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) + + f" (" + f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - " + f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})" + ) + + aggregation_data = { + "epc_breakdown_pre_retrofit": json.dumps( + reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict()) + ), + "epc_breakdown_post_retrofit": json.dumps( + reformat_epc_data(agg_data["post_retrofit_epc"].value_counts().to_dict()) + ), + "number_of_properties": int(n_units), + "n_units_to_retrofit": int(n_units_to_retrofit), + "co2_per_unit_pre_retrofit": str(round(agg_data["pre_retrofit_co2"].mean(), 2)) + "t", + "co2_per_unit_post_retrofit": str(round(agg_data["post_retrofit_co2"].mean(), 2)) + "t", + "energy_bill_per_unit_pre_retrofit": format_money(agg_data["pre_retrofit_energy_bill"].mean()), + "energy_bill_per_unit_post_retrofit": format_money(agg_data["post_retrofit_energy_bill"].mean()), + "energy_consumption_per_unit_pre_retrofit": str( + round(agg_data["pre_retrofit_energy_consumption"].mean())) + "kWh", + "energy_consumption_per_unit_post_retrofit": str( + round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh", + "valuation_improvement_per_unit": valuation_improvment_per_unit, + "cost_per_unit": format_money(agg_data["cost"].mean()), + "cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved), + "cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points), + "valuation_return_on_investment": valuation_return_on_investment, + # TODO: Could we add 10yr carbon credits value? + } + + return aggregation_data + + router = APIRouter( prefix="/plan", tags=["plan"], @@ -91,6 +246,12 @@ async def trigger_plan(body: PlanTriggerRequest): bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.already_installed_file_path ) + non_invasive_recommendations = [] + if body.non_invasive_recommendations_file_path: + non_invasive_recommendations = read_csv_from_s3( + bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.non_invasive_recommendations_file_path + ) + cleaning_data = read_dataframe_from_s3_parquet( bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet", ) @@ -146,6 +307,12 @@ async def trigger_plan(body: PlanTriggerRequest): x for x in already_installed if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) ), {}) + + property_non_invasive_recommendations = next(( + x for x in non_invasive_recommendations if + (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + ), {}) + input_properties.append( Property( id=property_id, @@ -153,6 +320,7 @@ async def trigger_plan(body: PlanTriggerRequest): postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, already_installed=property_already_installed, + non_invasive_recommendations=property_non_invasive_recommendations, **Property.extract_kwargs(config) ) ) @@ -243,7 +411,13 @@ async def trigger_plan(body: PlanTriggerRequest): property_instance = [p for p in input_properties if p.id == property_id][0] - recommendations_with_impact, current_adjusted_energy, expected_adjusted_energy = ( + ( + recommendations_with_impact, + current_adjusted_energy, + expected_adjusted_energy, + current_energy_bill, + expected_energy_bill + ) = ( Recommendations.calculate_recommendation_impact( property_instance=property_instance, all_predictions=all_predictions, @@ -254,7 +428,9 @@ async def trigger_plan(body: PlanTriggerRequest): # Store the resulting adjusted energy in the property instance property_instance.set_adjusted_energy( current_adjusted_energy=current_adjusted_energy, - expected_adjusted_energy=expected_adjusted_energy + expected_adjusted_energy=expected_adjusted_energy, + current_energy_bill=current_energy_bill, + expected_energy_bill=expected_energy_bill ) input_measures = prepare_input_measures(recommendations_with_impact, body.goal) @@ -316,6 +492,8 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Uploading recommendations to the database") property_valuation_increases = [] session.commit() + new_epc_bands = {} + property_value_increase_ranges = {} for i in range(0, len(input_properties), BATCH_SIZE): try: # Take a slice of the input_properties list to make a batch @@ -327,8 +505,10 @@ async def trigger_plan(body: PlanTriggerRequest): total_sap_points = sum([r["sap_points"] for r in default_recommendations]) new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points new_epc = sap_to_epc(new_sap_points) + new_epc_bands[p.id] = new_epc valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc) + property_value_increase_ranges[p.id] = valuations # Your existing operations property_details_epc = p.get_property_details_epc( @@ -392,11 +572,20 @@ async def trigger_plan(body: PlanTriggerRequest): [sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()] )) + aggregated_data = extract_portfolio_aggregation_data( + input_properties=input_properties, + total_valuation_increase=total_valuation_increase, + recommendations=recommendations, + new_epc_bands=new_epc_bands, + property_value_increase_ranges=property_value_increase_ranges + ) + aggregate_portfolio_recommendations( session, portfolio_id=body.portfolio_id, total_valuation_increase=total_valuation_increase, - labour_days=labour_days + labour_days=labour_days, + aggregated_data=aggregated_data ) # Commit final changes diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 76eb49d2..59c0ebef 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -11,6 +11,7 @@ class PlanTriggerRequest(BaseModel): trigger_file_path: str already_installed_file_path: Optional[str] = None patches_file_path: Optional[str] = None + non_invasive_recommendations_file_path: Optional[str] = None exclusions: Optional[conlist(str, min_items=1)] = None # Pre-defined list of possibilities for exclusions diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index e587cc25..6329a2be 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -21,6 +21,7 @@ council_tax_bands = pd.DataFrame(council_tax_bands) # This is information we need to override on the EPC itself, for instance if a new survey has been conducted and # that has not reached the API +# For 53 Bromley, the non-invasives found the walls to be partially filled patches = [ { 'address': '6 Beech Road', 'postcode': 'DY1 4BP', @@ -42,7 +43,11 @@ patches = [ 'energy-consumption-current': '491', 'co2-emissions-current': '5.0', 'potential-energy-efficiency': '87' - } + }, + { + 'address': '53 Bromley', 'postcode': 'DY5 4PJ', + 'walls-description': 'Cavity wall, partial insulation (assumed)', + }, ] # This is information that is found as a result of the non-invasives, that mean that certain measures @@ -56,6 +61,19 @@ already_installed = [ } ] +non_invasive_recommendations = [ + {'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'recommendations': []}, + {'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'recommendations': ['cavity_surveyed_as_filled_is_partial']}, + {'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'recommendations': ['cavity_extract_and_refill']}, + {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'recommendations': []}, + {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'recommendations': ['cavity_extract_and_refill']}, +] + def app(): raw_asset_list = read_excel_from_s3( @@ -102,6 +120,14 @@ def app(): file_name=patches_filename ) + # Store non-invasive recommendations in S3 + non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json" + save_csv_to_s3( + dataframe=pd.DataFrame(non_invasive_recommendations), + bucket_name="retrofit-plan-inputs-dev", + file_name=non_invasive_recommendations_filename + ) + # EPC C portoflio body = { "portfolio_id": str(PORTFOLIO_ID), @@ -111,6 +137,7 @@ def app(): "trigger_file_path": filename, "already_installed_file_path": already_installed_filename, "patches_file_path": patches_filename, + "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, "budget": None, } print(body) @@ -124,6 +151,7 @@ def app(): "trigger_file_path": filename, "already_installed_file_path": already_installed_filename, "patches_file_path": patches_filename, + "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, "budget": None, } print(body) diff --git a/etl/non_intrusive_surveys/photos/README.md b/etl/non_intrusive_surveys/photos/README.md new file mode 100644 index 00000000..9dbe951f --- /dev/null +++ b/etl/non_intrusive_surveys/photos/README.md @@ -0,0 +1,19 @@ +# Non Intrusive Surveys - photo upload + +This folder contains photos taken during non-intrusive surveys. Photos are stored in folders named after the survey ID. + +## Getting started + +Install the required packages by running the following command: + +```bash +pip install -r requirements.txt +``` + +## Usage + +The main application is found in the app.py file. To run the application, use the following command: + +```bash +python app.py +``` \ No newline at end of file diff --git a/etl/non_intrusive_surveys/photos/app.py b/etl/non_intrusive_surveys/photos/app.py new file mode 100644 index 00000000..c531355b --- /dev/null +++ b/etl/non_intrusive_surveys/photos/app.py @@ -0,0 +1,149 @@ +import boto3 +import os +from PIL import Image +from pathlib import Path +from dotenv import load_dotenv + +# Inputs +ENV_FILEPATH = "etl/non_intrusive_surveys/photos/.env" +PHOTO_DIRECTORY = "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data" +FOLDER_UPRN_LOOKUP = { + "91 Osprey Drive DY1 2JS": 90048026, + "195 Ashenhurst Rd DY1 2JB": 90051858, + "6 Beech Rd DY1 4BP": 90055152, + "53 Bromley DY5 4PJ": 90060989, + "5 Oaklands B62 0JA": 90028499, + "47 Fairfield Rd DY8 5UJ": 90077535, + "150 Huntingtree Rd B63 4HP": 90093693, + "27 Milton Rd DY1 2JB": 90106884, + "21 Wells Rd DY5 3TB": 90022227, + "8 Corporation Rd DY2 7PX": 90070461 +} + +load_dotenv(ENV_FILEPATH) +CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME = os.getenv("CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME", None) +CDN_BUCKET_NAME = os.getenv("CDN_BUCKET_NAME", None) + + +def list_subdirectories(directory_path): + """ + List all subdirectories within a given directory. + + :param directory_path: Path to the directory. + :return: A list of paths to the subdirectories. + """ + directory = Path(directory_path) + subdirectories = [subdir for subdir in directory.iterdir() if subdir.is_dir()] + return subdirectories + + +def list_files_in_directory(directory_path, file_extension=".jpg"): + """ + List all files with a specific extension within a given directory and its subdirectories. + + :param directory_path: Path to the directory to scan. + :param file_extension: File extension to filter by. + :return: A list of paths to the files. + """ + # Convert the directory path to a Path object if it's not already one + directory = Path(directory_path) if not isinstance(directory_path, Path) else directory_path + + # List all files of the specified type in the directory and subdirectories + file_list = [file for file in directory.rglob(f'*{file_extension}')] + + return file_list + + +def create_images(input_path, uprn): + # Define the base directory path + base_directory = f"non_intrusive_photos/{uprn}" + print(f"Creating directory: {base_directory}") # Debug: print the directory to be created + + # Need to create local directory if it doesn't exist + os.makedirs(base_directory, exist_ok=True) + + # Define output paths + thumbnail_path = os.path.join(base_directory, "thumbnail.jpg") + full_hd_path = os.path.join(base_directory, "1080p.jpg") + webp_path = os.path.join(base_directory, "webp.webp") # Save as WebP format + + # Load the image + with Image.open(input_path) as img: + # Create a thumbnail + thumbnail = img.copy() + thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS) + thumbnail.save(thumbnail_path, 'JPEG', quality=85) + + # Create a 1080p version + full_hd = img.copy() + full_hd.thumbnail((1920, 1080), Image.Resampling.LANCZOS) + full_hd.save(full_hd_path, 'JPEG', quality=90) + + # Convert to WebP for better compression + webp = img.copy() + webp.save(webp_path, 'WEBP', quality=90) + + # Return paths to the processed images + return thumbnail_path, full_hd_path, webp_path + + +def upload_to_s3(bucket_name, file_path, object_name): + s3_client = boto3.client('s3') + s3_client.upload_file(file_path, bucket_name, object_name) + print(f"Uploaded {object_name} to S3 bucket {bucket_name}") + + +def upload_photos_to_s3(bucket_name, photo_paths): + # Upload each photo + for path in photo_paths: + object_name = path.split('/')[-1] # Assuming the path format is folder/filename + upload_to_s3(bucket_name, path, object_name) + + +def generate_cdn_url(distribution_domain, object_name): + return f"https://{distribution_domain}/{object_name}" + + +def process_and_upload_images(uprn, input_image_path, bucket_name, distribution_domain): + # Create images + thumbnail, full_hd, original = create_images(input_image_path, uprn=str(uprn)) + + # Upload images + upload_photos_to_s3(bucket_name, photo_paths=[thumbnail, full_hd, original]) + + # Generate CDN links + cdn_links = [generate_cdn_url(distribution_domain, path.split('/')[-1]) for path in [thumbnail, full_hd, original]] + + # Delete local files + for path in [thumbnail, full_hd, original]: + os.remove(path) + + return cdn_links + + +def app(): + """ + This application is tasked with uploading the photos, recorded during the non-invasive surveys, to s3 and the + database. + To begin with, this app will simply read the files from the local machine, however we will come up with a more + efficient way to do this in the future. + + :return: + """ + + # List all files in the directory using pathlib + property_directories = list_subdirectories(PHOTO_DIRECTORY) + + # For each property, we want to list all of the photos in the directory + for property_dir in property_directories: + photo_files = list_files_in_directory(property_dir) + uprn = FOLDER_UPRN_LOOKUP[property_dir.name] + + # We now want to convert each file, and upload it to s3 + for photo_filepath in photo_files: + process_and_upload_images( + uprn=uprn, + input_image_path=photo_filepath, + bucket_name=CDN_BUCKET_NAME, + distribution_domain=CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME + ) diff --git a/etl/non_intrusive_surveys/photos/requirements.txt b/etl/non_intrusive_surveys/photos/requirements.txt new file mode 100644 index 00000000..2199a0b4 --- /dev/null +++ b/etl/non_intrusive_surveys/photos/requirements.txt @@ -0,0 +1,3 @@ +Pillow +boto3 +python-dotenv \ No newline at end of file diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 0e67b352..852bb11f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -91,6 +91,10 @@ DOUBLE_RADIATOR_COST = 300 FLUE_COST = 600 PIPEWORK_COST = 750 # Min cost is £500 +# This is the cost per meter squared for cavity extraction +# https://www.checkatrade.com/blog/cost-guides/cavity-wall-insulation-removal-cost/ +CAVITY_EXTRACTION_COST = 21.5 + class Costs: """ @@ -173,7 +177,7 @@ class Costs: if not self.labour_adjustment_factor: raise ValueError("Labour adjustment factor not found") - def cavity_wall_insulation(self, wall_area, material): + def cavity_wall_insulation(self, wall_area, material, is_extraction_and_refill=False): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -208,6 +212,13 @@ class Costs: # Assume a team of 2 labour_days = (labour_hours / 8) / 2 + if is_extraction_and_refill: + # bump up the cost of the work + total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area + # Additional 2 days work + labour_hours = labour_hours + (2 * 8) + labour_days = labour_days + 2 + return { "total": total_cost, "subtotal": subtotal_before_vat, diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 68fead16..5960d7be 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -149,12 +149,14 @@ class Recommendations: property_recommendations = self.insert_temp_recommendation_id(property_recommendations) # We also need to create the representative recommendations for each recommendation type - property_representative_recommendations = self.create_representative_recommendations(property_recommendations) + property_representative_recommendations = self.create_representative_recommendations( + property_recommendations, non_invasive_recommendations=self.property_instance.non_invasive_recommendations + ) return property_recommendations, property_representative_recommendations @staticmethod - def create_representative_recommendations(property_recommendations): + def create_representative_recommendations(property_recommendations, non_invasive_recommendations): """ This method will create a representative recommendation for each recommendation type In order to create a representative recommendation, we choose the recommendation that has: @@ -169,6 +171,13 @@ class Recommendations: for recommendations_by_type in property_recommendations: + # If the property was initially surveyed as filled, but the cavity was only partially filled, we don't + # want to include the cavity wall insulation recommendation in the defaults + # if (recommendations_by_type[0].get("type") == "cavity_wall_insulation") and ( + # "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations + # ): + # continue + if recommendations_by_type[0].get("type") == "mechanical_ventilation": continue @@ -238,13 +247,13 @@ class Recommendations: property_sap_predictions = all_predictions["sap_change_predictions"][ all_predictions["sap_change_predictions"]["property_id"] == str(property_instance.id) - ] + ].copy() property_heat_predictions = all_predictions["heat_demand_predictions"][ all_predictions["heat_demand_predictions"]["property_id"] == str(property_instance.id) - ] + ].copy() property_carbon_predictions = all_predictions["carbon_change_predictions"][ all_predictions["carbon_change_predictions"]["property_id"] == str(property_instance.id) - ] + ].copy() property_recommendations = recommendations[property_instance.id].copy() @@ -272,6 +281,8 @@ class Recommendations: current_epc_rating=property_instance.data["current-energy-rating"], ) + # TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are + # actually implemented expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( epc_energy_consumption=expected_heat_demand, current_epc_rating=property_instance.data["current-energy-rating"], @@ -281,6 +292,10 @@ class Recommendations: current_adjusted_energy - expected_adjusted_energy ) + # TODO: We should determine if the home is gas & electricity or just electricity + current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy) + expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy) + for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: @@ -355,4 +370,10 @@ class Recommendations: rec["heat_demand"] is None) or (rec["energy_cost_savings"] is None): raise ValueError("sap points, co2 or heat demand is missing") - return property_recommendations, current_adjusted_energy, expected_adjusted_energy + return ( + property_recommendations, + current_adjusted_energy, + expected_adjusted_energy, + current_energy_bill, + expected_energy_bill + ) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index feb2620b..20fc453c 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -113,7 +113,9 @@ class WallRecommendations(Definitions): insulation_thickness = self.property.walls["insulation_thickness"] # We check if the wall is already insulated and if so, we exit - if (insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]: + if ((insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]) and ( + "cavity_extract_and_refill" not in self.property.non_invasive_recommendations + ): return if u_value: @@ -216,15 +218,26 @@ class WallRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + is_extraction_and_refill = "cavity_extract_and_refill" in self.property.non_invasive_recommendations + cost_result = self.costs.cavity_wall_insulation( wall_area=self.property.insulation_wall_area, material=material.to_dict(), + is_extraction_and_refill=is_extraction_and_refill ) already_installed = "cavity_wall_insulation" in self.property.already_installed if already_installed: cost_result = override_costs(cost_result) + if is_extraction_and_refill: + description = f"Extract and refill cavity wall insulation with {material['description']}" + else: + description = self._make_description(material) + + # updated the new u-value with the best possible our installers have + new_u_value = max(0.31, new_u_value) + recommendations.append( { "phase": phase, @@ -237,7 +250,7 @@ class WallRecommendations(Definitions): ) ], "type": "cavity_wall_insulation", - "description": self._make_description(material), + "description": description, "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None,