diff --git a/.devcontainer/backend/Dockerfile b/.devcontainer/backend/Dockerfile index 99cd66d6..662f53b0 100644 --- a/.devcontainer/backend/Dockerfile +++ b/.devcontainer/backend/Dockerfile @@ -56,4 +56,11 @@ https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ tee /etc/apt/sources.list.d/hashicorp.list RUN apt update RUN apt-get install terraform -RUN terraform -install-autocomplete \ No newline at end of file +RUN terraform -install-autocomplete + +# Install postgres +RUN apt install -y wget gnupg2 lsb-release +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list +RUN wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +RUN apt update +RUN apt install -y postgresql-14 \ No newline at end of file diff --git a/.devcontainer/backend/requirements.txt b/.devcontainer/backend/requirements.txt index c84332dd..5cd40ced 100644 --- a/.devcontainer/backend/requirements.txt +++ b/.devcontainer/backend/requirements.txt @@ -18,6 +18,9 @@ sqlmodel pytest==9.0.2 pytest-cov==7.0.0 ipykernel>=6.25,<7 +dotenv +psycopg[binary] +pytest-postgresql # Formatting black==26.1.0 boto3-stubs \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6268360b..68e66052 100644 --- a/.gitignore +++ b/.gitignore @@ -279,4 +279,7 @@ cache/ *.png *.pptx -local_data* \ No newline at end of file +local_data* + +# pyright local config +pyrightconfig.json \ No newline at end of file diff --git a/.idea/Model.iml b/.idea/Model.iml index c6561970..1e51ede4 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -10,4 +10,7 @@ + + \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml new file mode 100644 index 00000000..60d7e26a --- /dev/null +++ b/.idea/watcherTasks.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 28e17e2a..dede3162 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -36,14 +36,12 @@ from dotenv import load_dotenv logger = setup_logger() load_dotenv(dotenv_path="../backend/.env") - # OpenAI API Key (set this in your environment variables for security) -OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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") +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") class DataRemapper: def __init__(self, standard_values, standard_map=None, max_tokens=1000): - print(f"{OPENAI_API_KEY}") """ Initialize the remapper with standard values and a predefined mapping. @@ -1298,8 +1296,8 @@ class AssetList: self.standardised_asset_list[ self.ATTRIBUTE_HAS_SOLAR ] = self.standardised_asset_list[ - self.FIND_EPC_DATA_NAMES["Solar photovoltaics"] - ] | ~self.standardised_asset_list[ + self.FIND_EPC_DATA_NAMES["Solar photovoltaics"] + ] | ~self.standardised_asset_list[ self.EPC_API_DATA_NAMES["photo-supply"] ].isin( ["0.0", 0, None, "", np.nan] @@ -1317,7 +1315,7 @@ class AssetList: property_type=( str(x[self.STANDARD_PROPERTY_TYPE]).title() if str(x[self.STANDARD_PROPERTY_TYPE]).title() - in accepted_epc_property_types + in accepted_epc_property_types else ( x[self.EPC_API_DATA_NAMES["property-type"]] if not pd.isnull( @@ -1375,9 +1373,9 @@ class AssetList: self.standardised_asset_list.apply( lambda x: estimate_perimeter( floor_area=x[self.EPC_API_DATA_NAMES["total-floor-area"]] - / x[self.ATTRIBUTE_NUMBER_OF_FLOORS], + / x[self.ATTRIBUTE_NUMBER_OF_FLOORS], num_rooms=x[self.EPC_API_DATA_NAMES["number-habitable-rooms"]] - / x[self.ATTRIBUTE_NUMBER_OF_FLOORS], + / x[self.ATTRIBUTE_NUMBER_OF_FLOORS], ), axis=1, ) @@ -1462,7 +1460,7 @@ class AssetList: year_lower_bound = ( 2007 if x[self.EPC_API_DATA_NAMES["construction-age-band"]] - == "England and Wales: 2007 onwards" + == "England and Wales: 2007 onwards" else 2012 ) @@ -1517,7 +1515,7 @@ class AssetList: age_band_matches = ( "EPC Age Band Matches Year Built" if x[self.STANDARD_YEAR_BUILT] - == int(x[self.EPC_API_DATA_NAMES["construction-age-band"]]) + == int(x[self.EPC_API_DATA_NAMES["construction-age-band"]]) else "EPC Age Band is different from Year Built" ) @@ -1547,7 +1545,7 @@ class AssetList: age_band_matches = ( "EPC Age Band Matches Year Built" if (x[self.STANDARD_YEAR_BUILT] >= float(lower_date)) - and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date)) + and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date)) else ( "EPC Age Band is older than Year Built" if x[self.STANDARD_YEAR_BUILT] > float(upper_date) @@ -1719,22 +1717,22 @@ class AssetList: if self.non_intrusives_present: if self.new_format_non_insturives_present_v2: non_intrusives_wall_filter = ( - self.standardised_asset_list["non-intrusives: Construction"] - == "CAVITY" - ) & self.standardised_asset_list["non-intrusives: Insulated"].isin( + self.standardised_asset_list["non-intrusives: Construction"] + == "CAVITY" + ) & self.standardised_asset_list["non-intrusives: Insulated"].isin( ["EMPTY", "PARTIAL", "EMPTY CAVITY"] ) else: non_intrusives_wall_filter = ( - self.standardised_asset_list["non-intrusives: Construction"] - == "CAVITY" - ) & self.standardised_asset_list["non-intrusives: Insulated"].isin( + self.standardised_asset_list["non-intrusives: Construction"] + == "CAVITY" + ) & self.standardised_asset_list["non-intrusives: Insulated"].isin( ["EMPTY", "PARTIAL"] ) elif self.old_format_non_intrusives_present: non_intrusives_wall_filter = self.standardised_asset_list[ - "non-intrusives: WFT Findings" - ].str.lower().str.strip().isin( + "non-intrusives: WFT Findings" + ].str.lower().str.strip().isin( [ "empty cavity", "partial fill", @@ -1744,18 +1742,18 @@ class AssetList: "empty cav", ] ) | ( - ( - self.standardised_asset_list["non-intrusives: WFT Findings"] - .str.lower() - .str.strip() - .str.contains("empty cavity|partial fill") - & ~self.standardised_asset_list["non-intrusives: WFT Findings"] - .astype(str) - .str.lower() - .str.strip() - .str.contains("major access issues") - ) - ) + ( + self.standardised_asset_list["non-intrusives: WFT Findings"] + .str.lower() + .str.strip() + .str.contains("empty cavity|partial fill") + & ~self.standardised_asset_list["non-intrusives: WFT Findings"] + .astype(str) + .str.lower() + .str.strip() + .str.contains("major access issues") + ) + ) else: # We set the filter to False, as we have no non-intrusives non_intrusives_wall_filter = False @@ -1767,12 +1765,12 @@ class AssetList: ) else: year_built_filter = ( - self.standardised_asset_list[self.STANDARD_YEAR_BUILT] - <= self.EMPTY_CAVITY_YEAR_THRESHOLD - ) | ( - self.standardised_asset_list["epc_year_upper_bound"] - <= self.EMPTY_CAVITY_YEAR_THRESHOLD - ) + self.standardised_asset_list[self.STANDARD_YEAR_BUILT] + <= self.EMPTY_CAVITY_YEAR_THRESHOLD + ) | ( + self.standardised_asset_list["epc_year_upper_bound"] + <= self.EMPTY_CAVITY_YEAR_THRESHOLD + ) # Criteria: # The property isn't a bedsit @@ -1813,8 +1811,8 @@ class AssetList: ] = ( ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity_has_solar" - ] + "non_intrusive_indicates_empty_cavity_has_solar" + ] & ( ~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin( ["bedsit"] @@ -1890,8 +1888,8 @@ class AssetList: .str.lower() .isin(self.EPC_NO_WALL_INSULATION_DESCRIPTIONS) | self.standardised_asset_list[self.STANDARD_WALL_CONSTRUCTION].isin( - ["uninsulated cavity"] - ) + ["uninsulated cavity"] + ) ) ###################################################### @@ -1928,8 +1926,8 @@ class AssetList: extraction_wall_filter = ( extraction_wall_filter & ~self.standardised_asset_list[ - "non-intrusives: Eligibility (Red/Yellow/Green)" - ].isin(["RED"]) + "non-intrusives: Eligibility (Red/Yellow/Green)" + ].isin(["RED"]) ) self.standardised_asset_list[ @@ -2025,26 +2023,26 @@ class AssetList: self.standardised_asset_list[ "solar_epc_data_indicates_correct_heating_system" ] = ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheat-description"] - ] - .str.lower() - .str.contains( - "air source heat pump|ground source heat pump|boiler and radiators, electric" - ) - ) | ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheat-description"] - ] - .str.lower() - .str.contains("electric storage heaters") - & ( self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheatcont-description"] + self.EPC_API_DATA_NAMES["mainheat-description"] ] - == "Controls for high heat retention storage heaters" + .str.lower() + .str.contains( + "air source heat pump|ground source heat pump|boiler and radiators, electric" + ) + ) | ( + self.standardised_asset_list[ + self.EPC_API_DATA_NAMES["mainheat-description"] + ] + .str.lower() + .str.contains("electric storage heaters") + & ( + self.standardised_asset_list[ + self.EPC_API_DATA_NAMES["mainheatcont-description"] + ] + == "Controls for high heat retention storage heaters" + ) ) - ) # If the landlord has given us the heating system, we default to that on heating upgrades. Because of the # poor heating in place, if the EPC indicates that this property had a low efficiency heating system but the @@ -2052,25 +2050,25 @@ class AssetList: self.standardised_asset_list[ "solar_epc_data_indicates_requires_heating_upgrade" ] = ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheat-description"] - ] - .str.lower() - .str.contains("electric storage heaters|room heaters") - & ( self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheatcont-description"] + self.EPC_API_DATA_NAMES["mainheat-description"] ] - != "Controls for high heat retention storage heaters" + .str.lower() + .str.contains("electric storage heaters|room heaters") + & ( + self.standardised_asset_list[ + self.EPC_API_DATA_NAMES["mainheatcont-description"] + ] + != "Controls for high heat retention storage heaters" + ) + ) & ( + ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin( + ["district heating", "communal heating", "communal gas boiler"] + ) + & ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM] + .astype(str) + .str.contains("gas ") ) - ) & ( - ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin( - ["district heating", "communal heating", "communal gas boiler"] - ) - & ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM] - .astype(str) - .str.contains("gas ") - ) # Basic check - both of the previous two shouldn't be true simultaneously if ( @@ -2150,8 +2148,8 @@ class AssetList: self.standardised_asset_list[ "solar_non_intrusives_walls_insulated" ] = self.standardised_asset_list[ - "non-intrusives: WFT Findings" - ].str.lower().str.strip().isin( + "non-intrusives: WFT Findings" + ].str.lower().str.strip().isin( [ "retro drilled", "retro filled", @@ -2160,8 +2158,8 @@ class AssetList: "retro drilled and filled", ] ) | self.standardised_asset_list[ - "non-intrusives: WFT Findings" - ].str.lower().str.strip().str.contains( + "non-intrusives: WFT Findings" + ].str.lower().str.strip().str.contains( "retro drilled" ) else: @@ -2178,14 +2176,19 @@ class AssetList: ) self.standardised_asset_list["solar_epc_walls_insulated"] = ( - self.standardised_asset_list[self.EPC_API_DATA_NAMES["walls-description"]] - .str.lower() - .str.contains("|".join(self.EPC_INSULATED_WALLS_SUBSTRINGS)) - ) | ( - self.standardised_asset_list["walls_u_value"].apply( - lambda x: x <= 0.7 if not pd.isnull(x) else False - ) - ) + self.standardised_asset_list[ + self.EPC_API_DATA_NAMES[ + "walls-description"]] + .str.lower() + .str.contains("|".join( + self.EPC_INSULATED_WALLS_SUBSTRINGS)) + ) | ( + self.standardised_asset_list[ + "walls_u_value"].apply( + lambda x: x <= 0.7 if not pd.isnull( + x) else False + ) + ) roof_data = [] for desc in self.standardised_asset_list[ @@ -2227,20 +2230,20 @@ class AssetList: self.standardised_asset_list[ "solar_epc_loft_needs_topup" ] = self.standardised_asset_list[ - self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS - ].apply( + self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS + ].apply( lambda x: int(x) < 200 if str(x).isdigit() else False ) | ( - ( - self.standardised_asset_list["is_loft"] - | self.standardised_asset_list["is_pitched"] + ( + self.standardised_asset_list["is_loft"] + | self.standardised_asset_list["is_pitched"] + ) + & ( + self.standardised_asset_list[ + self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS + ].isin(["below average", "none"]) + ) ) - & ( - self.standardised_asset_list[ - self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS - ].isin(["below average", "none"]) - ) - ) self.standardised_asset_list["epc_has_floor_recommendation"] = ( self.standardised_asset_list["epc_has_floor_recommendation"].fillna(False) @@ -2249,15 +2252,16 @@ class AssetList: # Check if the boiler is electric # We check if it contains both the terms boiler & electric self.standardised_asset_list["has_electric_boiler"] = ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["mainheat-description"] - ] - .str.lower() - .isin(["boiler and radiators, electric"]) - ) | ( - self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM] - == "electric boiler" - ) + self.standardised_asset_list[ + self.EPC_API_DATA_NAMES["mainheat-description"] + ] + .str.lower() + .isin(["boiler and radiators, electric"]) + ) | ( + self.standardised_asset_list[ + self.STANDARD_HEATING_SYSTEM] + == "electric boiler" + ) #################################### # Check solar eligibility @@ -2395,11 +2399,11 @@ class AssetList: empty_cavity_map = { "non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE - + ": ", + + ": ", "non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property " - "already has solar: ", + "already has solar: ", "non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, " - f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ", + f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ", } for variable, description in empty_cavity_map.items(): self.standardised_asset_list["cavity_reason"] = np.where( @@ -2415,8 +2419,8 @@ class AssetList: ( self.standardised_asset_list["epc_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity" - ] + "non_intrusive_indicates_empty_cavity" + ] & ( self.standardised_asset_list["non-intrusives: WFT Findings"] .str.lower() @@ -2441,8 +2445,8 @@ class AssetList: ( self.standardised_asset_list["epc_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity" - ] + "non_intrusive_indicates_empty_cavity" + ] & self.standardised_asset_list[ "non_intrusive_indicates_cavity_extraction" ] @@ -2457,8 +2461,8 @@ class AssetList: ( self.standardised_asset_list["epc_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity" - ] + "non_intrusive_indicates_empty_cavity" + ] & ( self.standardised_asset_list["non-intrusives: Insulated"] == "RETRO DRILLED" @@ -2474,8 +2478,8 @@ class AssetList: ( self.standardised_asset_list["epc_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity" - ] + "non_intrusive_indicates_empty_cavity" + ] & ( self.standardised_asset_list["non-intrusives: Insulated"] == "FILLED AT BUILD" @@ -2491,8 +2495,8 @@ class AssetList: ( self.standardised_asset_list["epc_indicates_empty_cavity"] & ~self.standardised_asset_list[ - "non_intrusive_indicates_empty_cavity" - ] + "non_intrusive_indicates_empty_cavity" + ] & pd.isnull(self.standardised_asset_list["cavity_reason"]) ), f"{self.EPC_EMPTY}: " + self.standardised_asset_list["SAP Category"], @@ -2636,7 +2640,7 @@ class AssetList: identified_work = self.standardised_asset_list[ ~pd.isnull(self.standardised_asset_list["cavity_reason"]) | ~pd.isnull(self.standardised_asset_list["solar_reason"]) - ][self.DOMNA_PROPERTY_ID].values + ][self.DOMNA_PROPERTY_ID].values if self.DOMNA_PROPERTY_ID in self.outcomes.columns: self.outcomes_for_output = self.outcomes[ @@ -2671,12 +2675,12 @@ class AssetList: blocks_of_flats = self.standardised_asset_list[ self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] == "block of flats" - ] + ] non_blocks_of_flats = self.standardised_asset_list[ self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "block of flats" - ] + ] # Produce some aggregate figures self.work_type_figures = { @@ -2719,7 +2723,7 @@ class AssetList: blocks = self.standardised_asset_list[ self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] == "block of flats" - ].copy() + ].copy() if blocks.empty: return @@ -2856,7 +2860,7 @@ class AssetList: self.standardised_asset_list = self.standardised_asset_list[ self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "block of flats" - ] + ] self.standardised_asset_list = pd.concat( [self.standardised_asset_list, expanded_blocks], ignore_index=True @@ -2936,7 +2940,7 @@ class AssetList: # find any block refs with more than 50% emptires viable_empty_blocks = self.block_analysis_df[ self.block_analysis_df["Percentage of Empties"] >= 0.50 - ] + ] if not viable_empty_blocks.empty: project_code_lookup = viable_empty_blocks[["Block Reference"]].copy() @@ -3175,7 +3179,7 @@ class AssetList: contact_details = pd.read_excel(local_filepath, sheet_name=sheet_name)[ [self.contact_detail_fields["landlord_property_id"]] + details_colnames - ] + ] contact_details = contact_details[ ~pd.isnull( contact_details[self.contact_detail_fields["landlord_property_id"]] @@ -3568,10 +3572,13 @@ class AssetList: "Non-Intrusives: Date Checked ": date_of_inspections, "Non-Intrusives: Wall Type ": non_intrusives_construction, "Non-intrusives: Insulation ": non_intrusives_insulated, - "Non-intrusives: Insulation Material ": non_intrusives_insulation_material, - "Non-Intrusives: CIGA Check Required ": non_intrusives_ciga_check_required, + "Non-intrusives: Insulation Material ": + non_intrusives_insulation_material, + "Non-Intrusives: CIGA Check Required ": + non_intrusives_ciga_check_required, "Non-Intrusives: PV Access Issues ": non_intrusives_pv_access, - "Non-Intrusives: Roof Orientation ": non_intrusives_roof_orientation, + "Non-Intrusives: Roof Orientation ": + non_intrusives_roof_orientation, "Non-Intrusives: Surveyor Notes ": non_intrusives_surveyor_notes, "Non-Intrusives: Surveyor Name ": non_intrusives_surveyor_name, "CIGA: Date Requested ": None, # TODO: Don't have this for the moment @@ -3748,8 +3755,8 @@ class AssetList: # We compare address line 1 to full address if any( df[self.STANDARD_FULL_ADDRESS] - .str.lower() - .str.contains(row["Address Line 1"].lower(), na=False) + .str.lower() + .str.contains(row["Address Line 1"].lower(), na=False) ): df = df[ df[self.STANDARD_FULL_ADDRESS] @@ -3989,7 +3996,7 @@ class AssetList: matched = matched[ matched["houseno"].astype(str) == house_no_to_match - ] + ] if matched.shape[0] == 1: lookup_i.append( { @@ -4014,7 +4021,7 @@ class AssetList: )[0] matched = matched[ matched[self.STANDARD_FULL_ADDRESS] == best_match - ] + ] lookup_i.append( { "row_id": x["row_id"], @@ -4325,7 +4332,7 @@ class AssetList: df = self.standardised_asset_list[ self.standardised_asset_list[self.STANDARD_LANDLORD_PROPERTY_ID] == row[master_id_colnames[idx]] - ] + ] if df.shape[0] == 1: matched.append( { @@ -4431,7 +4438,7 @@ class AssetList: )[1] ) > 90 - ] + ] if df.shape[0] == 0: unmatched.append(row["row_id"]) @@ -4439,8 +4446,8 @@ class AssetList: if any( df[self.STANDARD_FULL_ADDRESS] - .str.lower() - .str.contains( + .str.lower() + .str.contains( " ".join( [row[house_no_col], row["Street / Block Name"]] ).lower() @@ -4467,7 +4474,7 @@ class AssetList: row[property_type_col].split(" ")[-1].lower() ) & (df[self.STANDARD_PROPERTY_TYPE] != "block of flats") - ] + ] if df.shape[0] != 1: # We have multiple matches - it's likely because the landlord has a duplicate diff --git a/asset_list/app.py b/asset_list/app.py index 3e492118..a97bb8e0 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -18,7 +18,6 @@ EPC_AUTH_TOKEN = os.getenv( "EPC_AUTH_TOKEN", ) - OPENAI_API_KEY = os.getenv( "OPENAI_API_KEY", ) @@ -74,24 +73,25 @@ def app(): Property UPRN """ - data_folder = "/workspaces/model/asset_list" - data_filename = "assests.xlsx" + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lifespace Rentals/Missed" + # data_filename = "For Modelling - Final - reviewed.xlsx" + data_filename = "Missed Properties - with address.xlsx" sheet_name = "Sheet1" postcode_column = "Postcode" - address1_column = "Address" - address1_method = "house_number_extraction" - fulladdress_column = None - address_cols_to_concat = ["Address"] + address1_column = "address1" + address1_method = None + fulladdress_column = "address1" + address_cols_to_concat = [] missing_postcodes_method = None landlord_year_built = None landlord_os_uprn = "UPRN" - landlord_property_type = "Archetype" - landlord_built_form = "Bedroom Count" - landlord_wall_construction = "Wall Insulation Type" - landlord_roof_construction = "Roof Type" - landlord_heating_system = "Boiler Type" + landlord_property_type = "Type" + landlord_built_form = None + landlord_wall_construction = None + landlord_roof_construction = None + landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "Tab" + landlord_property_id = "Reference" landlord_sap = None outcomes_filename = None outcomes_sheetname = None @@ -243,7 +243,7 @@ def app(): if skip is not None and not force_retrieve_data: if i <= skip: continue - chunk = asset_list.standardised_asset_list[i : i + chunk_size] + chunk = asset_list.standardised_asset_list[i: i + chunk_size] epc_data_chunk, errors_chunk, no_epc_chunk = get_data( df=chunk, row_id_name=asset_list.DOMNA_PROPERTY_ID, @@ -386,7 +386,7 @@ def app(): # Retrieve just the data we need epc_df = epc_df[ [asset_list.DOMNA_PROPERTY_ID] + list(asset_list.EPC_API_DATA_NAMES.keys()) - ].rename(columns=asset_list.EPC_API_DATA_NAMES) + ].rename(columns=asset_list.EPC_API_DATA_NAMES) # Look for columns not in the find my EPC data, which will have happened if we didn't # retrieve it in the first place @@ -403,16 +403,12 @@ def app(): find_my_epc_data[ [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"] + list(asset_list.FIND_EPC_DATA_NAMES.keys()) - ].rename(columns=asset_list.FIND_EPC_DATA_NAMES), + ].rename(columns=asset_list.FIND_EPC_DATA_NAMES), how="left", on=asset_list.DOMNA_PROPERTY_ID, ) asset_list.merge_data(epc_df) - # asset_list.standardised_asset_list = asset_list.standardised_asset_list[ - # asset_list.standardised_asset_list["domna_full_address"] - # != "120 Airdrie Crescent, Burnley, Lancashire" - # ] asset_list.extract_attributes() asset_list.identify_worktypes() @@ -426,27 +422,6 @@ def app(): os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + " - Standardised.xlsx" ) - # Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data - - # Determine inspections priority - # solar_jobs = asset_list.standardised_asset_list[~pd.isnull(asset_list.standardised_asset_list["solar_reason"])][ - # "domna_postcode"].unique() - # asset_list.standardised_asset_list["in_solar_area"] = asset_list.standardised_asset_list["domna_postcode"].isin( - # solar_jobs - # ) - # # Same for cav - # cavity_jobs = asset_list.standardised_asset_list[ - # ~pd.isnull(asset_list.standardised_asset_list["cavity_reason"]) - # ]["domna_postcode"].unique() - # asset_list.standardised_asset_list["in_cavity_area"] = asset_list.standardised_asset_list["domna_postcode"].isin( - # cavity_jobs - # ) - # # We prioritise properties that are in solar areas and cavity areas - # import numpy as np - # asset_list.standardised_asset_list["inspection_priority"] = np.where( - # asset_list.standardised_asset_list["in_solar_area"] | asset_list.standardised_asset_list["in_cavity_area"], - # 1, 2 - # ) with pd.ExcelWriter(filename) as writer: asset_list.standardised_asset_list.to_excel( diff --git a/asset_list/mappings/built_form.py b/asset_list/mappings/built_form.py index d6466539..4842450d 100644 --- a/asset_list/mappings/built_form.py +++ b/asset_list/mappings/built_form.py @@ -528,6 +528,107 @@ BUILT_FORM_MAPPINGS = { 'House: Semi Detached: Top Floor': 'semi-detached', 'House: End Terrace: Ground Floor': 'end-terrace', 'Maisonette: Enclosed End Terrace: Mid Floor': 'enclosed end-terrace', - 'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace' + 'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace', + '2 BED MID TERRACED HOUSE': 'mid-terrace', + '4 BED SEMI DETACHED-PARLOURED': 'semi-detached', + '2 BED END TERRACED HOUSE': 'end-terrace', + '3 BED MID TERRACED HOUSE': 'mid-terrace', + '3 BED SEMI DETACHED HOUSE': 'semi-detached', + '3 BED MID TERRACE - PARLOURED': 'mid-terrace', + '3 BED END TERRACE - PARLOURED': 'end-terrace', + '4 BED+ END TERRACED HOUSE': 'end-terrace', + '3 BED END TERRACED HOUSE': 'end-terrace', + '3 BED SEMI DETACHED-PARLOURED': 'semi-detached', + '4 BED+ END TERRACE - PARLOURED': 'end-terrace', + '2 BED SEMI DETACHED HOUSE': 'semi-detached', + '3 BED DETACHED HOUSE': 'detached', + '2 BED GRD FLR COTT FLT-CNT STR': 'ground floor', + '2 BED 1ST FLOOR WALKUP FLAT': 'mid-floor', + '1 BED GRD FL COTT FLAT-OWN ENT': 'ground floor', + '1 BED 1ST FL WALK UP DECK ACC': 'mid-floor', + '2 BED MAISONETTE UPPER COM ENT': 'mid-floor', + '2 BED GRD FLR COTT FLT OWN ENT': 'ground floor', + '1 BED BUNGALOW': 'unknown', + '2 BED GRD FL COTT FLT-OWN ENTR': 'ground floor', + '1 BED 1ST FL COTT FLT-CNT STR': 'mid-floor', + '1 BED GRD FL WALK UP OWN ENT': 'ground floor', + '1 BED GRD FLOOR WALKUP FLAT': 'ground floor', + '2 BED GRD FLOOR WALKUP FLAT': 'ground floor', + '2 BED 1ST FLR FLT-SHELTERED': 'mid-floor', + '2 BED BUNGALOW': 'unknown', + '2 BED GRD FLR COTT FLT(P)-1950': 'ground floor', + + 'Ground Floor Front Left': 'ground floor', + 'End-Terrace House': 'end-terrace', + 'Ground floor': 'ground floor', + 'Ground Floor Front Right': 'ground floor', + 'End Terrace (GII List)': 'end-terrace', + 'Semi Detached House': 'semi-detached', + 'Ground Floor Right': 'ground floor', + 'PB Ground Floor Flat': 'ground floor', + 'Basement and Ground Floor': 'ground floor', + 'Semi-detached bungalow': 'detached', + 'Detached Cottage': 'detached', + 'Lower & Ground Floor': 'ground floor', + 'Ground FLoor Flat': 'ground floor', + 'ground floor': 'ground floor', + 'Ground Floor Left': 'ground floor', + 'Semi-detached House': 'detached', + 'Basement & Lower Ground': 'basement', + 'Semi-Detached House': 'detached', + 'Ground floor flat -': 'ground floor', + 'Basement Flat': 'basement', + 'semi-detached bungalow': 'semi-detached', + 'Lower Ground Floor Flat': 'ground floor', + 'Ground floor Flat': 'ground floor', + 'Ground Floor flat': 'ground floor', + 'Ground': 'ground floor', + 'Semi detached Bungalow': 'semi-detached', + 'ground floor flat': 'ground floor', + 'Mid terrace House': 'mid-terrace', + 'Raised Ground Floor': 'ground floor', + 'Basement Floor': 'basement', + 'Second floor flat': 'mid-floor', + 'Fourth Floor Flat': 'mid-floor', + 'First/Second Maisonette': 'mid-floor', + 'Ground/First': 'ground floor', + 'First and Second Floor': 'mid-floor', + 'Terrace House': 'mid-terrace', + '1st/2nd Floor Maisonette': 'mid-floor', + 'Semi-det House': 'semi-detached', + 'First': 'mid-floor', + 'Ground & First Floor': 'ground floor', + 'End of Terrace House': 'end-terrace', + '2nd Floor Purpose Built': 'mid-floor', + 'First/Second Floor Maison': 'mid-floor', + 'GFF purpose built': 'ground floor', + 'Second': 'mid-floor', + 'Semi-det House (GII List)': 'semi-detached', + '3rd and 4th Floor': 'mid-floor', + 'First Floor flat': 'mid-floor', + 'Mid-Terrace House': 'mid-terrace', + '1st & 2nd Floors': 'mid-floor', + 'Ground/first floor': 'ground floor', + 'FFF purpose built': 'mid-floor', + 'Second floor': 'mid-floor', + 'Second/Third floor': 'mid-floor', + 'First floor Flat': 'mid-floor', + 'First floor': 'mid-floor', + 'Lower Ground Flat': 'basement', + 'First Floor Rear Flat': 'mid-floor', + 'First & Second Floor': 'mid-floor', + 'Ground & Lower Ground': 'basement', + 'First Floor Rear': 'mid-floor', + 'First & Second': 'mid-floor', + 'First Floor Front': 'mid-floor', + 'First & Second Floors': 'mid-floor', + 'First/Second Floor': 'mid-floor', + 'Sem-detach house': 'semi-detached', + 'Second Floor Flat (Top)': 'top-floor', + '3 FloorTerrace House': 'mid-terrace', + 'First floor flat': 'mid-floor', + 'First & Second Floor Flat': 'mid-floor', + 'First Floor Purpose Built': 'mid-floor', + 'Purpose built First Floor': 'mid-floor', } diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index 272d6279..5f962108 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -498,6 +498,23 @@ HEATING_MAPPINGS = { 'Boiler: A rated Combi, System 2: Boiler: A rated Combi': 'gas combi boiler', 'System 2: Boiler: A rated Regular Boiler, Boiler: A rated Regular Boiler': 'gas boiler, radiators', - 'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler' + 'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler', + + 'IDEAL ISAR HE30': 'gas combi boiler', + 'WORCESTER GREENSTAR 25 SI': 'gas combi boiler', + 'POTTERTON PROMAX COMBI 28 HE PLUS': 'gas combi boiler', + 'WORCESTER GREENSTAR 28I JUNIOR': 'gas combi boiler', + 'BAXI ASSURE 25 COMBI': 'gas combi boiler', + 'POTTERTON PROMAX COMBI 28 HE PLUS A': 'gas combi boiler', + 'WORCESTER GREENSTAR 30 SI': 'gas combi boiler', + 'POTTERTON SUPRIMA 40L': 'gas boiler, radiators', + 'POTTERTON ASSURE 30 COMBI': 'gas combi boiler', + 'POTTERTON PROMAX 28 COMBI ERP': 'gas combi boiler', + 'BAXI ASSURE 30 COMBI': 'gas combi boiler', + 'POTTERTON PROMAX 18 SYSTEM ERP': 'gas boiler, radiators', + 'POTTERTON PROMAX COMBI 33 HE PLUS A': 'gas combi boiler', + 'POTTERTON SUPRIMA 40 HE': 'gas boiler, radiators', + 'FERROLI MODENA 102': 'gas boiler, radiators', + 'POTTERTON PROMAX COMBI 24 HE PLUS A': 'gas combi boiler' } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index 177a7549..71788c25 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -444,6 +444,9 @@ PROPERTY_MAPPING = { 'Warden Bungalow': 'bungalow', 'Warden Flat': 'flat', 'Upper Floor Flat': 'flat', - 'Extracare Scheme': 'other' + 'Extracare Scheme': 'other', + + 'SHELTERED': 'unknown', + 'PARLOUR': 'unknown', } diff --git a/asset_list/mappings/roof.py b/asset_list/mappings/roof.py index 70cc8742..192238e0 100644 --- a/asset_list/mappings/roof.py +++ b/asset_list/mappings/roof.py @@ -320,6 +320,8 @@ ROOF_CONSTRUCTION_MAPPINGS = { 'Pitched (slates or tiles) access to loft, 100mm': 'pitched insulated', 'Pitched (slates or tiles) no loft access, 200mm': 'pitched insulated', 'Pitched (slates or tiles) access to loft, 200mm': 'pitched insulated', - 'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation' + 'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation', + + 'Pitched roofs': 'pitched unknown insulation', } diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index 1a252b33..c369204d 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -369,6 +369,9 @@ WALL_CONSTRUCTION_MAPPINGS = { 'Solid Brick, As built': 'solid brick unknown insulation', 'System built, As built': 'system built unknown insulation', 'Timber frame, As built': 'timber frame unknown insulation', - 'Cavity, As built': 'cavity unknown insulation' + 'Cavity, As built': 'cavity unknown insulation', + 'FILLED CAVITY': 'filled cavity', + 'EXTERNAL': 'insulated solid brick', + 'AS BUILT': 'other' } diff --git a/backend/app/db/base.py b/backend/app/db/base.py new file mode 100644 index 00000000..fa2b68a5 --- /dev/null +++ b/backend/app/db/base.py @@ -0,0 +1,5 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass diff --git a/backend/app/db/models/addresses.py b/backend/app/db/models/addresses.py index 51e9540f..a813f58d 100644 --- a/backend/app/db/models/addresses.py +++ b/backend/app/db/models/addresses.py @@ -7,9 +7,7 @@ from sqlalchemy import ( func, UniqueConstraint, ) -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class PostcodeSearch(Base): diff --git a/backend/app/db/models/condition.py b/backend/app/db/models/condition.py index 77043366..96f601a7 100644 --- a/backend/app/db/models/condition.py +++ b/backend/app/db/models/condition.py @@ -7,12 +7,12 @@ from sqlalchemy import ( String, Enum as SqlEnum, ) -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import relationship from backend.condition.domain.aspect_type import AspectType from backend.condition.domain.element_type import ElementType -Base = declarative_base() +from backend.app.db.base import Base ElementTypeDb = SqlEnum( ElementType, diff --git a/backend/app/db/models/energy_assessments.py b/backend/app/db/models/energy_assessments.py index 46912c9b..41967903 100644 --- a/backend/app/db/models/energy_assessments.py +++ b/backend/app/db/models/energy_assessments.py @@ -1,10 +1,8 @@ -from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.dialects.postgresql import ENUM as PgEnum import enum from datetime import datetime - -Base = declarative_base() +from backend.app.db.base import Base +from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey +from sqlalchemy.dialects.postgresql import ENUM as PgEnum class EnergyAssessment(Base): @@ -190,7 +188,7 @@ class EnergyAssessmentDocuments(Base): id = Column(BigInteger, primary_key=True, autoincrement=True) uprn = Column(BigInteger, nullable=False) energy_assessment_id = Column(BigInteger, ForeignKey('energy_assessments.id'), nullable=False) - document_type = Column(PgEnum(DocumentTypeEnum, name="document_type", create_type=False), nullable=False) + document_type = Column(PgEnum(DocumentTypeEnum, name="document_type"), nullable=False) document_location = Column(Text, nullable=False) uploaded_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) scenario_id = Column(BigInteger, ForeignKey('energy_assessment_scenarios.id'), nullable=True) diff --git a/backend/app/db/models/epc.py b/backend/app/db/models/epc.py index 5a216040..ff0b40a0 100644 --- a/backend/app/db/models/epc.py +++ b/backend/app/db/models/epc.py @@ -4,11 +4,8 @@ from sqlalchemy import ( String, JSON, TIMESTAMP, - UniqueConstraint, ) -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class EpcStore(Base): diff --git a/backend/app/db/models/funding.py b/backend/app/db/models/funding.py index a7417e14..19e8203d 100644 --- a/backend/app/db/models/funding.py +++ b/backend/app/db/models/funding.py @@ -3,20 +3,17 @@ import enum from sqlalchemy import ( Column, Integer, - String, Float, Enum, TIMESTAMP, BigInteger, ForeignKey, ) -from sqlalchemy.orm import declarative_base from sqlalchemy.sql import func +from backend.app.db.base import Base from backend.app.db.models.recommendations import PlanModel from backend.app.db.models.materials import MaterialType, Material -Base = declarative_base() - class SchemeEnum(enum.Enum): eco4 = "eco4" diff --git a/backend/app/db/models/inspections.py b/backend/app/db/models/inspections.py index 473f8a02..2a42f589 100644 --- a/backend/app/db/models/inspections.py +++ b/backend/app/db/models/inspections.py @@ -9,11 +9,9 @@ from sqlalchemy import ( Enum, ForeignKey, ) -from sqlalchemy.ext.declarative import declarative_base +from backend.app.db.base import Base from backend.app.db.models.portfolio import PropertyModel -Base = declarative_base() - # ------------------------------------------------------------------- # ENUM DEFINITIONS (equivalent to drizzle pgEnum calls) diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 8a524491..101ac021 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -1,10 +1,9 @@ import enum from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP, Boolean -from sqlalchemy.orm import declarative_base from sqlalchemy.sql import func -Base = declarative_base() +from backend.app.db.base import Base class MaterialType(enum.Enum): diff --git a/backend/app/db/models/non_intrusive_surveys.py b/backend/app/db/models/non_intrusive_surveys.py index bc2d8adc..bbfb7a54 100644 --- a/backend/app/db/models/non_intrusive_surveys.py +++ b/backend/app/db/models/non_intrusive_surveys.py @@ -1,7 +1,5 @@ from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer -from sqlalchemy.orm import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class NonIntrusiveSurvey(Base): diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index f6a99a97..9eb26597 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -4,6 +4,7 @@ import datetime from sqlalchemy import ( Column, Integer, + BigInteger, Text, Boolean, Float, @@ -12,12 +13,10 @@ from sqlalchemy import ( ForeignKey, CheckConstraint, ) -from sqlalchemy.ext.declarative import declarative_base +from backend.app.db.base import Base from backend.app.db.models.users import UserModel # noqa from backend.app.db.models.materials import MaterialType -Base = declarative_base() - class PortfolioStatus(enum.Enum): SCOPING = "scoping" @@ -32,7 +31,7 @@ class PortfolioStatus(enum.Enum): NEEDS_REVIEW = "needs review" -class PortfolioGoal(enum.Enum): # TODO: Move to domain? +class PortfolioGoal(enum.Enum): # TODO: Move to domain? VALUATION_IMPROVEMENT = "Valuation Improvement" INCREASING_EPC = "Increasing EPC" REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions" @@ -116,9 +115,9 @@ class PropertyModel(Base): id = Column(Integer, primary_key=True, autoincrement=True) portfolio_id = Column(Integer, ForeignKey("portfolio.id"), nullable=False) creation_status = Column(Enum(PropertyCreationStatus), nullable=False) - uprn = Column(Integer) + uprn = Column(BigInteger) landlord_property_id = Column(Text) - building_reference_number = Column(Integer) + building_reference_number = Column(BigInteger) status = Column( Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]), nullable=False, diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index 538b11e3..27d03303 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -1,3 +1,4 @@ +import enum from typing import Iterable, List, NamedTuple, Optional, Type from sqlalchemy import ( Column, @@ -9,17 +10,15 @@ from sqlalchemy import ( ForeignKey, Enum, ) -from sqlalchemy.orm import declarative_base, Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql import func from datetime import datetime +from backend.app.db.base import Base from backend.app.db.models.portfolio import Portfolio, PortfolioGoal, PropertyModel from backend.app.db.models.materials import Material from backend.app.db.models.portfolio import Epc from datatypes.enums import QuantityUnits -import enum - -Base = declarative_base() def portfolio_goal_values(enum_cls: Type[PortfolioGoal]) -> List[str]: @@ -55,19 +54,47 @@ class Recommendation(Base): class RecommendationMaterials(Base): __tablename__ = "recommendation_materials" - id = Column(BigInteger, primary_key=True, autoincrement=True) - recommendation_id = Column( - BigInteger, ForeignKey("recommendation.id"), nullable=False + id: Mapped[int] = mapped_column( + BigInteger, primary_key=True, autoincrement=True ) - material_id = Column(BigInteger, ForeignKey(Material.id), nullable=False) - created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) - depth = Column(Float, nullable=False) - quantity = Column(Float, nullable=False) - quantity_unit = Column( + + recommendation_id: Mapped[int] = mapped_column( + BigInteger, + ForeignKey("recommendation.id"), + nullable=False, + ) + + material_id: Mapped[int] = mapped_column( + BigInteger, + ForeignKey(Material.id), + nullable=False, + ) + + created_at: Mapped[datetime] = mapped_column( + TIMESTAMP, + nullable=False, + server_default=func.now(), + ) + + depth: Mapped[float] = mapped_column( + Float, + nullable=False, + ) + + quantity: Mapped[float] = mapped_column( + Float, + nullable=False, + ) + + quantity_unit: Mapped[QuantityUnits] = mapped_column( Enum(QuantityUnits, values_callable=lambda x: [e.value for e in x]), nullable=False, ) - estimated_cost = Column(Float, nullable=False) + + estimated_cost: Mapped[float] = mapped_column( + Float, + nullable=False, + ) class PlanTypeEnum(enum.Enum): # TODO: move this to domain? diff --git a/backend/app/db/models/solar.py b/backend/app/db/models/solar.py index 88372bd3..dc1846f3 100644 --- a/backend/app/db/models/solar.py +++ b/backend/app/db/models/solar.py @@ -2,9 +2,7 @@ import datetime import pytz from enum import Enum as PyEnum from sqlalchemy import Column, Integer, Float, DateTime, JSON, BigInteger, ForeignKey, Enum, Boolean -from sqlalchemy.ext.declarative import declarative_base - -Base = declarative_base() +from backend.app.db.base import Base class Solar(Base): diff --git a/backend/app/db/models/users.py b/backend/app/db/models/users.py index 6e243815..7952b9b7 100644 --- a/backend/app/db/models/users.py +++ b/backend/app/db/models/users.py @@ -1,8 +1,6 @@ from sqlalchemy import Column, Integer, String, DateTime -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import func - -Base = declarative_base() +from backend.app.db.base import Base class UserModel(Base): diff --git a/backend/app/db/models/whlg.py b/backend/app/db/models/whlg.py index 29d907e4..5c5b7172 100644 --- a/backend/app/db/models/whlg.py +++ b/backend/app/db/models/whlg.py @@ -1,4 +1,3 @@ -import uuid from typing import Optional from sqlmodel import SQLModel, Field @@ -12,4 +11,4 @@ class Whlg(SQLModel, table=True): index=True, ) - postcode: str = Field(nullable=False) \ No newline at end of file + postcode: str = Field(nullable=False) diff --git a/backend/export/README.md b/backend/export/README.md new file mode 100644 index 00000000..b5715ced --- /dev/null +++ b/backend/export/README.md @@ -0,0 +1,169 @@ +# ๐Ÿงช Running Tests in PyCharm (macOS + pytest-postgresql) + +Our test suite uses `pytest` and `pytest-postgresql`, which +automatically spins up a temporary PostgreSQL instance. + +On Linux (including GitHub Actions), PostgreSQL binaries are installed +in standard system locations.\ +On macOS (Homebrew), they are not --- so PyCharm needs a small +configuration tweak to locate `pg_ctl`. + +This guide explains how to run and debug tests locally in PyCharm +without modifying test code. + +------------------------------------------------------------------------ + +## โœ… Prerequisites + +### Devcontainer + +Postgres install is included in the devcontainer, so no additional setup is needed. + +Running + +```bash +make test +``` + +Will instigate the test suite, which will automatically start a temporary PostgreSQL instance. + +### Local MacOS + +1. Install PostgreSQL via Homebrew: + +``` bash +brew install postgresql +``` + +2. Confirm `pg_ctl` exists: + +``` bash +which pg_ctl +``` + +Typical output: + + /opt/homebrew/bin/pg_ctl + +------------------------------------------------------------------------ + +# ๐Ÿš€ Running Tests in PyCharm + +## Step 1 --- Create a PyCharm pytest Run Configuration + +1. Open the test file. +2. Click the green โ–ถ next to the test. +3. Choose **"Edit Run Configuration..."** + +You should see something like: + +- **Target:** `backend/export/tests/test_export.py` +- **Working directory:** Project root (e.g.`Model/`) + +------------------------------------------------------------------------ + +## Step 2 --- Add Required Override (macOS Only) + +In the Run Configuration: + +### โžœ "Additional Arguments" + +Add: + + --override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl + +This tells `pytest-postgresql` where `pg_ctl` lives on macOS. + +Without this, PyCharm may fail with: + + ExecutableMissingException: Could not found pg_config executable + +------------------------------------------------------------------------ + +## Step 3 --- Run or Debug + +You can now: + +- Click โ–ถ Run\ +- Click ๐Ÿž Debug\ +- Set breakpoints normally + +The temporary PostgreSQL instance will start automatically. + +------------------------------------------------------------------------ + +# ๐Ÿ” Why This Is Needed + +`pytest-postgresql` defaults to a Linux-style path: + + /usr/lib/postgresql//bin/pg_ctl + +That path exists on Ubuntu (CI), but not on macOS. + +On macOS, Homebrew installs PostgreSQL in: + + /opt/homebrew/bin/ + +The `--override-ini` flag safely overrides the executable path +**locally**, without modifying: + +- test files\ +- `conftest.py`\ +- `pytest.ini`\ +- CI configuration + +This ensures: + +- โœ… Tests still work in GitHub Actions\ +- โœ… Tests still work for Linux users\ +- โœ… macOS developers can debug in PyCharm\ +- โœ… No repository-specific hacks are required + +------------------------------------------------------------------------ + +# ๐Ÿ›  Optional: Using a Local `.env` File + +If you prefer not to hardcode the override in the run configuration: + +1. Create a local file: + +```{=html} + +``` + + .env.local + +2. Add: + +```{=html} + +``` + + PYTEST_ADDOPTS=--override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl + +3. In PyCharm: + - Open the Run Configuration + - Add `.env.local` under **"Paths to .env files"** + +------------------------------------------------------------------------ + +# ๐Ÿงช Running Tests via Terminal (Recommended for CI Parity) + +For normal execution outside PyCharm: + +``` bash +make test +``` + +These already work without additional configuration. + +------------------------------------------------------------------------ + +# ๐Ÿง  Summary + +Environment Works Without Override? Needs `--override-ini`? + ------------------------ ------------------------- ------------------------- +GitHub Actions (Linux) โœ… Yes โŒ No +Linux local โœ… Yes โŒ No +macOS terminal (tox) โœ… Yes โŒ No +macOS PyCharm debugger โŒ No โœ… Yes diff --git a/backend/export/property_scenarios/db_functions.py b/backend/export/property_scenarios/db_functions.py new file mode 100644 index 00000000..e9b3d7e3 --- /dev/null +++ b/backend/export/property_scenarios/db_functions.py @@ -0,0 +1,227 @@ +from typing import List, Any, Dict, Optional, Tuple, Sequence +import pandas as pd +from sqlalchemy import select +from sqlalchemy.orm import Session +from sqlalchemy.engine import Row +from collections import defaultdict + +from backend.app.db.models.recommendations import ( + Recommendation, + PlanModel, + PlanRecommendations, + RecommendationMaterials, +) +from backend.app.db.models.portfolio import ( + PropertyModel, + PropertyDetailsEpcModel, +) +from backend.app.db.models.materials import Material +from utils.logger import setup_logger + +logger = setup_logger() + + +class DbMethods: + + def __init__(self, session: Session) -> None: + self.session = session + + def get_properties(self, portfolio_id: int) -> pd.DataFrame: + """ + Function to fetch the property data, for property scenario exports + :param portfolio_id: + :return: + """ + stmt = ( + select(PropertyModel, PropertyDetailsEpcModel) + .join( + PropertyDetailsEpcModel, + PropertyModel.id == PropertyDetailsEpcModel.property_id, + ) + .where(PropertyModel.portfolio_id == portfolio_id) + ) + + rows: Sequence[Row[Tuple[PropertyModel, PropertyDetailsEpcModel]]] = ( + self.session.execute(stmt).all() + ) + + data: List[Dict[str, Any]] = [ + { + **{ + col.name: getattr(property_model, col.name) + for col in PropertyModel.__table__.columns.values() + }, + **{ + col.name: getattr(epc_model, col.name) + for col in PropertyDetailsEpcModel.__table__.columns.values() + }, + } + for property_model, epc_model in rows + ] + + return pd.DataFrame(data) + + def get_latest_plans( + self, + portfolio_id: int, + scenario_ids: Optional[List[int]] = None, + default_only: bool = False, + ) -> pd.DataFrame: + """ + Fetch latest plans. + + Modes: + 1) Scenario mode: latest per (scenario_id, property_id) + 2) Default mode: latest default plan per property (ignores scenario_ids) + + """ + + # ----------------------------- + # Sanity checks + # ----------------------------- + if default_only and scenario_ids: + # Override scenario_ids to make it explicit that they will be ignored in the query + scenario_ids = None + + if not default_only and not scenario_ids: + raise ValueError( + "Either scenario_ids must be provided " + "or default_only must be True." + ) + + # ----------------------------- + # Filter on just the default plans - we ignore the scenario ids. NOTE - this is specific to postgres + # and relies on DISTINCT ON behaviour. + # ----------------------------- + if default_only: + # Latest default plan per property (ignore scenarios entirely) + # DISTINCT ON (property_id) keeps the first row per property, + # ordered by created_at DESC so we get the newest one. + + stmt = ( + select(PlanModel) + .where( + PlanModel.portfolio_id == portfolio_id, + PlanModel.is_default.is_(True), + ) + .distinct(PlanModel.property_id) + .order_by( + PlanModel.property_id, + PlanModel.created_at.desc(), + ) + ) + + else: + # Latest plan per (scenario_id, property_id) + # DISTINCT ON (scenario_id, property_id) keeps the newest + # plan per scenario/property combination. + + assert scenario_ids is not None + + stmt = ( + select(PlanModel) + .where( + PlanModel.portfolio_id == portfolio_id, + PlanModel.scenario_id.in_(scenario_ids), + ) + .distinct( + PlanModel.scenario_id, + PlanModel.property_id, + ) + .order_by( + PlanModel.scenario_id, + PlanModel.property_id, + PlanModel.created_at.desc(), + ) + ) + + logger.info("Fetching plans") + + plans: Sequence[PlanModel] = self.session.scalars(stmt).all() + + return pd.DataFrame( + [ + { + col.name: getattr(plan, col.name) + for col in PlanModel.__table__.columns.values() + } + for plan in plans + ] + ) + + def get_recommendations(self, plan_ids: List[int]) -> pd.DataFrame: + + if not plan_ids: + logger.info("No plan ids provided") + return pd.DataFrame() + + stmt = ( + select(Recommendation, PlanModel.scenario_id, PlanModel.name) + .join( + PlanRecommendations, + Recommendation.id == PlanRecommendations.recommendation_id, + ) + .join(PlanModel, PlanModel.id == PlanRecommendations.plan_id) + .where( + PlanRecommendations.plan_id.in_(plan_ids), + Recommendation.default.is_(True), + Recommendation.already_installed.is_(False), + ) + ) + + rows: Sequence[Tuple[Recommendation, Optional[int], Optional[str]]] = ( + self.session.execute(stmt).tuples().all() + ) + + data: List[Dict[str, Any]] = [ + { + **{ + col.name: getattr(rec_model, col.name) + for col in Recommendation.__table__.columns.values() + }, + "scenario_id": scenario_id, + "plan_name": plan_name, + } + for rec_model, scenario_id, plan_name in rows + ] + + return pd.DataFrame(data) + + def attach_materials(self, recommendations_df: pd.DataFrame) -> pd.DataFrame: + + if recommendations_df.empty: + recommendations_df["materials"] = [] + return recommendations_df + + rec_ids: List[int] = recommendations_df["id"].astype(int).tolist() + + stmt = ( + select(RecommendationMaterials, Material) + .join(Material, RecommendationMaterials.material_id == Material.id) + .where(RecommendationMaterials.recommendation_id.in_(rec_ids)) + ) + + rows: Sequence[Tuple[RecommendationMaterials, Material]] = ( + self.session.execute(stmt).tuples().all() + ) + + materials_map: Dict[int, List[Dict[str, Any]]] = defaultdict(list) + + for rec_mat, material in rows: + materials_map[rec_mat.recommendation_id].append( + { + "material_id": rec_mat.material_id, + "depth": rec_mat.depth, + "quantity": rec_mat.quantity, + "quantity_unit": rec_mat.quantity_unit, + "estimated_cost": rec_mat.estimated_cost, + "type": material.type.value if material.type else None, + "includes_battery": material.includes_battery, + } + ) + + recommendations_df["materials"] = recommendations_df["id"].astype(int).apply( + lambda x: materials_map.get(x, []) + ) + + return recommendations_df diff --git a/backend/export/property_scenarios/input_schema.py b/backend/export/property_scenarios/input_schema.py new file mode 100644 index 00000000..f6fa5965 --- /dev/null +++ b/backend/export/property_scenarios/input_schema.py @@ -0,0 +1,40 @@ +from typing import Optional, Union, List +from pydantic import BaseModel, model_validator, PrivateAttr + + +class ExportRequest(BaseModel): + # uuid which maps to a specific export request, used for tracking and logging + task_id: Union[str, None] + # uuid which maps to a specific export operation, used for tracking and logging. subtask is the child of the + # task, where the work has been distributed across workers + subtask_id: Union[str, None] + # associated portfolio id for the export request + portfolio_id: int + # list of scenario ids to export + scenario_ids: List[int] + # boolean which will overwrite the scenario ids. If this is true, we will only export the default plan for each + # property and will ignore the scenario ids + default_plans_only: Optional[bool] = False + + # Private attribute to indicate whether scenario_ids should be ignored due to default_plans_only being True + _scenario_ids_ignored: bool = PrivateAttr(default=False) + + @model_validator(mode="after") + def validate_default_plan_override(self): + """ + If default_plans_only is True and scenario_ids were provided, + we allow execution but make it explicit that scenario_ids + will be ignored. + """ + if self.default_plans_only and self.scenario_ids: + # We do NOT raise โ€” we allow execution. + # We just mark the object so the handler can log/return a warning. + object.__setattr__(self, "_scenario_ids_ignored", True) + else: + object.__setattr__(self, "_scenario_ids_ignored", False) + + return self + + @property + def scenario_ids_ignored(self) -> bool: + return self._scenario_ids_ignored diff --git a/backend/export/property_scenarios/main.py b/backend/export/property_scenarios/main.py new file mode 100644 index 00000000..d38db4c9 --- /dev/null +++ b/backend/export/property_scenarios/main.py @@ -0,0 +1,179 @@ +import json +from typing import Optional, Any, Mapping, Dict, Union, List + +import pandas as pd +from sqlalchemy.orm import Session + +from backend.export.property_scenarios.input_schema import ExportRequest +from backend.export.property_scenarios.db_functions import DbMethods +from backend.app.db.connection import db_read_session +from backend.app.utils import sap_to_epc +from utils.logger import setup_logger + +logger = setup_logger() + + +def choose_group_keys(payload: ExportRequest) -> List[Union[int, str]]: + if payload.default_plans_only: + return ["default_plans"] # Single export, no scenario grouping + return payload.scenario_ids + + +def has_solar_with_battery(materials_list: Optional[List[Dict[str, Any]]]) -> bool: + """ + Simple check to determine if any material in the list is a solar PV measure that includes a battery. + :param materials_list: + :return: + """ + for m in materials_list or []: + if ( + m.get("type") == "solar_pv" + and m.get("includes_battery") is True + ): + return True + return False + + +def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]: + export_files: Dict[Union[str, int], pd.DataFrame] = {} + + db_methods = DbMethods(session) + + properties_df = db_methods.get_properties(payload.portfolio_id) + + logger.info("Retrieved %s properties for export", len(properties_df)) + + plans_df: pd.DataFrame = db_methods.get_latest_plans( + portfolio_id=payload.portfolio_id, + scenario_ids=payload.scenario_ids, + default_only=bool(payload.default_plans_only), + ) + + logger.info("Retrieved %s plans for export", len(plans_df)) + + if plans_df.empty: + logger.info("Empty plans dataframe - no plans to export. Returning empty export.") + return export_files + plan_ids: List[int] = plans_df["id"].tolist() + recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids) + + logger.info("Retrieved %s recommendations for export", len(recommendations_df)) + + recommendations_df = db_methods.attach_materials(recommendations_df) + + recommendations_df["has_solar_with_battery"] = ( + recommendations_df["materials"].apply(has_solar_with_battery) + ) + + _filter = ( + (recommendations_df["measure_type"] == "solar_pv") + & (recommendations_df["has_solar_with_battery"]) + ) + + recommendations_df.loc[_filter, "measure_type"] = ( + recommendations_df.loc[_filter, "measure_type"] + "_with_battery" + ) + + group_keys: List[Union[str, int]] = choose_group_keys(payload) + + for group_key in group_keys: + + if payload.default_plans_only: + scenario_recs = recommendations_df + else: + scenario_recs = recommendations_df[ + recommendations_df["scenario_id"] == group_key + ] + + if scenario_recs.empty: + logger.info("No recommendations found for group_key %s - skipping export for this group", group_key) + continue + + measures_df: pd.DataFrame = scenario_recs[ + ["property_id", "measure_type", "plan_name", "estimated_cost"] + ].drop_duplicates() + + pivot: pd.DataFrame = measures_df.pivot( + index=["property_id", "plan_name"], + columns="measure_type", + values="estimated_cost", + ).reset_index() + + pivot["total_retrofit_cost"] = ( + pivot.drop(columns=["property_id", "plan_name"]).sum(axis=1) + ) + + post_sap: pd.DataFrame = ( + scenario_recs.groupby("property_id")[["sap_points"]] + .sum() + .reset_index() + ) + + df: pd.DataFrame = ( + properties_df.rename(columns={"solar_pv": "existing_solar_pv"}) + .merge(pivot, how="left", on="property_id") + .merge(post_sap, how="left", on="property_id") + ) + + df["sap_points"] = df["sap_points"].fillna(0) + df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] + df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc) + + export_files[group_key] = df + + return export_files + + +# ============================================================ +# Lambda Handler +# ============================================================ + +def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Union[int, str]]: + """ + Example event: + body_dict = { + "task_id": "test", + "subtask_id": "test", + "portfolio_id": 569, + "scenario_ids": [], + "default_plans_only": True, + } + :param event: Lambda event containing export request details + :param context: Lambda context (not used in this handler but included for completeness) + :return: HTTP response indicating success or failure of the export operation + """ + for record in event.get("Records", []): + try: + body_dict = json.loads(record["body"]) + + logger.debug("Validating request body") + payload = ExportRequest.model_validate(body_dict) + + if payload.scenario_ids_ignored: + logger.warning( + "Received scenario_ids in request body but they will be ignored " + "because default_plans_only is set to True" + ) + + logger.debug("Successfully validated request body") + with db_read_session() as session: + exported_files = process_export(payload, session) + + # TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url + _ = exported_files + return { + "statusCode": 200, + "body": json.dumps({}), + } + + except Exception as e: + logger.error(f"Failed to process record: {e}") + return { + "statusCode": 500, + "body": json.dumps({"message": "Failed to process export request"}), + } + + return { + "statusCode": 201, + "body": json.dumps({"message": "No records to process"}), + } diff --git a/backend/export/tests/conftest.py b/backend/export/tests/conftest.py new file mode 100644 index 00000000..10bfa971 --- /dev/null +++ b/backend/export/tests/conftest.py @@ -0,0 +1,55 @@ +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from backend.app.db.base import Base + + +@pytest.fixture(scope="function") +def engine(postgresql): + """ + Create a SQLAlchemy engine bound to the ephemeral + pytest-postgresql database. + """ + + # Build SQLAlchemy URL from psycopg connection info + connection_string = ( + f"postgresql+psycopg://" + f"{postgresql.info.user}:" + f"{postgresql.info.password}@" + f"{postgresql.info.host}:" + f"{postgresql.info.port}/" + f"{postgresql.info.dbname}" + ) + + engine = create_engine(connection_string) + + # Create tables once per test session + Base.metadata.create_all(engine) + + # Yeild will split this function into two phase. 1) setup and 2) teardown, the latter of which will run after all + # tests have completed + yield engine + + # Clean-up after entire test session + Base.metadata.drop_all(engine) + engine.dispose() + + +@pytest.fixture(scope="function") +def db_session(engine): + """ + Provides a clean transactional session per test. + + Rolls back after each test to keep isolation. + """ + + connection = engine.connect() + transaction = connection.begin() + + session = sessionmaker(bind=connection)() + + yield session + + session.close() + transaction.rollback() + connection.close() diff --git a/backend/export/tests/fixtures/plan_recs_569.csv b/backend/export/tests/fixtures/plan_recs_569.csv new file mode 100644 index 00000000..01df3c96 --- /dev/null +++ b/backend/export/tests/fixtures/plan_recs_569.csv @@ -0,0 +1,14 @@ +id,plan_id,recommendation_id +24799722,1604277,24798968 +24799726,1604277,24798972 +24801150,1604367,24800396 +24802703,1604448,24801949 +24802724,1604448,24801970 +24805327,1604577,24804573 +24805397,1604579,24804643 +24805401,1604579,24804647 +24813000,1605111,24812246 +24813002,1605111,24812248 +24813004,1605111,24812250 +24813006,1605112,24812252 +24813009,1605112,24812255 diff --git a/backend/export/tests/fixtures/plans_569.csv b/backend/export/tests/fixtures/plans_569.csv new file mode 100644 index 00000000..7580163f --- /dev/null +++ b/backend/export/tests/fixtures/plans_569.csv @@ -0,0 +1,11 @@ +id,name,portfolio_id,property_id,scenario_id,created_at,is_default,valuation_increase_lower_bound,valuation_increase_upper_bound,valuation_increase_average,plan_type,post_sap_points,post_epc_rating,post_co2_emissions,co2_savings,post_energy_bill,energy_bill_savings,post_energy_consumption,energy_consumption_savings,valuation_post_retrofit,valuation_increase,cost_of_works,contingency_cost +1604277,,569,660478,1060,2026-02-19 16:14:45.560816,True,0.0302,0.07,0.048226666,,71.5,Epc.C,4.1813498,0.71865046,1447.5204,691.6662,15303.688,3276.7622,,,6984.568,1003.9568 +1604448,,569,660529,1060,2026-02-19 16:14:52.052740,True,0.0302,0.07,0.048226666,,70.0,Epc.C,7.32816,1.5818402,2978.734,2314.7651,16558.295,1837.0155,,,13528.6,2844.636 +1604367,,569,660538,1060,2026-02-19 16:14:48.517937,True,0.02,0.03,0.025,,71.0,Epc.C,5.003036,0.43696404,1933.2236,521.5316,19190.531,1883.4657,,,5520.0,828.0 +1604577,,569,660688,1060,2026-02-19 16:15:04.461456,True,0.02,0.03,0.025,,70.0,Epc.C,3.6019807,0.20801921,1610.3181,248.27809,13746.731,896.6345,,,5100.0,765.0 +1604579,,569,660690,1060,2026-02-19 16:15:04.461456,True,0.02,0.03,0.025,,70.0,Epc.C,4.7473392,0.5326607,1867.537,699.7881,18730.615,2527.2231,,,5469.0,825.74 +1605110,,569,660598,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,70.0,Epc.C,1.89,0.0,1125.7338,0.0,7268.866,0.0,,,0.0,0.0 +1605111,,569,660599,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,68.7,Epc.D,2.02,1.1,1174.9326,319.18213,7748.233,3924.9,,,1218.584,124.0984 +1605080,,569,660448,1069,2026-02-19 16:18:57.581528,True,0.0,0.0,0.0,,71.0,Epc.C,1.79,0.0,1101.9677,0.0,6821.7285,0.0,,,0.0,0.0 +1605112,,569,660600,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,64.9,Epc.D,1.89,0.8,1131.3535,172.0886,7241.062,2466.7,,,3885.834,716.7084 +1605404,,569,660652,1069,2026-02-19 16:19:28.383096,True,0.0,0.0,0.0,,71.0,Epc.C,3.18,0.0,1757.515,0.0,11929.814,0.0,,,0.0,0.0 diff --git a/backend/export/tests/fixtures/portfolio_569.csv b/backend/export/tests/fixtures/portfolio_569.csv new file mode 100644 index 00000000..7cbcbab9 --- /dev/null +++ b/backend/export/tests/fixtures/portfolio_569.csv @@ -0,0 +1,2 @@ +id,name,budget,status,goal,cost,number_of_properties,co2_equivalent_savings,energy_savings,energy_cost_savings,property_valuation_increase,rental_yield_increase,total_work_hours,labour_days,created_at,updated_at,epc_breakdown_pre_retrofit,epc_breakdown_post_retrofit,n_units_to_retrofit,co2_per_unit_pre_retrofit,co2_per_unit_post_retrofit,energy_bill_per_unit_pre_retrofit,energy_bill_per_unit_post_retrofit,energy_consumption_per_unit_pre_retrofit,energy_consumption_per_unit_post_retrofit,valuation_improvement_per_unit,cost_per_unit,cost_per_co2_saved,cost_per_sap_point,valuation_return_on_investment +569,Lifespace Rentals - Sample Retrofit Plans,,PortfolioStatus.SCOPING,PortfolioGoal.NONE,,,,,,,,,,2026-02-12 21:23:37.862000+00:00,2026-02-12 21:23:37.862000+00:00,,,,,,,,,,,,,, diff --git a/backend/export/tests/fixtures/properties_569.csv b/backend/export/tests/fixtures/properties_569.csv new file mode 100644 index 00000000..ac1934bd --- /dev/null +++ b/backend/export/tests/fixtures/properties_569.csv @@ -0,0 +1,11 @@ +,id,portfolio_id,creation_status,uprn,landlord_property_id,building_reference_number,status,address,postcode,has_pre_condition_report,has_recommendations,created_at,updated_at,property_type,built_form,local_authority,constituency,number_of_rooms,year_built,tenure,current_epc_rating,current_sap_points,current_valuation,installed_measures_sap_point_adjustment,is_sap_points_adjusted_for_installed_measures,original_sap_points +0,660478,569,PropertyCreationStatus.READY,100090438731.0,BARR052,3460742868.0,PortfolioStatus.ASSESSMENT,"52, Barrack Street",CO1 2LR,True,True,2026-02-12 21:59:02.744427,2026-02-19 16:18:57.941443,House,End-Terrace,Colchester,Colchester,4.0,1900.0,rental (private),Epc.E,53.0,0.0,0.0,False,53.0 +1,660448,569,PropertyCreationStatus.READY,100090678548.0,BOUR110A,10002385993.0,PortfolioStatus.ASSESSMENT,Upper 110a Bournemouth Park Road,SS2 5LS,True,True,2026-02-12 21:59:02.388473,2026-02-19 16:18:57.578330,Flat,Detached,Southend-on-Sea,Rochford and Southend East,2.0,1950.0,Rented (private),Epc.C,71.0,0.0,0.0,False,71.0 +2,660538,569,PropertyCreationStatus.READY,10033423541.0,CHUR099,8188570968.0,PortfolioStatus.ASSESSMENT,"99, Church Road",RM3 0SH,True,True,2026-02-12 21:59:03.203854,2026-02-19 16:19:03.748571,House,Mid-Terrace,Havering,Hornchurch and Upminster,5.0,1900.0,rental (private),Epc.D,58.0,0.0,0.0,False,58.0 +3,660529,569,PropertyCreationStatus.READY,100091596678.0,CHER003,8961772668.0,PortfolioStatus.ASSESSMENT,"3, Brickfield Cottages",SS4 1PP,True,True,2026-02-12 21:59:02.935502,2026-02-19 16:18:55.971569,House,Mid-Terrace,Rochford,Rochford and Southend East,4.0,1900.0,rental (private),Epc.E,41.0,0.0,0.0,False,41.0 +4,660598,569,PropertyCreationStatus.READY,100090663644.0,FLEM049B,10006705876.0,PortfolioStatus.ASSESSMENT,49b Flemming Crescent,SS9 4HR,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,Semi-Detached,Southend-on-Sea,,2.0,1930.0,Rented (social),Epc.C,70.0,0.0,0.0,False,70.0 +5,660599,569,PropertyCreationStatus.READY,10012149765.0,FORE003A,9740118668.0,PortfolioStatus.ASSESSMENT,"3a, Forest Avenue",SS1 2HU,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,End-Terrace,Southend-on-Sea,Rochford and Southend East,2.0,1930.0,rental (private),Epc.D,56.0,0.0,0.0,False,56.0 +6,660600,569,PropertyCreationStatus.READY,10012149797.0,FORE003GFF,1436818568.0,PortfolioStatus.ASSESSMENT,"3, Forest Avenue",SS1 2HU,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,End-Terrace,Southend-on-Sea,Rochford and Southend East,2.0,1900.0,rental (private),Epc.D,59.0,0.0,0.0,False,59.0 +7,660652,569,PropertyCreationStatus.READY,100022668838.0,MANT061,10000429573.0,PortfolioStatus.ASSESSMENT,61 MANTILLA ROAD,SW17 8DY,True,True,2026-02-12 21:59:04.711717,2026-02-19 16:19:28.379512,Flat,Mid-Terrace,Wandsworth,Tooting,4.0,1900.0,Owner-occupied,Epc.C,71.0,0.0,0.0,False,71.0 +8,660690,569,PropertyCreationStatus.READY,100021987220.0,MERR008,9050743578.0,PortfolioStatus.ASSESSMENT,"8, Merritt Road",SE4 1DY,True,True,2026-02-12 21:59:09.459245,2026-02-19 16:19:32.826638,House,Mid-Terrace,Lewisham,"Lewisham, Deptford",6.0,1900.0,owner-occupied,Epc.D,58.0,0.0,0.0,False,58.0 +9,660688,569,PropertyCreationStatus.READY,207158120.0,MEDC048,208210678.0,PortfolioStatus.ASSESSMENT,"48, Medcalf Road",EN3 6HL,True,True,2026-02-12 21:59:09.459245,2026-02-19 16:19:32.826638,House,Mid-Terrace,Enfield,Enfield North,4.0,1900.0,rental (private),Epc.D,61.0,0.0,0.0,False,61.0 diff --git a/backend/export/tests/fixtures/property_details_epc_569.csv b/backend/export/tests/fixtures/property_details_epc_569.csv new file mode 100644 index 00000000..16f48e54 --- /dev/null +++ b/backend/export/tests/fixtures/property_details_epc_569.csv @@ -0,0 +1,11 @@ +,id,property_id,portfolio_id,full_address,lodgement_date,is_expired,total_floor_area,walls,walls_rating,roof,roof_rating,floor,floor_rating,windows,windows_rating,heating,heating_rating,heating_controls,heating_controls_rating,hot_water,hot_water_rating,lighting,lighting_rating,mainfuel,ventilation,solar_pv,solar_hot_water,wind_turbine,floor_height,number_heated_rooms,heat_loss_corridor,unheated_corridor_length,number_of_open_fireplaces,number_of_extensions,number_of_storeys,mains_gas,energy_tariff,primary_energy_consumption,co2_emissions,current_energy_demand,current_energy_demand_heating_hotwater,estimated,sap_05_overwritten,sap_05_score,sap_05_epc_rating,heating_cost_current,hot_water_cost_current,lighting_cost_current,appliances_cost_current,gas_standing_charge,electricity_standing_charge,original_co2_emissions,original_primary_energy_consumption,original_current_energy_demand,original_current_energy_demand_heating_hotwater,installed_measures_co2_adjustment,installed_measures_energy_demand_adjustment,installed_measures_total_energy_bill_adjustment,installed_measures_heat_demand_adjustment,is_epc_adjusted_for_installed_measures +44,1534934,660688,569,"48, Medcalf Road",2018-09-05,False,68.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Solid, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.55,,False,,0,0,,True,Single,278.0,3.81,14643.366,12185.6,False,False,,,711.0628,139.06198,70.770935,609.7844,128.0785,199.8375,3.81,278.0,14643.366,12185.6,0.0,0.0,0.0,0.0,False +53,1534816,660600,569,"3, Forest Avenue",2020-02-27,False,35.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Suspended, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 83% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.64,,False,,0,0,,True,Single,389.0,2.69,9707.762,8267.8,False,False,,,466.75378,110.046844,53.1057,345.6198,128.0785,199.8375,2.69,389.0,9707.762,8267.8,0.0,0.0,0.0,0.0,False +292,1534754,660478,569,"52, Barrack Street",2019-09-11,False,67.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Solid, no insulation",,Partial double glazing,2,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,Low energy lighting in 78% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.36,,False,,0,1,,True,Single,374.0,4.9,18580.451,16094.1,False,False,,,980.4243,142.37581,86.25319,602.2173,128.0785,199.8375,4.9,374.0,18580.451,16094.1,0.0,0.0,0.0,0.0,False +295,1534868,660652,569,"61 MANTILLA ROAD, LONDON",2020-12-10,False,79.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Solid, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.63,,False,,0,0,,True,off-peak 7 hour,184.0,3.18,11929.814,9046.1,False,False,,,487.25763,143.84087,110.2875,688.2131,128.0785,199.8375,3.18,184.0,11929.814,9046.1,0.0,0.0,0.0,0.0,False +310,1534964,660448,569,Upper 110a Bournemouth Park Road,2022-02-22,False,35.0,"Solid brick, as built, no insulation",1,"Pitched, 100 mm loft insulation",3.0,(another dwelling below),,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 80% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.41,,False,,0,0,,True,Unknown,238.0,1.79,6821.7285,5382.4,False,False,,,272.55676,102.9448,52.930252,345.6198,128.0785,199.8375,1.79,238.0,6821.7285,5382.4,0.0,0.0,0.0,0.0,False +344,1534936,660690,569,"8, Merritt Road",2017-08-15,False,101.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Suspended, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,No low energy lighting,1,Mains gas not community,natural,0.0,False,0.0,2.6,,False,,0,1,,True,Unknown,260.0,5.28,21257.838,17606.3,False,False,,,1074.1602,154.13814,194.25749,816.8532,128.0785,199.8375,5.28,260.0,21257.838,17606.3,0.0,0.0,0.0,0.0,False +460,1535385,660529,569,"3, Brickfield Cottages, Cherry Orchard Lane",2020-04-09,False,85.0,"Solid brick, as built, no insulation",2,"Pitched, 200 mm loft insulation",4.0,"Suspended, no insulation",,Fully double glazed,3,Electric storage heaters,3,Manual charge control,2,"Electric immersion, off-peak",3,Low energy lighting in 58% of fixed outlets,4,Electricity not community,natural,0.0,False,0.0,2.45,,False,,0,1,,True,dual,577.0,8.91,18395.31,15230.1,False,False,,,3550.6333,666.58136,149.46556,726.9812,0.0,199.8375,8.91,577.0,18395.31,15230.1,0.0,0.0,0.0,0.0,False +485,1534784,660538,569,"99, Church Road, Harold Wood",2019-09-03,False,92.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Suspended, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 80% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.52,,False,,0,1,,True,Single,297.0,5.44,21073.996,17904.0,False,False,,,1092.4246,156.6427,109.16419,768.6077,128.0785,199.8375,5.44,297.0,21073.996,17904.0,0.0,0.0,0.0,0.0,False +494,1534814,660598,569,49b Flemming Crescent,2024-10-03,False,35.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Suspended, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.42,,False,,0,0,,True,Single,261.0,1.89,7268.866,5865.4,False,False,,,304.39737,104.800545,43.0,345.6198,128.0785,199.8375,1.89,261.0,7268.866,5865.4,0.0,0.0,0.0,0.0,False +741,1534815,660599,569,"3a, Forest Avenue",2020-06-05,False,40.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,(another dwelling below),,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 38% of fixed outlets,3,Mains gas not community,natural,0.0,False,0.0,2.58,,False,,0,0,,True,Unknown,396.0,3.12,11673.133,9974.6,False,False,,,587.73975,108.13529,85.62337,384.70035,128.0785,199.8375,3.12,396.0,11673.133,9974.6,0.0,0.0,0.0,0.0,False diff --git a/backend/export/tests/fixtures/recommendations_569.csv b/backend/export/tests/fixtures/recommendations_569.csv new file mode 100644 index 00000000..769643ea --- /dev/null +++ b/backend/export/tests/fixtures/recommendations_569.csv @@ -0,0 +1,14 @@ +Unnamed: 0,id,property_id,created_at,type,measure_type,description,estimated_cost,default,starting_u_value,new_u_value,sap_points,heat_demand,kwh_savings,co2_equivalent_savings,energy_savings,energy_cost_savings,property_valuation_increase,rental_yield_increase,total_work_hours,labour_days,already_installed,plan_name +49705,24798968,660478,2026-02-19 16:14:45.560816,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",874.568,True,,,1.5,14.9,1041.2,0.2,14.9,72.639015,,,4.16,1.0,False,whatever +49709,24798972,660478,2026-02-19 16:14:45.560816,solar_pv,solar_pv,"8 panel system, 400W solar panels, 5.8kw Growatt battery - 3.2 kWp system",6110.0,True,,,17.0,79.1,2235.5623,0.5186504,79.1,619.02716,,,48.0,2.0,False,whatever +51133,24800396,660538,2026-02-19 16:14:48.517937,solar_pv,solar_pv,"10 panel system, 400W solar panels - 4.0 kWp system",5520.0,True,,,13.0,58.5,1883.4657,0.43696404,58.5,521.5316,,,48.0,2.0,False,whatever +52686,24801949,660529,2026-02-19 16:14:52.052740,heating,boiler_upgrade,"Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs",8008.6,True,,,12.9,132.9,0.0,1.1556525,132.9,1806.0955,,,26.5,4.0,False,whatever +52707,24801970,660529,2026-02-19 16:14:52.052740,solar_pv,solar_pv,"10 panel system, 400W solar panels - 4.0 kWp system",5520.0,True,,,16.1,68.8,1837.0155,0.4261876,68.8,508.6696,,,48.0,2.0,False,whatever +55310,24804573,660688,2026-02-19 16:15:04.461456,solar_pv,solar_pv,"5 panel system, 400W solar panels - 2.0 kWp system",5100.0,True,,,9.0,41.4,896.6345,0.20801921,41.4,248.27809,,,48.0,2.0,False,whatever +55380,24804643,660690,2026-02-19 16:15:04.461456,low_energy_lighting,low_energy_lighting,Install low energy lighting in 14 outlets,49.0,True,,,2.0,18.2,766.5,0.124173,18.2,212.24385,,,1.0,0.125,False,whatever +55384,24804647,660690,2026-02-19 16:15:04.461456,solar_pv,solar_pv,"9 panel system, 400W solar panels - 3.6 kWp system",5420.0,True,,,10.0,43.9,1760.723,0.40848774,43.9,487.54422,,,48.0,2.0,False,whatever +62983,24812246,660599,2026-02-19 16:18:57.606337,loft_insulation,loft_insulation,Install 300mm of Knauf Loft Roll 44 glass fibre roll in your loft,600.0,True,2.3,2.3,8.4,102.8,3178.2,0.9,102.8,221.72618,,,8.0,1.0,False,whatever +62985,24812248,660599,2026-02-19 16:18:57.606337,low_energy_lighting,low_energy_lighting,Install low energy lighting in 4 outlets,14.0,True,,,1.0,14.2,219.0,0.0,14.2,60.6411,,,1.0,0.125,False,whatever +62987,24812250,660599,2026-02-19 16:18:57.606337,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",604.584,True,,,3.3,18.4,527.7,0.2,18.4,36.814835,,,3.08,1.0,False,whatever +62989,24812252,660600,2026-02-19 16:18:57.606337,suspended_floor_insulation,suspended_floor_insulation,Install 75mm Q-bot underfloor insulation insulation in suspended floor,3281.25,True,0.87,0.22,4.0,99.2,1816.6,0.6,99.2,126.734566,,,57.05,2.3770833,False,whatever +62992,24812255,660600,2026-02-19 16:18:57.606337,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",604.584,True,,,1.9,17.7,650.1,0.2,17.7,45.354034,,,3.08,1.0,False,whatever diff --git a/backend/export/tests/test_export.py b/backend/export/tests/test_export.py new file mode 100644 index 00000000..823882b5 --- /dev/null +++ b/backend/export/tests/test_export.py @@ -0,0 +1,540 @@ +import pandas as pd +import numpy as np +from pathlib import Path +import time + +from backend.export.property_scenarios.main import process_export +from backend.export.property_scenarios.input_schema import ExportRequest +from backend.app.db.models.portfolio import PropertyModel, Epc, Portfolio, PortfolioStatus, PortfolioGoal, \ + PropertyCreationStatus, PropertyDetailsEpcModel +from backend.app.db.models.recommendations import PlanModel, Recommendation, PlanRecommendations, \ + RecommendationMaterials +from backend.app.db.models.materials import Material +from utils.logger import setup_logger + +FIXTURE_PATH = Path("backend/export/tests/fixtures") +logger = setup_logger() + + +def load_csv(name: str) -> pd.DataFrame: + df = pd.read_csv(FIXTURE_PATH / name) + df = df.replace({np.nan: None}) + return df + + +def test_default_export_integration(db_session): + # ---------------------------------------- + # 1) Load csvs + # ---------------------------------------- + t0 = time.perf_counter() + portfolio_df = load_csv("portfolio_569.csv") + properties_df = load_csv("properties_569.csv") + property_details_epc_df = load_csv("property_details_epc_569.csv") + plans_df = load_csv("plans_569.csv") + plan_recs_df = load_csv("plan_recs_569.csv") + recommendations_df = load_csv("recommendations_569.csv") + + logger.info( + "Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s", + time.perf_counter() - t0, + len(properties_df), + len(plans_df), + len(recommendations_df), + ) + + logger.info("Starting database load") + db_load_t0 = time.perf_counter() + + # ---------------------------------------- + # 2) Insert test portfolio + # ---------------------------------------- + + portfolios = [] + for row in portfolio_df.itertuples(index=False): + portfolios.append( + Portfolio( + id=row.id, + name=row.name, + status=PortfolioStatus[row.status.split(".")[-1]], + goal=PortfolioGoal[row.goal.split(".")[-1]] if row.goal else None, + ) + ) + + db_session.bulk_save_objects(portfolios) + db_session.flush() + # ---------------------------------------- + # 3) Insert test property + # ---------------------------------------- + + properties = [] + + for row in properties_df.itertuples(index=False): + row_dict = row._asdict() + + row_dict["uprn"] = int(row_dict["uprn"]) if row_dict.get("uprn") else None + row_dict["building_reference_number"] = ( + int(row_dict["building_reference_number"]) + if row_dict.get("building_reference_number") + else None + ) + + prop = PropertyModel(**{ + col: row_dict[col] + for col in PropertyModel.__table__.columns.keys() + if col in row_dict + }) + + prop.creation_status = PropertyCreationStatus[ + row_dict["creation_status"].split(".")[-1] + ] + prop.status = PortfolioStatus[row_dict["status"].split(".")[-1]] + + if row_dict.get("current_epc_rating"): + prop.current_epc_rating = Epc[ + row_dict["current_epc_rating"].split(".")[-1] + ] + + properties.append(prop) + + db_session.bulk_save_objects(properties) + db_session.flush() + + # ---------------------------------------- + # 4) Insert property details - EPC + # ---------------------------------------- + + epc_rows = [] + + for row in property_details_epc_df.itertuples(index=False): + row_dict = row._asdict() + + # Build only fields that exist on the model + epc_data = { + col.name: row_dict[col.name] + for col in PropertyDetailsEpcModel.__table__.columns.values() + if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"] + } + + epc = PropertyDetailsEpcModel( + property_id=row.property_id, + portfolio_id=row.portfolio_id, + **epc_data, + ) + + epc_rows.append(epc) + + db_session.bulk_save_objects(epc_rows) + db_session.flush() + + # ---------------------------------------- + # 4) Insert default plan + # ---------------------------------------- + + plans = [] + + for row in plans_df.itertuples(index=False): + row_dict = row._asdict() + + if row_dict.get("post_epc_rating"): + row_dict["post_epc_rating"] = Epc[ + row_dict["post_epc_rating"].split(".")[-1] + ] + + row_dict["scenario_id"] = None + + plan = PlanModel(**{ + col: row_dict[col] + for col in PlanModel.__table__.columns.keys() + if col in row_dict + }) + + plans.append(plan) + + db_session.bulk_save_objects(plans) + db_session.flush() + + # ---------------------------------------- + # 5) Insert recommendation + # ---------------------------------------- + + recs = [ + Recommendation(**{ + col: row[col] + for col in Recommendation.__table__.columns.keys() + if col in row + }) + for _, row in recommendations_df.iterrows() + ] + + db_session.bulk_save_objects(recs) + db_session.flush() + + # ---------------------------------------- + # 6) Insert PlanRecommendations + # ---------------------------------------- + links = [ + PlanRecommendations( + plan_id=row.plan_id, + recommendation_id=row.recommendation_id, + ) + for row in plan_recs_df.itertuples(index=False) + ] + + db_session.bulk_save_objects(links) + db_session.commit() + logger.info("Inserted all data in %.2f seconds", time.perf_counter() - db_load_t0) + + # ---------------------------------------- + # 6) Build payload + # ---------------------------------------- + + body_dict = { + "task_id": "test", + "subtask_id": "test", + "portfolio_id": 569, + "scenario_ids": [], + "default_plans_only": True, + } + + payload = ExportRequest.model_validate(body_dict) + + # ---------------------------------------- + # 7) Call process_export + # ---------------------------------------- + + logger.info( + "Recommendation count in DB: %s", + db_session.query(Recommendation).count() + ) + + logger.info( + "Property count in DB: %s", + db_session.query(PropertyModel).count() + ) + + logger.info( + "Property EPC in DB: %s", + db_session.query(PropertyDetailsEpcModel).count() + ) + + logger.info( + "Plan count in DB: %s", + db_session.query(PlanModel).count() + ) + + logger.info( + "PlanRecommendatons count in DB: %s", + db_session.query(PlanModel).count() + ) + + logger.info("Starting process_export") + process_t0 = time.perf_counter() + + result = process_export(payload, session=db_session) + + logger.info("process_export finished in %.2f seconds", time.perf_counter() - process_t0) + + # ---------------------------------------- + # 8) Assertions + # ---------------------------------------- + + assert "default_plans" in result, "Expected 'default_plans' in export result, got {}".format(result.keys()) + + df = result["default_plans"] + + assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0]) + + failed = df[df["predicted_post_works_sap"] < 69] + failed_property_types = failed["property_type"].value_counts().to_dict() + assert failed_property_types["Flat"] == 2 + # Check the houses + + assert failed.shape[0] + + assert df["total_retrofit_cost"].sum() == 41706.585999999996, ( + "Expected total retrofit cost to be 10000, got {}".format(df["total_retrofit_cost"].sum()) + ) + + assert df["predicted_post_works_sap"].sum() == 698.1, ( + "Expected total predicted post works SAP to be 698.1, got {}".format(df["predicted_post_works_sap"].sum()) + ) + + assert df["sap_points"].sum() == 100.10000000000001, ( + "Expected total SAP points increase to be 100.10000000000001, got {}".format(df["sap_points"].sum()) + ) + + assert df.shape == (10, 95), "Expected dataframe shape to be (10, 11), got {}".format(df.shape) + + +def test_solar_with_battery_example(db_session): + test_portfolio_id = 1 + test_property_id = 1 + + portfolio_df = pd.DataFrame( + [{'id': test_portfolio_id, 'name': 'Example', 'budget': None, + 'status': 'PortfolioStatus.SCOPING', 'goal': 'PortfolioGoal.NONE', 'cost': None, 'number_of_properties': None, + 'co2_equivalent_savings': None, 'energy_savings': None, 'energy_cost_savings': None, + 'property_valuation_increase': None, 'rental_yield_increase': None, 'total_work_hours': None, + 'labour_days': None, 'created_at': '2026-02-12 21:23:37.862000+00:00', + 'updated_at': '2026-02-12 21:23:37.862000+00:00', 'epc_breakdown_pre_retrofit': None, + 'epc_breakdown_post_retrofit': None, 'n_units_to_retrofit': None, 'co2_per_unit_pre_retrofit': None, + 'co2_per_unit_post_retrofit': None, 'energy_bill_per_unit_pre_retrofit': None, + 'energy_bill_per_unit_post_retrofit': None, 'energy_consumption_per_unit_pre_retrofit': None, + 'energy_consumption_per_unit_post_retrofit': None, 'valuation_improvement_per_unit': None, + 'cost_per_unit': None, 'cost_per_co2_saved': None, 'cost_per_sap_point': None, + 'valuation_return_on_investment': None}] + ) + + properties_df = pd.DataFrame( + [{'id': test_property_id, 'portfolio_id': test_portfolio_id, 'creation_status': 'PropertyCreationStatus.READY', + 'uprn': 100090438731, 'landlord_property_id': 'BARR052', 'building_reference_number': 3460742868.0, + 'status': 'PortfolioStatus.ASSESSMENT', 'address': '52, Barrack Street', 'postcode': 'CO1 2LR', + 'has_pre_condition_report': True, 'has_recommendations': True, 'created_at': '2026-02-12 21:59:02.744427', + 'updated_at': '2026-02-19 16:18:57.941443', 'property_type': 'House', 'built_form': 'End-Terrace', + 'local_authority': 'Colchester', 'constituency': 'Colchester', 'number_of_rooms': 4.0, 'year_built': 1900.0, + 'tenure': 'rental (private)', 'current_epc_rating': 'Epc.E', 'current_sap_points': 53.0, + 'current_valuation': 0.0, 'installed_measures_sap_point_adjustment': 0.0, + 'is_sap_points_adjusted_for_installed_measures': False, 'original_sap_points': 53.0}] + ) + + property_details_epc_df = pd.DataFrame( + [ + {'id': 1534934, 'property_id': test_property_id, 'portfolio_id': test_portfolio_id, + 'full_address': '48, Medcalf Road', 'lodgement_date': '2018-09-05', 'is_expired': False, + 'total_floor_area': 68.0, 'walls': 'Solid brick, as built, no insulation', 'walls_rating': 1, + 'roof': 'Pitched, no insulation', 'roof_rating': 1.0, 'floor': 'Solid, no insulation', + 'floor_rating': None, + 'windows': 'Fully double glazed', 'windows_rating': 4, 'heating': 'Boiler and radiators, mains gas', + 'heating_rating': 4, 'heating_controls': 'Programmer, room thermostat and trvs', + 'heating_controls_rating': 4, + 'hot_water': 'From main system', 'hot_water_rating': 4, + 'lighting': 'Low energy lighting in all fixed outlets', 'lighting_rating': 5, + 'mainfuel': 'Mains gas not community', 'ventilation': 'natural', 'solar_pv': 0.0, 'solar_hot_water': False, + 'wind_turbine': 0.0, 'floor_height': 2.55, 'number_heated_rooms': None, 'heat_loss_corridor': False, + 'unheated_corridor_length': None, 'number_of_open_fireplaces': 0, 'number_of_extensions': 0, + 'number_of_storeys': None, 'mains_gas': True, 'energy_tariff': 'Single', + 'primary_energy_consumption': 278.0, + 'co2_emissions': 3.81, 'current_energy_demand': 14643.366, + 'current_energy_demand_heating_hotwater': 12185.6, + 'estimated': False, 'sap_05_overwritten': False, 'sap_05_score': None, 'sap_05_epc_rating': None, + 'heating_cost_current': 711.0628, 'hot_water_cost_current': 139.06198, 'lighting_cost_current': 70.770935, + 'appliances_cost_current': 609.7844, 'gas_standing_charge': 128.0785, + 'electricity_standing_charge': 199.8375, + 'original_co2_emissions': 3.81, 'original_primary_energy_consumption': 278.0, + 'original_current_energy_demand': 14643.366, 'original_current_energy_demand_heating_hotwater': 12185.6, + 'installed_measures_co2_adjustment': 0.0, 'installed_measures_energy_demand_adjustment': 0.0, + 'installed_measures_total_energy_bill_adjustment': 0.0, 'installed_measures_heat_demand_adjustment': 0.0, + 'is_epc_adjusted_for_installed_measures': False} + ] + ) + + plans_df = pd.DataFrame( + [ + {'id': 0, 'name': None, 'portfolio_id': test_portfolio_id, 'property_id': test_property_id, + 'scenario_id': 1060, 'created_at': '2026-02-19 16:14:45.560816', 'is_default': True, + 'valuation_increase_lower_bound': 0.0302, + 'valuation_increase_upper_bound': 0.07, 'valuation_increase_average': 0.048226666, 'plan_type': None, + 'post_sap_points': 71.5, 'post_epc_rating': 'Epc.C', 'post_co2_emissions': 4.1813498, + 'co2_savings': 0.71865046, 'post_energy_bill': 1447.5204, 'energy_bill_savings': 691.6662, + 'post_energy_consumption': 15303.688, 'energy_consumption_savings': 3276.7622, + 'valuation_post_retrofit': None, 'valuation_increase': None, 'cost_of_works': 6984.568, + 'contingency_cost': 1003.9568} + ] + ) + + plan_recs_df = pd.DataFrame( + [{'id': 0, 'plan_id': 0, 'recommendation_id': 0}] + ) + + recommendations_df = pd.DataFrame( + [{'id': 0, 'property_id': test_property_id, 'created_at': '2026-02-19 16:14:45.560816', + 'type': 'solar_pv', 'measure_type': 'solar_pv', + 'description': 'Fit solar', + 'estimated_cost': 10000, 'default': True, 'starting_u_value': None, 'new_u_value': None, 'sap_points': 1.5, + 'heat_demand': 14.9, 'kwh_savings': 1041.2, 'co2_equivalent_savings': 0.2, 'energy_savings': 14.9, + 'energy_cost_savings': 72.639015, 'property_valuation_increase': None, 'rental_yield_increase': None, + 'total_work_hours': 4.16, 'labour_days': 1.0, 'already_installed': False, 'plan_name': 'whatever'} + ] + ) + + recommendations_materials_df = pd.DataFrame( + [ + { + "id": 0, "recommendation_id": 0, "material_id": 0, "depth": None, "quantity": 1.0, + "quantity_unit": "part", + "estimated_cost": 10000, "created_at": '2026-02-19 16:14:45.560816', + "updated_at": '2026-02-19 16:14:45.560816', + } + ] + ) + + materials_df = pd.DataFrame( + [ + {'id': 0, 'type': 'solar_pv', 'description': 'Some solar product', + 'depth': 75.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'Test', + 'created_at': "'2026-02-19 16:14:45.560816", 'is_active': True, + 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0, + 'total_cost': 10000, + 'notes': None, 'is_installer_quote': True, 'innovation_rate': 0.25, 'size': None, 'size_unit': None, + 'includes_scaffolding': True, 'includes_battery': True, 'battery_size': 5.8} + ] + ) + + # Load into db + # ------------------------------------------------- + # Insert Portfolio + # ------------------------------------------------- + for row in portfolio_df.itertuples(index=False): + db_session.add( + Portfolio( + id=row.id, + name=row.name, + status=PortfolioStatus[row.status.split(".")[-1]], + goal=PortfolioGoal[row.goal.split(".")[-1]], + ) + ) + db_session.flush() + + # ------------------------------------------------- + # Insert Property + # ------------------------------------------------- + for row in properties_df.itertuples(index=False): + prop = PropertyModel( + id=row.id, + portfolio_id=row.portfolio_id, + creation_status=PropertyCreationStatus[row.creation_status.split(".")[-1]], + status=PortfolioStatus[row.status.split(".")[-1]], + uprn=row.uprn, + property_type=row.property_type, + current_sap_points=row.current_sap_points, + current_epc_rating=Epc[row.current_epc_rating.split(".")[-1]], + ) + db_session.add(prop) + db_session.flush() + + # ------------------------------------------------- + # Insert EPC Details + # ------------------------------------------------- + for row in property_details_epc_df.itertuples(index=False): + epc = PropertyDetailsEpcModel( + property_id=row.property_id, + portfolio_id=row.portfolio_id, + full_address=row.full_address, + total_floor_area=row.total_floor_area, + walls=row.walls, + roof=row.roof, + windows=row.windows, + heating=row.heating, + solar_pv=row.solar_pv, + ) + db_session.add(epc) + db_session.flush() + + # ------------------------------------------------- + # Insert Plan (default) + # ------------------------------------------------- + for row in plans_df.itertuples(index=False): + plan = PlanModel( + id=row.id, + portfolio_id=row.portfolio_id, + property_id=row.property_id, + scenario_id=None, # default mode + is_default=row.is_default, + ) + db_session.add(plan) + db_session.flush() + + # ------------------------------------------------- + # IMPORTANT: Force recommendation to be solar_pv + # ------------------------------------------------- + recommendations_df.loc[0, "measure_type"] = "solar_pv" + + for row in recommendations_df.itertuples(index=False): + rec = Recommendation( + id=row.id, + property_id=row.property_id, + measure_type=row.measure_type, + estimated_cost=row.estimated_cost, + default=row.default, + already_installed=row.already_installed, + sap_points=row.sap_points, + type=row.type, + description=row.description + ) + db_session.add(rec) + db_session.flush() + + # ------------------------------------------------- + # Link Plan -> Recommendation + # ------------------------------------------------- + for row in plan_recs_df.itertuples(index=False): + db_session.add( + PlanRecommendations( + plan_id=row.plan_id, + recommendation_id=row.recommendation_id, + ) + ) + db_session.flush() + + # ------------------------------------------------- + # Insert Material (includes_battery=True) + # ------------------------------------------------- + for row in materials_df.itertuples(index=False): + material = Material( + id=row.id, + type=row.type, + description=row.description, + depth_unit=row.depth_unit, + cost_unit=row.cost_unit, + r_value_unit=row.r_value_unit, + thermal_conductivity_unit=row.thermal_conductivity_unit, + includes_battery=row.includes_battery, + is_active=row.is_active, + ) + db_session.add(material) + db_session.flush() + + # ------------------------------------------------- + # Link Recommendation -> Material + # ------------------------------------------------- + for row in recommendations_materials_df.itertuples(index=False): + db_session.add( + RecommendationMaterials( + recommendation_id=row.recommendation_id, + material_id=row.material_id, + depth=row.depth or 0.0, + quantity=row.quantity, + quantity_unit=row.quantity_unit, + estimated_cost=row.estimated_cost, + ) + ) + + db_session.commit() + + payload = ExportRequest.model_validate({ + "task_id": "test", + "subtask_id": "test", + "portfolio_id": test_portfolio_id, + "scenario_ids": [], + "default_plans_only": True, + }) + + result = process_export(payload, session=db_session) + + assert "default_plans" in result + + df = result["default_plans"] + + assert "solar_pv_with_battery" in df.columns + + # solar_pv should NOT exist + assert "solar_pv" not in df.columns + + assert df.shape[0] == 1, "Expected 1 property in the export, got {}".format(df.shape[0]) + + # Cost should land in correct column + assert df["solar_pv_with_battery"].iloc[0] == 10000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..72ec3f0c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pyright] +reportUnknownMemberType = false +reportUnknownVariableType = false \ No newline at end of file diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..d4e0e2a4 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,8 @@ +{ + "typeCheckingMode": "strict", + "venvPath": "/Users/khalimconn-kowlessar/opt/anaconda3/envs/", + "venv": "Fastapi-backend", + "include": [ + "." + ] +} \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 9c9f8234..608d5e0c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,6 @@ [pytest] pythonpath = . +log_cli = true +log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial -testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests +testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests diff --git a/recommendations/optimiser/funding_optimiser.py b/recommendations/optimiser/funding_optimiser.py index 69a6bc48..48c3cf03 100644 --- a/recommendations/optimiser/funding_optimiser.py +++ b/recommendations/optimiser/funding_optimiser.py @@ -754,7 +754,7 @@ def optimise_with_scenarios( # Wall measures could be IWI or EWI remaining_wall_measures = [ x for x in all_measure_types if x in WALL_INSULATION_MEASURES + [ - "internal_wall_insulation+mechanical_ventilation", "external_wall_insulation+mechanical_ventilation" + "internal_wall_insulation+mechanical_ventilation", "external_wall_insulation+mechanical_ventilation", ] ] remaining_roof_measures = [x for x in all_measure_types if x in ROOF_INSULATION_MEASURES] diff --git a/sfr/principal_pitch/2_export_data.py b/sfr/principal_pitch/2_export_data.py index 4f430209..374ed16d 100644 --- a/sfr/principal_pitch/2_export_data.py +++ b/sfr/principal_pitch/2_export_data.py @@ -28,15 +28,15 @@ from sqlalchemy import func # PORTFOLIO_ID = 206 # SCENARIOS = [389] -PORTFOLIO_ID = 568 +PORTFOLIO_ID = 597 SCENARIOS = [ - 1059, + 1099 ] scenario_names = { - 1059: "EPC C - 10k budget", + 1099: "ยฃ10k cost capped - no solid wall or floor", } -project_name = "manchester" +project_name = "Livespace Rentals" def get_data(portfolio_id, scenario_ids): @@ -234,7 +234,7 @@ for scenario_id in SCENARIOS: # Get recs for this scenario recommended_measures_df = recommendations_df[ recommendations_df["scenario_id"] == scenario_id - ][["property_id", "measure_type", "estimated_cost", "default"]] + ][["property_id", "measure_type", "estimated_cost", "default"]] recommended_measures_df = recommended_measures_df[ recommended_measures_df["default"] ] @@ -242,7 +242,7 @@ for scenario_id in SCENARIOS: post_install_sap = recommendations_df[ recommendations_df["scenario_id"] == scenario_id - ][["property_id", "default", "sap_points"]] + ][["property_id", "default", "sap_points"]] post_install_sap = post_install_sap[post_install_sap["default"]] # Sum up the sap points by property id post_install_sap = ( @@ -320,7 +320,7 @@ for scenario_id in SCENARIOS: z = df2[ (df2["predicted_post_works_epc"] != "D") & (df2["post_epc_rating"].astype(str) == "Epc.D") - ] + ] df2["predicted_post_works_epc"].value_counts() df2["post_epc_rating"].astype(str).value_counts() diff --git a/test.requirements.txt b/test.requirements.txt index d31371a6..d8b8b777 100644 --- a/test.requirements.txt +++ b/test.requirements.txt @@ -2,4 +2,6 @@ pytest mock pytest-cov pytest-mock -dotenv \ No newline at end of file +dotenv +psycopg[binary] +pytest-postgresql \ No newline at end of file