junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

resolved

Browse source
This commit is contained in:
Jun-te Kim 2026-03-06 13:35:15 +00:00
commit f659718a7e
65 changed files with 2361 additions and 717 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.devcontainer/backend/Dockerfile
View file

@ -56,4 +56,11 @@ https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \
tee /etc/apt/sources.list.d/hashicorp.list
RUN apt update
RUN apt-get install terraform
RUN terraform -install-autocomplete
RUN terraform -install-autocomplete
# Install postgres
RUN apt install -y wget gnupg2 lsb-release
RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
RUN wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
RUN apt update
RUN apt install -y postgresql-14

3
.devcontainer/backend/requirements.txt
View file

@ -18,6 +18,9 @@ sqlmodel
pytest==9.0.2
pytest-cov==7.0.0
ipykernel>=6.25,<7
dotenv
psycopg[binary]
pytest-postgresql
# Formatting
black==26.1.0
boto3-stubs

35
.github/workflows/_deploy_lambda.yml vendored
View file

@ -42,6 +42,22 @@ on:
required: true
AWS_REGION:
required: true
TF_VAR_db_host:
required: false
TF_VAR_db_name:
required: false
TF_VAR_db_port:
required: false
TF_VAR_api_key:
required: false
TF_VAR_secret_key:
required: false
TF_VAR_domain_name:
required: false
TF_VAR_epc_auth_token:
required: false
TF_VAR_google_solar_api_key:
required: false
jobs:
deploy:
@ -90,6 +106,15 @@ jobs:
- name: Terraform Plan
working-directory: ${{ inputs.lambda_path }}
env:
TF_VAR_db_host: ${{ secrets.TF_VAR_db_host }}
TF_VAR_db_name: ${{ secrets.TF_VAR_db_name }}
TF_VAR_db_port: ${{ secrets.TF_VAR_db_port }}
TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }}
TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }}
TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
run: |
terraform plan \
-var="stage=${{ inputs.stage }}" \
@ -106,10 +131,18 @@ jobs:
- name: Terraform Destroy
if: inputs.terraform_destroy == 'true' && inputs.terraform_apply != 'true'
working-directory: ${{ inputs.lambda_path }}
env:
TF_VAR_db_host: ${{ secrets.TF_VAR_db_host }}
TF_VAR_db_name: ${{ secrets.TF_VAR_db_name }}
TF_VAR_db_port: ${{ secrets.TF_VAR_db_port }}
TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }}
TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }}
TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
run: |
terraform destroy -auto-approve \
-var="stage=${{ inputs.stage }}" \
-var="lambda_name=${{ inputs.lambda_name }}" \
-var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \
-var="image_digest=${{ inputs.image_digest }}"

45
.github/workflows/deploy_terraform.yml vendored
View file

@ -241,6 +241,46 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
# ============================================================
# Ara Engine image and Push
# ============================================================
ara_engine_image:
needs: [determine_stage, shared_terraform]
uses: ./.github/workflows/_build_image.yml
with:
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
dockerfile_path: backend/docker/engine.Dockerfile
build_context: .
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
# ============================================================
# Deploy Ara Engine Lambda
# ============================================================
ara_engine_lambda:
needs: [ara_engine_image, determine_stage]
uses: ./.github/workflows/_deploy_lambda.yml
with:
lambda_name: ara_engine
lambda_path: infrastructure/terraform/lambda/engine
stage: ${{ needs.determine_stage.outputs.stage }}
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
image_digest: ${{ needs.ara_engine_image.outputs.image_digest }}
terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
TF_VAR_api_key: ${{ secrets.DEV_API_KEY }}
TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }}
TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }}
TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
# ============================================================
# 2️⃣ Build OrdanceSurvey image and Push
@ -276,8 +316,3 @@ jobs:
stage: ${{ needs.determine_stage.outputs.stage }}
ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }}
image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }}
terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}

5
.gitignore vendored
View file

@ -279,4 +279,7 @@ cache/
*.png
*.pptx
local_data*
local_data*
# pyright local config
pyrightconfig.json

3
.idea/Model.iml generated
View file

@ -10,4 +10,7 @@
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="py.test" />
</component>
</module>

25
.idea/watcherTasks.xml generated Normal file
View file

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectTasksOptions">
<TaskOptions isEnabled="false">
<option name="arguments" value="$FilePath$" />
<option name="checkSyntaxErrors" value="true" />
<option name="description" />
<option name="exitCodeBehavior" value="ERROR" />
<option name="fileExtension" value="py" />
<option name="immediateSync" value="true" />
<option name="name" value="Pyright" />
<option name="output" value="" />
<option name="outputFilters">
<array />
</option>
<option name="outputFromStdout" value="false" />
<option name="program" value="$USER_HOME$/.nvm/versions/node/v18.15.0/bin/pyright" />
<option name="runOnExternalChanges" value="true" />
<option name="scopeName" value="Project Files" />
<option name="trackOnlyRoot" value="false" />
<option name="workingDir" value="$ProjectFileDir$" />
<envs />
</TaskOptions>
</component>
</project>

287
asset_list/AssetList.py
View file

@ -36,14 +36,12 @@ from dotenv import load_dotenv
logger = setup_logger()
load_dotenv(dotenv_path="../backend/.env")
# OpenAI API Key (set this in your environment variables for security)
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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")
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
class DataRemapper:
def __init__(self, standard_values, standard_map=None, max_tokens=1000):
print(f"{OPENAI_API_KEY}")
"""
Initialize the remapper with standard values and a predefined mapping.
@ -1298,8 +1296,8 @@ class AssetList:
self.standardised_asset_list[
self.ATTRIBUTE_HAS_SOLAR
] = self.standardised_asset_list[
self.FIND_EPC_DATA_NAMES["Solar photovoltaics"]
] | ~self.standardised_asset_list[
self.FIND_EPC_DATA_NAMES["Solar photovoltaics"]
] | ~self.standardised_asset_list[
self.EPC_API_DATA_NAMES["photo-supply"]
].isin(
["0.0", 0, None, "", np.nan]
@ -1317,7 +1315,7 @@ class AssetList:
property_type=(
str(x[self.STANDARD_PROPERTY_TYPE]).title()
if str(x[self.STANDARD_PROPERTY_TYPE]).title()
in accepted_epc_property_types
in accepted_epc_property_types
else (
x[self.EPC_API_DATA_NAMES["property-type"]]
if not pd.isnull(
@ -1375,9 +1373,9 @@ class AssetList:
self.standardised_asset_list.apply(
lambda x: estimate_perimeter(
floor_area=x[self.EPC_API_DATA_NAMES["total-floor-area"]]
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
num_rooms=x[self.EPC_API_DATA_NAMES["number-habitable-rooms"]]
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
/ x[self.ATTRIBUTE_NUMBER_OF_FLOORS],
),
axis=1,
)
@ -1462,7 +1460,7 @@ class AssetList:
year_lower_bound = (
2007
if x[self.EPC_API_DATA_NAMES["construction-age-band"]]
== "England and Wales: 2007 onwards"
== "England and Wales: 2007 onwards"
else 2012
)
@ -1517,7 +1515,7 @@ class AssetList:
age_band_matches = (
"EPC Age Band Matches Year Built"
if x[self.STANDARD_YEAR_BUILT]
== int(x[self.EPC_API_DATA_NAMES["construction-age-band"]])
== int(x[self.EPC_API_DATA_NAMES["construction-age-band"]])
else "EPC Age Band is different from Year Built"
)
@ -1547,7 +1545,7 @@ class AssetList:
age_band_matches = (
"EPC Age Band Matches Year Built"
if (x[self.STANDARD_YEAR_BUILT] >= float(lower_date))
and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date))
and (x[self.STANDARD_YEAR_BUILT] <= float(upper_date))
else (
"EPC Age Band is older than Year Built"
if x[self.STANDARD_YEAR_BUILT] > float(upper_date)
@ -1719,22 +1717,22 @@ class AssetList:
if self.non_intrusives_present:
if self.new_format_non_insturives_present_v2:
non_intrusives_wall_filter = (
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
["EMPTY", "PARTIAL", "EMPTY CAVITY"]
)
else:
non_intrusives_wall_filter = (
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
self.standardised_asset_list["non-intrusives: Construction"]
== "CAVITY"
) & self.standardised_asset_list["non-intrusives: Insulated"].isin(
["EMPTY", "PARTIAL"]
)
elif self.old_format_non_intrusives_present:
non_intrusives_wall_filter = self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
[
"empty cavity",
"partial fill",
@ -1744,18 +1742,18 @@ class AssetList:
"empty cav",
]
) | (
(
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
.str.strip()
.str.contains("empty cavity|partial fill")
& ~self.standardised_asset_list["non-intrusives: WFT Findings"]
.astype(str)
.str.lower()
.str.strip()
.str.contains("major access issues")
)
)
(
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
.str.strip()
.str.contains("empty cavity|partial fill")
& ~self.standardised_asset_list["non-intrusives: WFT Findings"]
.astype(str)
.str.lower()
.str.strip()
.str.contains("major access issues")
)
)
else:
# We set the filter to False, as we have no non-intrusives
non_intrusives_wall_filter = False
@ -1767,12 +1765,12 @@ class AssetList:
)
else:
year_built_filter = (
self.standardised_asset_list[self.STANDARD_YEAR_BUILT]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
) | (
self.standardised_asset_list["epc_year_upper_bound"]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
)
self.standardised_asset_list[self.STANDARD_YEAR_BUILT]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
) | (
self.standardised_asset_list["epc_year_upper_bound"]
<= self.EMPTY_CAVITY_YEAR_THRESHOLD
)
# Criteria:
# The property isn't a bedsit
@ -1813,8 +1811,8 @@ class AssetList:
] = (
~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity_has_solar"
]
"non_intrusive_indicates_empty_cavity_has_solar"
]
& (
~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(
["bedsit"]
@ -1890,8 +1888,8 @@ class AssetList:
.str.lower()
.isin(self.EPC_NO_WALL_INSULATION_DESCRIPTIONS)
| self.standardised_asset_list[self.STANDARD_WALL_CONSTRUCTION].isin(
["uninsulated cavity"]
)
["uninsulated cavity"]
)
)
######################################################
@ -1928,8 +1926,8 @@ class AssetList:
extraction_wall_filter = (
extraction_wall_filter
& ~self.standardised_asset_list[
"non-intrusives: Eligibility (Red/Yellow/Green)"
].isin(["RED"])
"non-intrusives: Eligibility (Red/Yellow/Green)"
].isin(["RED"])
)
self.standardised_asset_list[
@ -2025,26 +2023,26 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_data_indicates_correct_heating_system"
] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains(
"air source heat pump|ground source heat pump|boiler and radiators, electric"
)
) | (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
self.EPC_API_DATA_NAMES["mainheat-description"]
]
== "Controls for high heat retention storage heaters"
.str.lower()
.str.contains(
"air source heat pump|ground source heat pump|boiler and radiators, electric"
)
) | (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
]
== "Controls for high heat retention storage heaters"
)
)
)
# If the landlord has given us the heating system, we default to that on heating upgrades. Because of the
# poor heating in place, if the EPC indicates that this property had a low efficiency heating system but the
@ -2052,25 +2050,25 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_data_indicates_requires_heating_upgrade"
] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.str.contains("electric storage heaters|room heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
self.EPC_API_DATA_NAMES["mainheat-description"]
]
!= "Controls for high heat retention storage heaters"
.str.lower()
.str.contains("electric storage heaters|room heaters")
& (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheatcont-description"]
]
!= "Controls for high heat retention storage heaters"
)
) & (
~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
["district heating", "communal heating", "communal gas boiler"]
)
& ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
.astype(str)
.str.contains("gas ")
)
) & (
~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
["district heating", "communal heating", "communal gas boiler"]
)
& ~self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
.astype(str)
.str.contains("gas ")
)
# Basic check - both of the previous two shouldn't be true simultaneously
if (
@ -2150,8 +2148,8 @@ class AssetList:
self.standardised_asset_list[
"solar_non_intrusives_walls_insulated"
] = self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
"non-intrusives: WFT Findings"
].str.lower().str.strip().isin(
[
"retro drilled",
"retro filled",
@ -2160,8 +2158,8 @@ class AssetList:
"retro drilled and filled",
]
) | self.standardised_asset_list[
"non-intrusives: WFT Findings"
].str.lower().str.strip().str.contains(
"non-intrusives: WFT Findings"
].str.lower().str.strip().str.contains(
"retro drilled"
)
else:
@ -2178,14 +2176,19 @@ class AssetList:
)
self.standardised_asset_list["solar_epc_walls_insulated"] = (
self.standardised_asset_list[self.EPC_API_DATA_NAMES["walls-description"]]
.str.lower()
.str.contains("|".join(self.EPC_INSULATED_WALLS_SUBSTRINGS))
) | (
self.standardised_asset_list["walls_u_value"].apply(
lambda x: x <= 0.7 if not pd.isnull(x) else False
)
)
self.standardised_asset_list[
self.EPC_API_DATA_NAMES[
"walls-description"]]
.str.lower()
.str.contains("|".join(
self.EPC_INSULATED_WALLS_SUBSTRINGS))
) | (
self.standardised_asset_list[
"walls_u_value"].apply(
lambda x: x <= 0.7 if not pd.isnull(
x) else False
)
)
roof_data = []
for desc in self.standardised_asset_list[
@ -2227,20 +2230,20 @@ class AssetList:
self.standardised_asset_list[
"solar_epc_loft_needs_topup"
] = self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].apply(
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].apply(
lambda x: int(x) < 200 if str(x).isdigit() else False
) | (
(
self.standardised_asset_list["is_loft"]
| self.standardised_asset_list["is_pitched"]
(
self.standardised_asset_list["is_loft"]
| self.standardised_asset_list["is_pitched"]
)
& (
self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].isin(["below average", "none"])
)
)
& (
self.standardised_asset_list[
self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS
].isin(["below average", "none"])
)
)
self.standardised_asset_list["epc_has_floor_recommendation"] = (
self.standardised_asset_list["epc_has_floor_recommendation"].fillna(False)
@ -2249,15 +2252,16 @@ class AssetList:
# Check if the boiler is electric
# We check if it contains both the terms boiler & electric
self.standardised_asset_list["has_electric_boiler"] = (
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.isin(["boiler and radiators, electric"])
) | (
self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM]
== "electric boiler"
)
self.standardised_asset_list[
self.EPC_API_DATA_NAMES["mainheat-description"]
]
.str.lower()
.isin(["boiler and radiators, electric"])
) | (
self.standardised_asset_list[
self.STANDARD_HEATING_SYSTEM]
== "electric boiler"
)
####################################
# Check solar eligibility
@ -2395,11 +2399,11 @@ class AssetList:
empty_cavity_map = {
"non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE
+ ": ",
+ ": ",
"non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property "
"already has solar: ",
"already has solar: ",
"non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, "
f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
}
for variable, description in empty_cavity_map.items():
self.standardised_asset_list["cavity_reason"] = np.where(
@ -2415,8 +2419,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: WFT Findings"]
.str.lower()
@ -2441,8 +2445,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& self.standardised_asset_list[
"non_intrusive_indicates_cavity_extraction"
]
@ -2457,8 +2461,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: Insulated"]
== "RETRO DRILLED"
@ -2474,8 +2478,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& (
self.standardised_asset_list["non-intrusives: Insulated"]
== "FILLED AT BUILD"
@ -2491,8 +2495,8 @@ class AssetList:
(
self.standardised_asset_list["epc_indicates_empty_cavity"]
& ~self.standardised_asset_list[
"non_intrusive_indicates_empty_cavity"
]
"non_intrusive_indicates_empty_cavity"
]
& pd.isnull(self.standardised_asset_list["cavity_reason"])
),
f"{self.EPC_EMPTY}: " + self.standardised_asset_list["SAP Category"],
@ -2636,7 +2640,7 @@ class AssetList:
identified_work = self.standardised_asset_list[
~pd.isnull(self.standardised_asset_list["cavity_reason"])
| ~pd.isnull(self.standardised_asset_list["solar_reason"])
][self.DOMNA_PROPERTY_ID].values
][self.DOMNA_PROPERTY_ID].values
if self.DOMNA_PROPERTY_ID in self.outcomes.columns:
self.outcomes_for_output = self.outcomes[
@ -2671,12 +2675,12 @@ class AssetList:
blocks_of_flats = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
== "block of flats"
]
]
non_blocks_of_flats = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
!= "block of flats"
]
]
# Produce some aggregate figures
self.work_type_figures = {
@ -2719,7 +2723,7 @@ class AssetList:
blocks = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
== "block of flats"
].copy()
].copy()
if blocks.empty:
return
@ -2856,7 +2860,7 @@ class AssetList:
self.standardised_asset_list = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE]
!= "block of flats"
]
]
self.standardised_asset_list = pd.concat(
[self.standardised_asset_list, expanded_blocks], ignore_index=True
@ -2936,7 +2940,7 @@ class AssetList:
# find any block refs with more than 50% emptires
viable_empty_blocks = self.block_analysis_df[
self.block_analysis_df["Percentage of Empties"] >= 0.50
]
]
if not viable_empty_blocks.empty:
project_code_lookup = viable_empty_blocks[["Block Reference"]].copy()
@ -3175,7 +3179,7 @@ class AssetList:
contact_details = pd.read_excel(local_filepath, sheet_name=sheet_name)[
[self.contact_detail_fields["landlord_property_id"]] + details_colnames
]
]
contact_details = contact_details[
~pd.isnull(
contact_details[self.contact_detail_fields["landlord_property_id"]]
@ -3568,10 +3572,13 @@ class AssetList:
"Non-Intrusives: Date Checked <LISTING non_intrusives__date_checked>": date_of_inspections,
"Non-Intrusives: Wall Type <LISTING non_intrusives__wall_type>": non_intrusives_construction,
"Non-intrusives: Insulation <LISTING non_intrusives__insulation>": non_intrusives_insulated,
"Non-intrusives: Insulation Material <LISTING non_intrusives__insulation_material>": non_intrusives_insulation_material,
"Non-Intrusives: CIGA Check Required <LISTING non_intrusives__ciga_check_required>": non_intrusives_ciga_check_required,
"Non-intrusives: Insulation Material <LISTING non_intrusives__insulation_material>":
non_intrusives_insulation_material,
"Non-Intrusives: CIGA Check Required <LISTING non_intrusives__ciga_check_required>":
non_intrusives_ciga_check_required,
"Non-Intrusives: PV Access Issues <LISTING non_intrusives__access_issues>": non_intrusives_pv_access,
"Non-Intrusives: Roof Orientation <LISTING non_intrusives__roof_orientation>": non_intrusives_roof_orientation,
"Non-Intrusives: Roof Orientation <LISTING non_intrusives__roof_orientation>":
non_intrusives_roof_orientation,
"Non-Intrusives: Surveyor Notes <LISTING non_intrusives__surveyor_notes>": non_intrusives_surveyor_notes,
"Non-Intrusives: Surveyor Name <LISTING non_intrusives__surveyor_name>": non_intrusives_surveyor_name,
"CIGA: Date Requested <LISTING ciga__date_requested>": None, # TODO: Don't have this for the moment
@ -3748,8 +3755,8 @@ class AssetList:
# We compare address line 1 to full address
if any(
df[self.STANDARD_FULL_ADDRESS]
.str.lower()
.str.contains(row["Address Line 1"].lower(), na=False)
.str.lower()
.str.contains(row["Address Line 1"].lower(), na=False)
):
df = df[
df[self.STANDARD_FULL_ADDRESS]
@ -3989,7 +3996,7 @@ class AssetList:
matched = matched[
matched["houseno"].astype(str) == house_no_to_match
]
]
if matched.shape[0] == 1:
lookup_i.append(
{
@ -4014,7 +4021,7 @@ class AssetList:
)[0]
matched = matched[
matched[self.STANDARD_FULL_ADDRESS] == best_match
]
]
lookup_i.append(
{
"row_id": x["row_id"],
@ -4325,7 +4332,7 @@ class AssetList:
df = self.standardised_asset_list[
self.standardised_asset_list[self.STANDARD_LANDLORD_PROPERTY_ID]
== row[master_id_colnames[idx]]
]
]
if df.shape[0] == 1:
matched.append(
{
@ -4431,7 +4438,7 @@ class AssetList:
)[1]
)
> 90
]
]
if df.shape[0] == 0:
unmatched.append(row["row_id"])
@ -4439,8 +4446,8 @@ class AssetList:
if any(
df[self.STANDARD_FULL_ADDRESS]
.str.lower()
.str.contains(
.str.lower()
.str.contains(
" ".join(
[row[house_no_col], row["Street / Block Name"]]
).lower()
@ -4467,7 +4474,7 @@ class AssetList:
row[property_type_col].split(" ")[-1].lower()
)
& (df[self.STANDARD_PROPERTY_TYPE] != "block of flats")
]
]
if df.shape[0] != 1:
# We have multiple matches - it's likely because the landlord has a duplicate

49
asset_list/app.py
View file

@ -18,7 +18,6 @@ EPC_AUTH_TOKEN = os.getenv(
"EPC_AUTH_TOKEN",
)
OPENAI_API_KEY = os.getenv(
"OPENAI_API_KEY",
)
@ -74,24 +73,25 @@ def app():
Property UPRN
"""
data_folder = "/workspaces/model/asset_list"
data_filename = "assests.xlsx"
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lifespace Rentals/Missed"
# data_filename = "For Modelling - Final - reviewed.xlsx"
data_filename = "Missed Properties - with address.xlsx"
sheet_name = "Sheet1"
postcode_column = "POSTCODE"
address1_column = "ADDRESS"
address1_method = "house_number_extraction"
fulladdress_column = None
address_cols_to_concat = ["ADDRESS"]
postcode_column = "Postcode"
address1_column = "address1"
address1_method = None
fulladdress_column = "address1"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = None
landlord_os_uprn = "UPRN"
landlord_property_type = "Type"
landlord_built_form = None
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "UPRN"
landlord_property_id = "Reference"
landlord_sap = None
outcomes_filename = None
outcomes_sheetname = None
@ -409,10 +409,6 @@ def app():
)
asset_list.merge_data(epc_df)
# asset_list.standardised_asset_list = asset_list.standardised_asset_list[
# asset_list.standardised_asset_list["domna_full_address"]
# != "120 Airdrie Crescent, Burnley, Lancashire"
# ]
asset_list.extract_attributes()
asset_list.identify_worktypes()
@ -426,27 +422,6 @@ def app():
os.path.join(data_folder, ".".join(data_filename.split(".")[:-1]))
+ " - Standardised.xlsx"
)
# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data
# Determine inspections priority
# solar_jobs = asset_list.standardised_asset_list[~pd.isnull(asset_list.standardised_asset_list["solar_reason"])][
# "domna_postcode"].unique()
# asset_list.standardised_asset_list["in_solar_area"] = asset_list.standardised_asset_list["domna_postcode"].isin(
# solar_jobs
# )
# # Same for cav
# cavity_jobs = asset_list.standardised_asset_list[
# ~pd.isnull(asset_list.standardised_asset_list["cavity_reason"])
# ]["domna_postcode"].unique()
# asset_list.standardised_asset_list["in_cavity_area"] = asset_list.standardised_asset_list["domna_postcode"].isin(
# cavity_jobs
# )
# # We prioritise properties that are in solar areas and cavity areas
# import numpy as np
# asset_list.standardised_asset_list["inspection_priority"] = np.where(
# asset_list.standardised_asset_list["in_solar_area"] | asset_list.standardised_asset_list["in_cavity_area"],
# 1, 2
# )
with pd.ExcelWriter(filename) as writer:
asset_list.standardised_asset_list.to_excel(
@ -488,5 +463,3 @@ def app():
asset_list.duplicated_addresses.to_excel(
writer, sheet_name="Duplicate Properties", index=False
)

103
asset_list/mappings/built_form.py
View file

@ -528,6 +528,107 @@ BUILT_FORM_MAPPINGS = {
'House: Semi Detached: Top Floor': 'semi-detached',
'House: End Terrace: Ground Floor': 'end-terrace',
'Maisonette: Enclosed End Terrace: Mid Floor': 'enclosed end-terrace',
'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace'
'Bungalow: EnclosedEndTerrace': 'enclosed end-terrace',
'2 BED MID TERRACED HOUSE': 'mid-terrace',
'4 BED SEMI DETACHED-PARLOURED': 'semi-detached',
'2 BED END TERRACED HOUSE': 'end-terrace',
'3 BED MID TERRACED HOUSE': 'mid-terrace',
'3 BED SEMI DETACHED HOUSE': 'semi-detached',
'3 BED MID TERRACE - PARLOURED': 'mid-terrace',
'3 BED END TERRACE - PARLOURED': 'end-terrace',
'4 BED+ END TERRACED HOUSE': 'end-terrace',
'3 BED END TERRACED HOUSE': 'end-terrace',
'3 BED SEMI DETACHED-PARLOURED': 'semi-detached',
'4 BED+ END TERRACE - PARLOURED': 'end-terrace',
'2 BED SEMI DETACHED HOUSE': 'semi-detached',
'3 BED DETACHED HOUSE': 'detached',
'2 BED GRD FLR COTT FLT-CNT STR': 'ground floor',
'2 BED 1ST FLOOR WALKUP FLAT': 'mid-floor',
'1 BED GRD FL COTT FLAT-OWN ENT': 'ground floor',
'1 BED 1ST FL WALK UP DECK ACC': 'mid-floor',
'2 BED MAISONETTE UPPER COM ENT': 'mid-floor',
'2 BED GRD FLR COTT FLT OWN ENT': 'ground floor',
'1 BED BUNGALOW': 'unknown',
'2 BED GRD FL COTT FLT-OWN ENTR': 'ground floor',
'1 BED 1ST FL COTT FLT-CNT STR': 'mid-floor',
'1 BED GRD FL WALK UP OWN ENT': 'ground floor',
'1 BED GRD FLOOR WALKUP FLAT': 'ground floor',
'2 BED GRD FLOOR WALKUP FLAT': 'ground floor',
'2 BED 1ST FLR FLT-SHELTERED': 'mid-floor',
'2 BED BUNGALOW': 'unknown',
'2 BED GRD FLR COTT FLT(P)-1950': 'ground floor',
'Ground Floor Front Left': 'ground floor',
'End-Terrace House': 'end-terrace',
'Ground floor': 'ground floor',
'Ground Floor Front Right': 'ground floor',
'End Terrace (GII List)': 'end-terrace',
'Semi Detached House': 'semi-detached',
'Ground Floor Right': 'ground floor',
'PB Ground Floor Flat': 'ground floor',
'Basement and Ground Floor': 'ground floor',
'Semi-detached bungalow': 'detached',
'Detached Cottage': 'detached',
'Lower & Ground Floor': 'ground floor',
'Ground FLoor Flat': 'ground floor',
'ground floor': 'ground floor',
'Ground Floor Left': 'ground floor',
'Semi-detached House': 'detached',
'Basement & Lower Ground': 'basement',
'Semi-Detached House': 'detached',
'Ground floor flat -': 'ground floor',
'Basement Flat': 'basement',
'semi-detached bungalow': 'semi-detached',
'Lower Ground Floor Flat': 'ground floor',
'Ground floor Flat': 'ground floor',
'Ground Floor flat': 'ground floor',
'Ground': 'ground floor',
'Semi detached Bungalow': 'semi-detached',
'ground floor flat': 'ground floor',
'Mid terrace House': 'mid-terrace',
'Raised Ground Floor': 'ground floor',
'Basement Floor': 'basement',
'Second floor flat': 'mid-floor',
'Fourth Floor Flat': 'mid-floor',
'First/Second Maisonette': 'mid-floor',
'Ground/First': 'ground floor',
'First and Second Floor': 'mid-floor',
'Terrace House': 'mid-terrace',
'1st/2nd Floor Maisonette': 'mid-floor',
'Semi-det House': 'semi-detached',
'First': 'mid-floor',
'Ground & First Floor': 'ground floor',
'End of Terrace House': 'end-terrace',
'2nd Floor Purpose Built': 'mid-floor',
'First/Second Floor Maison': 'mid-floor',
'GFF purpose built': 'ground floor',
'Second': 'mid-floor',
'Semi-det House (GII List)': 'semi-detached',
'3rd and 4th Floor': 'mid-floor',
'First Floor flat': 'mid-floor',
'Mid-Terrace House': 'mid-terrace',
'1st & 2nd Floors': 'mid-floor',
'Ground/first floor': 'ground floor',
'FFF purpose built': 'mid-floor',
'Second floor': 'mid-floor',
'Second/Third floor': 'mid-floor',
'First floor Flat': 'mid-floor',
'First floor': 'mid-floor',
'Lower Ground Flat': 'basement',
'First Floor Rear Flat': 'mid-floor',
'First & Second Floor': 'mid-floor',
'Ground & Lower Ground': 'basement',
'First Floor Rear': 'mid-floor',
'First & Second': 'mid-floor',
'First Floor Front': 'mid-floor',
'First & Second Floors': 'mid-floor',
'First/Second Floor': 'mid-floor',
'Sem-detach house': 'semi-detached',
'Second Floor Flat (Top)': 'top-floor',
'3 FloorTerrace House': 'mid-terrace',
'First floor flat': 'mid-floor',
'First & Second Floor Flat': 'mid-floor',
'First Floor Purpose Built': 'mid-floor',
'Purpose built First Floor': 'mid-floor',
}

19
asset_list/mappings/heating_systems.py
View file

@ -498,6 +498,23 @@ HEATING_MAPPINGS = {
'Boiler: A rated Combi, System 2: Boiler: A rated Combi': 'gas combi boiler',
'System 2: Boiler: A rated Regular Boiler, Boiler: A rated Regular Boiler': 'gas boiler, radiators',
'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler'
'Boiler: A rated Combi, System 2: Boiler: C rated Combi': 'gas combi boiler',
'IDEAL ISAR HE30': 'gas combi boiler',
'WORCESTER GREENSTAR 25 SI': 'gas combi boiler',
'POTTERTON PROMAX COMBI 28 HE PLUS': 'gas combi boiler',
'WORCESTER GREENSTAR 28I JUNIOR': 'gas combi boiler',
'BAXI ASSURE 25 COMBI': 'gas combi boiler',
'POTTERTON PROMAX COMBI 28 HE PLUS A': 'gas combi boiler',
'WORCESTER GREENSTAR 30 SI': 'gas combi boiler',
'POTTERTON SUPRIMA 40L': 'gas boiler, radiators',
'POTTERTON ASSURE 30 COMBI': 'gas combi boiler',
'POTTERTON PROMAX 28 COMBI ERP': 'gas combi boiler',
'BAXI ASSURE 30 COMBI': 'gas combi boiler',
'POTTERTON PROMAX 18 SYSTEM ERP': 'gas boiler, radiators',
'POTTERTON PROMAX COMBI 33 HE PLUS A': 'gas combi boiler',
'POTTERTON SUPRIMA 40 HE': 'gas boiler, radiators',
'FERROLI MODENA 102': 'gas boiler, radiators',
'POTTERTON PROMAX COMBI 24 HE PLUS A': 'gas combi boiler'
}

5
asset_list/mappings/property_type.py
View file

@ -444,6 +444,9 @@ PROPERTY_MAPPING = {
'Warden Bungalow': 'bungalow',
'Warden Flat': 'flat',
'Upper Floor Flat': 'flat',
'Extracare Scheme': 'other'
'Extracare Scheme': 'other',
'SHELTERED': 'unknown',
'PARLOUR': 'unknown',
}

4
asset_list/mappings/roof.py
View file

@ -320,6 +320,8 @@ ROOF_CONSTRUCTION_MAPPINGS = {
'Pitched (slates or tiles) access to loft, 100mm': 'pitched insulated',
'Pitched (slates or tiles) no loft access, 200mm': 'pitched insulated',
'Pitched (slates or tiles) access to loft, 200mm': 'pitched insulated',
'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation'
'Pitched (slates or tiles) access to loft, 50mm': 'pitched less than 100mm insulation',
'Pitched roofs': 'pitched unknown insulation',
}

5
asset_list/mappings/walls.py
View file

@ -369,6 +369,9 @@ WALL_CONSTRUCTION_MAPPINGS = {
'Solid Brick, As built': 'solid brick unknown insulation',
'System built, As built': 'system built unknown insulation',
'Timber frame, As built': 'timber frame unknown insulation',
'Cavity, As built': 'cavity unknown insulation'
'Cavity, As built': 'cavity unknown insulation',
'FILLED CAVITY': 'filled cavity',
'EXTERNAL': 'insulated solid brick',
'AS BUILT': 'other'
}

198
backend/Property.py
View file

@ -660,8 +660,6 @@ class Property:
self.set_floor_type()
self.set_floor_level()
self.set_windows_count()
self.set_energy_source()
self.find_energy_sources()
self.set_current_energy(kwh_client, kwh_predictions)
def set_solar_panel_configuration(self, solar_panel_configuration):
@ -1168,202 +1166,6 @@ class Property:
if condition_data.get("windows_area") is not None \
else None
def set_energy_source(self):
"""
This method sets the energy source of the property, based on the mains gas flag and energy tariff.
"""
# Default to "electricity_and_gas" to cover most scenarios including when mains_gas_flag is True
energy_source = "electricity_and_gas"
# If the tariff explicitly indicates electricity use without a dual indication and mains_gas_flag is not True
# We check for the common electricity tariffs
if not self.data["mains-gas-flag"] and self.data["energy-tariff"] in [
"Single",
"off-peak 7 hour",
"off-peak 10 hour",
"off-peak 18 hour",
"standard tariff",
"24 hour",
]:
energy_source = "electricity"
# Set the energy source based on the conditions above
self.energy_source = energy_source
def find_energy_sources(self):
# Based on the heating and the hot water
heating_fuel_mapping = {
'has_mains_gas': 'Natural Gas',
'has_electric': 'Electricity',
'has_oil': 'Oil',
'has_wood_logs': 'Wood Logs',
'has_coal': 'Coal',
'has_anthracite': 'Anthracite',
'has_smokeless_fuel': 'Smokeless Fuel',
'has_lpg': 'LPG',
'has_b30k': 'B30K Biofuel',
'has_air_source_heat_pump': 'Electricity',
'has_ground_source_heat_pump': 'Electricity',
'has_water_source_heat_pump': 'Electricity',
'has_electric_heat_pump': 'Electricity',
'has_solar_assisted_heat_pump': 'Electricity',
'has_exhaust_source_heat_pump': 'Electricity',
'has_community_heat_pump': 'Electricity',
'has_wood_pellets': 'Wood Pellets',
'has_community_scheme': 'Varied (Community Scheme)',
"has_dual_fuel_mineral_and_wood": 'Wood Logs',
"has_electricaire": 'Electricity',
"has_wood_chips": 'Wood Logs'
}
# Hot water
heater_type_to_fuel = {
'gas instantaneous': 'Natural Gas',
'electric heat pump': 'Electricity',
'electric immersion': 'Electricity',
'gas boiler': 'Natural Gas',
'oil boiler': 'Oil',
'electric instantaneous': 'Electricity',
'gas multipoint': 'Natural Gas',
'heat pump': 'Electricity',
'solid fuel boiler': 'Solid Fuel',
'solid fuel range cooker': 'Solid Fuel',
'room heaters': 'Varied', # Could be any fuel, further specifics needed based on context
"single-point gas": "Natural Gas"
}
# Define a mapping from system types to general categories or modifications of fuel types
system_type_modification = {
'from main system': 'Main System',
'from secondary system': 'Secondary System',
'from second main heating system': 'Secondary System',
'community scheme': 'Community Scheme'
}
hotwater_appliance_to_fuel = {
'gas range cooker': 'Natural Gas',
'oil range cooker': 'Oil'
}
fuel_map = {
None: "Natural Gas (Community Scheme)",
"mains gas": "Natural Gas (Community Scheme)",
"biomass": "Smokeless Fuel",
"electricity": "Electricity",
"biogas": "Smokeless Fuel",
"heat network": "Natural Gas (Community Scheme)",
"lpg": 'LPG',
"biodiesel": "Smokeless Fuel",
"b30d": "B30K Biofuel",
"coal": "Coal",
"oil": "Oil",
"unknown": None # Handle - anything post 2020 is electricity else gas
}
self.heating_energy_source = list({
fuel for key, fuel in heating_fuel_mapping.items() if self.main_heating.get(key, False)
})
if set(self.heating_energy_source) == {'Electricity', 'Natural Gas'}:
# It means they have mixed heating so we take the primary one, based on main fuel
# This will probably happen in the case of an extension
if self.main_fuel["clean_description"] in ["Mains gas not community", "Mains gas community"]:
self.heating_energy_source = ['Natural Gas']
else:
self.heating_energy_source = ['Electricity']
if set(self.heating_energy_source) == {'Electricity', 'LPG'}:
if self.main_fuel["clean_description"] in ["Lpg not community", "Lpg community"]:
self.heating_energy_source = ['LPG']
else:
self.heating_energy_source = ['Electricity']
if set(self.heating_energy_source) == {'Natural Gas', 'Wood Logs'}:
# It means they have mixed heating so we take the primary one, based on main fuel
# This will probably happen in the case of an extension
if self.main_fuel["clean_description"] in ["Mains gas not community", "Mains gas community"]:
self.heating_energy_source = ['Natural Gas']
else:
self.heating_energy_source = ['Wood Logs']
if len(self.heating_energy_source) > 1 and "Varied (Community Scheme)" not in self.heating_energy_source:
# We might have something like heating energy source equal to ['Natural Gas', 'Varied (Community Scheme)']
# so we treat this as community heating
raise Exception("Investigate me")
if len(self.heating_energy_source) == 0:
heating_flags = {
v for k, v in self.main_heating.items() if k not in ["original_description", "clean_description"]
}
hotwater_flags = {
v for k, v in self.hotwater.items() if k not in ["original_description", "clean_description"]
}
# If all flags are zero, we have a no data example
if (heating_flags == {False} or hotwater_flags == {None}) and (
hotwater_flags == {False} or hotwater_flags == {None}):
# We have nodata so we try and rely on main fuel
if self.main_fuel["fuel_type"] in fuel_map: # We assume when None as it's unknown
mapped_fuel = fuel_map[self.main_fuel["fuel_type"]]
self.heating_energy_source = mapped_fuel
self.hot_water_energy_source = mapped_fuel
return
else:
raise NotImplementedError(f"Unhandled fuel {self.main_fuel['fuel_type']}")
# We handle edge case where no heating system is indicated
if self.main_fuel["fuel_type"] in fuel_map:
mapped_fuel = fuel_map[self.main_fuel["fuel_type"]]
self.heating_energy_source = mapped_fuel
self.hot_water_energy_source = mapped_fuel
return
if len(self.heating_energy_source) > 1:
# We treat this as a community scheme
self.heating_energy_source = ["Varied (Community Scheme)"]
self.heating_energy_source = self.heating_energy_source[0]
if self.heating_energy_source == "Varied (Community Scheme)":
if self.main_fuel["fuel_type"] in fuel_map: # We assume when None as it's unknown
mapped_to = fuel_map[self.main_fuel["fuel_type"]]
if mapped_to is None and self.main_fuel["fuel_type"] == "unknown":
# Handle logic based on age band
if self.year_built >= 2020:
self.heating_energy_source = "Electricity"
else:
self.heating_energy_source = "Natural Gas (Community Scheme)"
else:
self.heating_energy_source = mapped_to
else:
raise NotImplementedError(f"Unhandled fuel {self.main_fuel['fuel_type']}")
if self.hotwater["heater_type"] is not None:
self.hot_water_energy_source = heater_type_to_fuel[self.hotwater["heater_type"]]
if self.hotwater["extra_features"] == "plus solar":
self.hot_water_energy_source = self.heating_energy_source + " + Solar Thermal"
return
elif self.hotwater["system_type"] is not None:
fuel = system_type_modification[self.hotwater["system_type"]]
if self.hotwater["extra_features"] == "plus solar":
self.hot_water_energy_source = self.heating_energy_source + " + Solar Thermal"
return
if fuel in ['Main System', "Community Scheme"]:
self.hot_water_energy_source = self.heating_energy_source
elif fuel in ['Secondary System']:
# Check the secondary heating system
secondary_heating = self.data["secondheat-description"]
self.hot_water_energy_source = assumptions.DESCRIPTIONS_TO_FUEL_TYPES[secondary_heating]["fuel"]
else:
raise NotImplementedError(f"Investiage me - unhandled hot water fuel {fuel}")
else:
self.hot_water_energy_source = hotwater_appliance_to_fuel[self.hotwater["appliance"]]
def is_ashp_valid(self, measures):
if "air_source_heat_pump" in self.non_invasive_recommendations:

2
backend/README.md
View file

@ -364,5 +364,5 @@ Here's what you should do:
function.
By following these steps, you should have your custom domain properly configured and pointing to your AWS Lambda
function via the CloudFront distribution.
function via the CloudFront distribution

5
backend/app/db/base.py Normal file
View file

@ -0,0 +1,5 @@
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
pass

6
backend/app/db/functions/tasks/Tasks.py
View file

@ -11,7 +11,7 @@ from sqlmodel import Session, select
from backend.app.db.connection import get_db_session
# ---- Models ----
from backend.app.db.models.tasks import Task, SubTask
from backend.app.db.models.tasks import SourceEnum, Task, SubTask
# ============================================================
@ -268,6 +268,8 @@ class TasksInterface:
service: Optional[str] = None,
inputs: Optional[Dict[str, Any]] = None,
task_only: bool = False,
source: Optional[SourceEnum] = None,
source_id: Optional[str] = None,
):
"""
Create a new Task record, and an initial SubTask in waiting state. Can also be used to create just
@ -286,6 +288,8 @@ class TasksInterface:
status="waiting",
job_started=now,
job_completed=None,
source=source,
source_id=source_id,
)
session.add(task)

4
backend/app/db/models/addresses.py
View file

@ -7,9 +7,7 @@ from sqlalchemy import (
func,
UniqueConstraint,
)
from sqlalchemy.orm import declarative_base
Base = declarative_base()
from backend.app.db.base import Base
class PostcodeSearch(Base):

4
backend/app/db/models/condition.py
View file

@ -7,12 +7,12 @@ from sqlalchemy import (
String,
Enum as SqlEnum,
)
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.orm import relationship
from backend.condition.domain.aspect_type import AspectType
from backend.condition.domain.element_type import ElementType
Base = declarative_base()
from backend.app.db.base import Base
ElementTypeDb = SqlEnum(
ElementType,

10
backend/app/db/models/energy_assessments.py
View file

@ -1,10 +1,8 @@
from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.dialects.postgresql import ENUM as PgEnum
import enum
from datetime import datetime
Base = declarative_base()
from backend.app.db.base import Base
from sqlalchemy import Column, Integer, BigInteger, Text, Float, DateTime, Boolean, Date, ForeignKey
from sqlalchemy.dialects.postgresql import ENUM as PgEnum
class EnergyAssessment(Base):
@ -190,7 +188,7 @@ class EnergyAssessmentDocuments(Base):
id = Column(BigInteger, primary_key=True, autoincrement=True)
uprn = Column(BigInteger, nullable=False)
energy_assessment_id = Column(BigInteger, ForeignKey('energy_assessments.id'), nullable=False)
document_type = Column(PgEnum(DocumentTypeEnum, name="document_type", create_type=False), nullable=False)
document_type = Column(PgEnum(DocumentTypeEnum, name="document_type"), nullable=False)
document_location = Column(Text, nullable=False)
uploaded_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow)
scenario_id = Column(BigInteger, ForeignKey('energy_assessment_scenarios.id'), nullable=True)

5
backend/app/db/models/epc.py
View file

@ -4,11 +4,8 @@ from sqlalchemy import (
String,
JSON,
TIMESTAMP,
UniqueConstraint,
)
from sqlalchemy.orm import declarative_base
Base = declarative_base()
from backend.app.db.base import Base
class EpcStore(Base):

5
backend/app/db/models/funding.py
View file

@ -3,20 +3,17 @@ import enum
from sqlalchemy import (
Column,
Integer,
String,
Float,
Enum,
TIMESTAMP,
BigInteger,
ForeignKey,
)
from sqlalchemy.orm import declarative_base
from sqlalchemy.sql import func
from backend.app.db.base import Base
from backend.app.db.models.recommendations import PlanModel
from backend.app.db.models.materials import MaterialType, Material
Base = declarative_base()
class SchemeEnum(enum.Enum):
eco4 = "eco4"

4
backend/app/db/models/inspections.py
View file

@ -9,11 +9,9 @@ from sqlalchemy import (
Enum,
ForeignKey,
)
from sqlalchemy.ext.declarative import declarative_base
from backend.app.db.base import Base
from backend.app.db.models.portfolio import PropertyModel
Base = declarative_base()
# -------------------------------------------------------------------
# ENUM DEFINITIONS (equivalent to drizzle pgEnum calls)

3
backend/app/db/models/materials.py
View file

@ -1,10 +1,9 @@
import enum
from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP, Boolean
from sqlalchemy.orm import declarative_base
from sqlalchemy.sql import func
Base = declarative_base()
from backend.app.db.base import Base
class MaterialType(enum.Enum):

4
backend/app/db/models/non_intrusive_surveys.py
View file

@ -1,7 +1,5 @@
from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer
from sqlalchemy.orm import declarative_base
Base = declarative_base()
from backend.app.db.base import Base
class NonIntrusiveSurvey(Base):

11
backend/app/db/models/portfolio.py
View file

@ -4,6 +4,7 @@ import datetime
from sqlalchemy import (
Column,
Integer,
BigInteger,
Text,
Boolean,
Float,
@ -12,12 +13,10 @@ from sqlalchemy import (
ForeignKey,
CheckConstraint,
)
from sqlalchemy.ext.declarative import declarative_base
from backend.app.db.base import Base
from backend.app.db.models.users import UserModel # noqa
from backend.app.db.models.materials import MaterialType
Base = declarative_base()
class PortfolioStatus(enum.Enum):
SCOPING = "scoping"
@ -32,7 +31,7 @@ class PortfolioStatus(enum.Enum):
NEEDS_REVIEW = "needs review"
class PortfolioGoal(enum.Enum): # TODO: Move to domain?
class PortfolioGoal(enum.Enum): # TODO: Move to domain?
VALUATION_IMPROVEMENT = "Valuation Improvement"
INCREASING_EPC = "Increasing EPC"
REDUCING_CO2_EMISSIONS = "Reducing CO2 emissions"
@ -116,9 +115,9 @@ class PropertyModel(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
portfolio_id = Column(Integer, ForeignKey("portfolio.id"), nullable=False)
creation_status = Column(Enum(PropertyCreationStatus), nullable=False)
uprn = Column(Integer)
uprn = Column(BigInteger)
landlord_property_id = Column(Text)
building_reference_number = Column(Integer)
building_reference_number = Column(BigInteger)
status = Column(
Enum(PortfolioStatus, values_callable=lambda x: [e.value for e in x]),
nullable=False,

53
backend/app/db/models/recommendations.py
View file

@ -1,3 +1,4 @@
import enum
from typing import Iterable, List, NamedTuple, Optional, Type
from sqlalchemy import (
Column,
@ -9,17 +10,15 @@ from sqlalchemy import (
ForeignKey,
Enum,
)
from sqlalchemy.orm import declarative_base, Mapped, mapped_column
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.sql import func
from datetime import datetime
from backend.app.db.base import Base
from backend.app.db.models.portfolio import Portfolio, PortfolioGoal, PropertyModel
from backend.app.db.models.materials import Material
from backend.app.db.models.portfolio import Epc
from datatypes.enums import QuantityUnits
import enum
Base = declarative_base()
def portfolio_goal_values(enum_cls: Type[PortfolioGoal]) -> List[str]:
@ -55,19 +54,47 @@ class Recommendation(Base):
class RecommendationMaterials(Base):
__tablename__ = "recommendation_materials"
id = Column(BigInteger, primary_key=True, autoincrement=True)
recommendation_id = Column(
BigInteger, ForeignKey("recommendation.id"), nullable=False
id: Mapped[int] = mapped_column(
BigInteger, primary_key=True, autoincrement=True
)
material_id = Column(BigInteger, ForeignKey(Material.id), nullable=False)
created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())
depth = Column(Float, nullable=False)
quantity = Column(Float, nullable=False)
quantity_unit = Column(
recommendation_id: Mapped[int] = mapped_column(
BigInteger,
ForeignKey("recommendation.id"),
nullable=False,
)
material_id: Mapped[int] = mapped_column(
BigInteger,
ForeignKey(Material.id),
nullable=False,
)
created_at: Mapped[datetime] = mapped_column(
TIMESTAMP,
nullable=False,
server_default=func.now(),
)
depth: Mapped[float] = mapped_column(
Float,
nullable=False,
)
quantity: Mapped[float] = mapped_column(
Float,
nullable=False,
)
quantity_unit: Mapped[QuantityUnits] = mapped_column(
Enum(QuantityUnits, values_callable=lambda x: [e.value for e in x]),
nullable=False,
)
estimated_cost = Column(Float, nullable=False)
estimated_cost: Mapped[float] = mapped_column(
Float,
nullable=False,
)
class PlanTypeEnum(enum.Enum): # TODO: move this to domain?

4
backend/app/db/models/solar.py
View file

@ -2,9 +2,7 @@ import datetime
import pytz
from enum import Enum as PyEnum
from sqlalchemy import Column, Integer, Float, DateTime, JSON, BigInteger, ForeignKey, Enum, Boolean
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
from backend.app.db.base import Base
class Solar(Base):

33
backend/app/db/models/tasks.py
View file

@ -1,14 +1,24 @@
import enum
from typing import Optional
from datetime import datetime
from uuid import UUID, uuid4
from sqlalchemy import Column, Enum
from sqlmodel import SQLModel, Field, Relationship
class SourceEnum(enum.Enum): # TODO: move to domain?
PORTFOLIO = "portfolio_id"
class Task(SQLModel, table=True):
__tablename__ = "tasks"
id: UUID = Field(default_factory=uuid4, primary_key=True, index=True, )
id: UUID = Field(
default_factory=uuid4,
primary_key=True,
index=True,
)
task_source: str
job_started: Optional[datetime] = None
job_completed: Optional[datetime] = None
@ -16,13 +26,32 @@ class Task(SQLModel, table=True):
service: Optional[str] = None
updated_at: datetime = Field(default_factory=datetime.utcnow)
# source: Mapped[Optional[SourceEnum]] = mapped_column(Enum(SourceEnum)) <- SQLAlchemy not SQLModel
source: Optional[SourceEnum] = Field(
default=None,
sa_column=Column(
Enum(
SourceEnum,
name="source",
values_callable=lambda e: [m.value for m in e],
),
nullable=True,
),
)
source_id: Optional[str] = None
sub_tasks: list["SubTask"] = Relationship(back_populates="task")
class SubTask(SQLModel, table=True):
__tablename__ = "sub_task"
id: UUID = Field(default_factory=uuid4, primary_key=True, index=True, )
id: UUID = Field(
default_factory=uuid4,
primary_key=True,
index=True,
)
task_id: UUID = Field(foreign_key="tasks.id")
job_started: Optional[datetime] = None

4
backend/app/db/models/users.py
View file

@ -1,8 +1,6 @@
from sqlalchemy import Column, Integer, String, DateTime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import func
Base = declarative_base()
from backend.app.db.base import Base
class UserModel(Base):

3
backend/app/db/models/whlg.py
View file

@ -1,4 +1,3 @@
import uuid
from typing import Optional
from sqlmodel import SQLModel, Field
@ -12,4 +11,4 @@ class Whlg(SQLModel, table=True):
index=True,
)
postcode: str = Field(nullable=False)
postcode: str = Field(nullable=False)

14
backend/app/plan/router.py
View file

@ -9,6 +9,8 @@ import asyncio
from datetime import datetime
from fastapi import APIRouter, Depends
from backend.app.db.connection import db_session
from backend.app.db.models.tasks import SourceEnum
from backend.app.dependencies import validate_token
from backend.app.plan.schemas import PlanTriggerRequest
from backend.app.config import get_settings
@ -58,9 +60,9 @@ async def trigger_categorisation(
num_property_batches: int = math.ceil(len(property_ids) / properties_per_batch)
logger.info("total_plans_to_update", total_plans_to_update)
logger.info("properties_per_batch", properties_per_batch)
logger.info("num_property_buckets", num_property_batches)
logger.info("total_plans_to_update: %s", total_plans_to_update)
logger.info("properties_per_batch: %s", properties_per_batch)
logger.info("num_property_batchess: %s", num_property_batches)
# Create task
task_id, _ = TasksInterface.create_task(
@ -68,6 +70,8 @@ async def trigger_categorisation(
service="plan_categorisation",
inputs=payload.model_dump(),
task_only=True,
source=SourceEnum.PORTFOLIO,
source_id=str(payload.portfolio_id),
)
# Dispatch requests to lambdas
@ -102,7 +106,9 @@ async def trigger_categorisation(
)
logger.info(
f"Chunk {batch_index} sent to SQS. {len(batch_property_ids)} Property IDs in batch (total {len(property_ids)}). Property IDs {min(batch_property_ids)}–{max(batch_property_ids)}. Message ID: {response.get('MessageId')}"
f"Chunk {batch_index} sent to SQS. {len(batch_property_ids)} Property IDs in batch (total "
f"{len(property_ids)}). Property IDs {min(batch_property_ids)}–{max(batch_property_ids)}. Message ID: "
f"{response.get('MessageId')}"
)
await asyncio.sleep(0.05) # Small delay to avoid SQS throttling

4
backend/app/plan/schemas.py
View file

@ -12,6 +12,10 @@ WALL_INSULATION_MEASURES = ["internal_wall_insulation", "external_wall_insulatio
ROOF_INSULATION_MEASURES = [
"loft_insulation", "flat_roof_insulation", "room_roof_insulation", "sloping_ceiling_insulation"
]
WALL_INSULATION_WITH_VENTILATION_MEASURES = [
"internal_wall_insulation+mechanical_ventilation", "external_wall_insulation+mechanical_ventilation",
"cavity_wall_insulation+mechanical_ventilation"
]
# Both all and roof insulaiton measures are eligible for ECO4. These are the remaining fabric and heating measures
# This is based on th measures we have recommendations for

169
backend/export/README.md Normal file
View file

@ -0,0 +1,169 @@
# 🧪 Running Tests in PyCharm (macOS + pytest-postgresql)
Our test suite uses `pytest` and `pytest-postgresql`, which
automatically spins up a temporary PostgreSQL instance.
On Linux (including GitHub Actions), PostgreSQL binaries are installed
in standard system locations.\
On macOS (Homebrew), they are not --- so PyCharm needs a small
configuration tweak to locate `pg_ctl`.
This guide explains how to run and debug tests locally in PyCharm
without modifying test code.
------------------------------------------------------------------------
## ✅ Prerequisites
### Devcontainer
Postgres install is included in the devcontainer, so no additional setup is needed.
Running
```bash
make test
```
Will instigate the test suite, which will automatically start a temporary PostgreSQL instance.
### Local MacOS
1. Install PostgreSQL via Homebrew:
``` bash
brew install postgresql
```
2. Confirm `pg_ctl` exists:
``` bash
which pg_ctl
```
Typical output:
/opt/homebrew/bin/pg_ctl
------------------------------------------------------------------------
# 🚀 Running Tests in PyCharm
## Step 1 --- Create a PyCharm pytest Run Configuration
1. Open the test file.
2. Click the green ▶ next to the test.
3. Choose **"Edit Run Configuration..."**
You should see something like:
- **Target:** `backend/export/tests/test_export.py`
- **Working directory:** Project root (e.g.`Model/`)
------------------------------------------------------------------------
## Step 2 --- Add Required Override (macOS Only)
In the Run Configuration:
### ➜ "Additional Arguments"
Add:
--override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl
This tells `pytest-postgresql` where `pg_ctl` lives on macOS.
Without this, PyCharm may fail with:
ExecutableMissingException: Could not found pg_config executable
------------------------------------------------------------------------
## Step 3 --- Run or Debug
You can now:
- Click ▶ Run\
- Click 🐞 Debug\
- Set breakpoints normally
The temporary PostgreSQL instance will start automatically.
------------------------------------------------------------------------
# 🔍 Why This Is Needed
`pytest-postgresql` defaults to a Linux-style path:
/usr/lib/postgresql/<version>/bin/pg_ctl
That path exists on Ubuntu (CI), but not on macOS.
On macOS, Homebrew installs PostgreSQL in:
/opt/homebrew/bin/
The `--override-ini` flag safely overrides the executable path
**locally**, without modifying:
- test files\
- `conftest.py`\
- `pytest.ini`\
- CI configuration
This ensures:
- ✅ Tests still work in GitHub Actions\
- ✅ Tests still work for Linux users\
- ✅ macOS developers can debug in PyCharm\
- ✅ No repository-specific hacks are required
------------------------------------------------------------------------
# 🛠 Optional: Using a Local `.env` File
If you prefer not to hardcode the override in the run configuration:
1. Create a local file:
```{=html}
<!-- -->
```
.env.local
2. Add:
```{=html}
<!-- -->
```
PYTEST_ADDOPTS=--override-ini=postgresql_exec=/opt/homebrew/bin/pg_ctl
3. In PyCharm:
- Open the Run Configuration
- Add `.env.local` under **"Paths to .env files"**
------------------------------------------------------------------------
# 🧪 Running Tests via Terminal (Recommended for CI Parity)
For normal execution outside PyCharm:
``` bash
make test
```
These already work without additional configuration.
------------------------------------------------------------------------
# 🧠 Summary
Environment Works Without Override? Needs `--override-ini`?
------------------------ ------------------------- -------------------------
GitHub Actions (Linux) ✅ Yes ❌ No
Linux local ✅ Yes ❌ No
macOS terminal (tox) ✅ Yes ❌ No
macOS PyCharm debugger ❌ No ✅ Yes

227
backend/export/property_scenarios/db_functions.py Normal file
View file

@ -0,0 +1,227 @@
from typing import List, Any, Dict, Optional, Tuple, Sequence
import pandas as pd
from sqlalchemy import select
from sqlalchemy.orm import Session
from sqlalchemy.engine import Row
from collections import defaultdict
from backend.app.db.models.recommendations import (
Recommendation,
PlanModel,
PlanRecommendations,
RecommendationMaterials,
)
from backend.app.db.models.portfolio import (
PropertyModel,
PropertyDetailsEpcModel,
)
from backend.app.db.models.materials import Material
from utils.logger import setup_logger
logger = setup_logger()
class DbMethods:
def __init__(self, session: Session) -> None:
self.session = session
def get_properties(self, portfolio_id: int) -> pd.DataFrame:
"""
Function to fetch the property data, for property scenario exports
:param portfolio_id:
:return:
"""
stmt = (
select(PropertyModel, PropertyDetailsEpcModel)
.join(
PropertyDetailsEpcModel,
PropertyModel.id == PropertyDetailsEpcModel.property_id,
)
.where(PropertyModel.portfolio_id == portfolio_id)
)
rows: Sequence[Row[Tuple[PropertyModel, PropertyDetailsEpcModel]]] = (
self.session.execute(stmt).all()
)
data: List[Dict[str, Any]] = [
{
**{
col.name: getattr(property_model, col.name)
for col in PropertyModel.__table__.columns.values()
},
**{
col.name: getattr(epc_model, col.name)
for col in PropertyDetailsEpcModel.__table__.columns.values()
},
}
for property_model, epc_model in rows
]
return pd.DataFrame(data)
def get_latest_plans(
self,
portfolio_id: int,
scenario_ids: Optional[List[int]] = None,
default_only: bool = False,
) -> pd.DataFrame:
"""
Fetch latest plans.
Modes:
1) Scenario mode: latest per (scenario_id, property_id)
2) Default mode: latest default plan per property (ignores scenario_ids)
"""
# -----------------------------
# Sanity checks
# -----------------------------
if default_only and scenario_ids:
# Override scenario_ids to make it explicit that they will be ignored in the query
scenario_ids = None
if not default_only and not scenario_ids:
raise ValueError(
"Either scenario_ids must be provided "
"or default_only must be True."
)
# -----------------------------
# Filter on just the default plans - we ignore the scenario ids. NOTE - this is specific to postgres
# and relies on DISTINCT ON behaviour.
# -----------------------------
if default_only:
# Latest default plan per property (ignore scenarios entirely)
# DISTINCT ON (property_id) keeps the first row per property,
# ordered by created_at DESC so we get the newest one.
stmt = (
select(PlanModel)
.where(
PlanModel.portfolio_id == portfolio_id,
PlanModel.is_default.is_(True),
)
.distinct(PlanModel.property_id)
.order_by(
PlanModel.property_id,
PlanModel.created_at.desc(),
)
)
else:
# Latest plan per (scenario_id, property_id)
# DISTINCT ON (scenario_id, property_id) keeps the newest
# plan per scenario/property combination.
assert scenario_ids is not None
stmt = (
select(PlanModel)
.where(
PlanModel.portfolio_id == portfolio_id,
PlanModel.scenario_id.in_(scenario_ids),
)
.distinct(
PlanModel.scenario_id,
PlanModel.property_id,
)
.order_by(
PlanModel.scenario_id,
PlanModel.property_id,
PlanModel.created_at.desc(),
)
)
logger.info("Fetching plans")
plans: Sequence[PlanModel] = self.session.scalars(stmt).all()
return pd.DataFrame(
[
{
col.name: getattr(plan, col.name)
for col in PlanModel.__table__.columns.values()
}
for plan in plans
]
)
def get_recommendations(self, plan_ids: List[int]) -> pd.DataFrame:
if not plan_ids:
logger.info("No plan ids provided")
return pd.DataFrame()
stmt = (
select(Recommendation, PlanModel.scenario_id, PlanModel.name)
.join(
PlanRecommendations,
Recommendation.id == PlanRecommendations.recommendation_id,
)
.join(PlanModel, PlanModel.id == PlanRecommendations.plan_id)
.where(
PlanRecommendations.plan_id.in_(plan_ids),
Recommendation.default.is_(True),
Recommendation.already_installed.is_(False),
)
)
rows: Sequence[Tuple[Recommendation, Optional[int], Optional[str]]] = (
self.session.execute(stmt).tuples().all()
)
data: List[Dict[str, Any]] = [
{
**{
col.name: getattr(rec_model, col.name)
for col in Recommendation.__table__.columns.values()
},
"scenario_id": scenario_id,
"plan_name": plan_name,
}
for rec_model, scenario_id, plan_name in rows
]
return pd.DataFrame(data)
def attach_materials(self, recommendations_df: pd.DataFrame) -> pd.DataFrame:
if recommendations_df.empty:
recommendations_df["materials"] = []
return recommendations_df
rec_ids: List[int] = recommendations_df["id"].astype(int).tolist()
stmt = (
select(RecommendationMaterials, Material)
.join(Material, RecommendationMaterials.material_id == Material.id)
.where(RecommendationMaterials.recommendation_id.in_(rec_ids))
)
rows: Sequence[Tuple[RecommendationMaterials, Material]] = (
self.session.execute(stmt).tuples().all()
)
materials_map: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
for rec_mat, material in rows:
materials_map[rec_mat.recommendation_id].append(
{
"material_id": rec_mat.material_id,
"depth": rec_mat.depth,
"quantity": rec_mat.quantity,
"quantity_unit": rec_mat.quantity_unit,
"estimated_cost": rec_mat.estimated_cost,
"type": material.type.value if material.type else None,
"includes_battery": material.includes_battery,
}
)
recommendations_df["materials"] = recommendations_df["id"].astype(int).apply(
lambda x: materials_map.get(x, [])
)
return recommendations_df

40
backend/export/property_scenarios/input_schema.py Normal file
View file

@ -0,0 +1,40 @@
from typing import Optional, Union, List
from pydantic import BaseModel, model_validator, PrivateAttr
class ExportRequest(BaseModel):
# uuid which maps to a specific export request, used for tracking and logging
task_id: Union[str, None]
# uuid which maps to a specific export operation, used for tracking and logging. subtask is the child of the
# task, where the work has been distributed across workers
subtask_id: Union[str, None]
# associated portfolio id for the export request
portfolio_id: int
# list of scenario ids to export
scenario_ids: List[int]
# boolean which will overwrite the scenario ids. If this is true, we will only export the default plan for each
# property and will ignore the scenario ids
default_plans_only: Optional[bool] = False
# Private attribute to indicate whether scenario_ids should be ignored due to default_plans_only being True
_scenario_ids_ignored: bool = PrivateAttr(default=False)
@model_validator(mode="after")
def validate_default_plan_override(self):
"""
If default_plans_only is True and scenario_ids were provided,
we allow execution but make it explicit that scenario_ids
will be ignored.
"""
if self.default_plans_only and self.scenario_ids:
# We do NOT raise — we allow execution.
# We just mark the object so the handler can log/return a warning.
object.__setattr__(self, "_scenario_ids_ignored", True)
else:
object.__setattr__(self, "_scenario_ids_ignored", False)
return self
@property
def scenario_ids_ignored(self) -> bool:
return self._scenario_ids_ignored

179
backend/export/property_scenarios/main.py Normal file
View file

@ -0,0 +1,179 @@
import json
from typing import Optional, Any, Mapping, Dict, Union, List
import pandas as pd
from sqlalchemy.orm import Session
from backend.export.property_scenarios.input_schema import ExportRequest
from backend.export.property_scenarios.db_functions import DbMethods
from backend.app.db.connection import db_read_session
from backend.app.utils import sap_to_epc
from utils.logger import setup_logger
logger = setup_logger()
def choose_group_keys(payload: ExportRequest) -> List[Union[int, str]]:
if payload.default_plans_only:
return ["default_plans"] # Single export, no scenario grouping
return payload.scenario_ids
def has_solar_with_battery(materials_list: Optional[List[Dict[str, Any]]]) -> bool:
"""
Simple check to determine if any material in the list is a solar PV measure that includes a battery.
:param materials_list:
:return:
"""
for m in materials_list or []:
if (
m.get("type") == "solar_pv"
and m.get("includes_battery") is True
):
return True
return False
def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]:
export_files: Dict[Union[str, int], pd.DataFrame] = {}
db_methods = DbMethods(session)
properties_df = db_methods.get_properties(payload.portfolio_id)
logger.info("Retrieved %s properties for export", len(properties_df))
plans_df: pd.DataFrame = db_methods.get_latest_plans(
portfolio_id=payload.portfolio_id,
scenario_ids=payload.scenario_ids,
default_only=bool(payload.default_plans_only),
)
logger.info("Retrieved %s plans for export", len(plans_df))
if plans_df.empty:
logger.info("Empty plans dataframe - no plans to export. Returning empty export.")
return export_files
plan_ids: List[int] = plans_df["id"].tolist()
recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids)
logger.info("Retrieved %s recommendations for export", len(recommendations_df))
recommendations_df = db_methods.attach_materials(recommendations_df)
recommendations_df["has_solar_with_battery"] = (
recommendations_df["materials"].apply(has_solar_with_battery)
)
_filter = (
(recommendations_df["measure_type"] == "solar_pv")
& (recommendations_df["has_solar_with_battery"])
)
recommendations_df.loc[_filter, "measure_type"] = (
recommendations_df.loc[_filter, "measure_type"] + "_with_battery"
)
group_keys: List[Union[str, int]] = choose_group_keys(payload)
for group_key in group_keys:
if payload.default_plans_only:
scenario_recs = recommendations_df
else:
scenario_recs = recommendations_df[
recommendations_df["scenario_id"] == group_key
]
if scenario_recs.empty:
logger.info("No recommendations found for group_key %s - skipping export for this group", group_key)
continue
measures_df: pd.DataFrame = scenario_recs[
["property_id", "measure_type", "plan_name", "estimated_cost"]
].drop_duplicates()
pivot: pd.DataFrame = measures_df.pivot(
index=["property_id", "plan_name"],
columns="measure_type",
values="estimated_cost",
).reset_index()
pivot["total_retrofit_cost"] = (
pivot.drop(columns=["property_id", "plan_name"]).sum(axis=1)
)
post_sap: pd.DataFrame = (
scenario_recs.groupby("property_id")[["sap_points"]]
.sum()
.reset_index()
)
df: pd.DataFrame = (
properties_df.rename(columns={"solar_pv": "existing_solar_pv"})
.merge(pivot, how="left", on="property_id")
.merge(post_sap, how="left", on="property_id")
)
df["sap_points"] = df["sap_points"].fillna(0)
df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc)
export_files[group_key] = df
return export_files
# ============================================================
# Lambda Handler
# ============================================================
def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Union[int, str]]:
"""
Example event:
body_dict = {
"task_id": "test",
"subtask_id": "test",
"portfolio_id": 569,
"scenario_ids": [],
"default_plans_only": True,
}
:param event: Lambda event containing export request details
:param context: Lambda context (not used in this handler but included for completeness)
:return: HTTP response indicating success or failure of the export operation
"""
for record in event.get("Records", []):
try:
body_dict = json.loads(record["body"])
logger.debug("Validating request body")
payload = ExportRequest.model_validate(body_dict)
if payload.scenario_ids_ignored:
logger.warning(
"Received scenario_ids in request body but they will be ignored "
"because default_plans_only is set to True"
)
logger.debug("Successfully validated request body")
with db_read_session() as session:
exported_files = process_export(payload, session)
# TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url
_ = exported_files
return {
"statusCode": 200,
"body": json.dumps({}),
}
except Exception as e:
logger.error(f"Failed to process record: {e}")
return {
"statusCode": 500,
"body": json.dumps({"message": "Failed to process export request"}),
}
return {
"statusCode": 201,
"body": json.dumps({"message": "No records to process"}),
}

55
backend/export/tests/conftest.py Normal file
View file

@ -0,0 +1,55 @@
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from backend.app.db.base import Base
@pytest.fixture(scope="function")
def engine(postgresql):
"""
Create a SQLAlchemy engine bound to the ephemeral
pytest-postgresql database.
"""
# Build SQLAlchemy URL from psycopg connection info
connection_string = (
f"postgresql+psycopg://"
f"{postgresql.info.user}:"
f"{postgresql.info.password}@"
f"{postgresql.info.host}:"
f"{postgresql.info.port}/"
f"{postgresql.info.dbname}"
)
engine = create_engine(connection_string)
# Create tables once per test session
Base.metadata.create_all(engine)
# Yeild will split this function into two phase. 1) setup and 2) teardown, the latter of which will run after all
# tests have completed
yield engine
# Clean-up after entire test session
Base.metadata.drop_all(engine)
engine.dispose()
@pytest.fixture(scope="function")
def db_session(engine):
"""
Provides a clean transactional session per test.
Rolls back after each test to keep isolation.
"""
connection = engine.connect()
transaction = connection.begin()
session = sessionmaker(bind=connection)()
yield session
session.close()
transaction.rollback()
connection.close()

14
backend/export/tests/fixtures/plan_recs_569.csv vendored Normal file
View file

@ -0,0 +1,14 @@
id,plan_id,recommendation_id
24799722,1604277,24798968
24799726,1604277,24798972
24801150,1604367,24800396
24802703,1604448,24801949
24802724,1604448,24801970
24805327,1604577,24804573
24805397,1604579,24804643
24805401,1604579,24804647
24813000,1605111,24812246
24813002,1605111,24812248
24813004,1605111,24812250
24813006,1605112,24812252
24813009,1605112,24812255
1 id plan_id recommendation_id
2 24799722 1604277 24798968
3 24799726 1604277 24798972
4 24801150 1604367 24800396
5 24802703 1604448 24801949
6 24802724 1604448 24801970
7 24805327 1604577 24804573
8 24805397 1604579 24804643
9 24805401 1604579 24804647
10 24813000 1605111 24812246
11 24813002 1605111 24812248
12 24813004 1605111 24812250
13 24813006 1605112 24812252
14 24813009 1605112 24812255

11
backend/export/tests/fixtures/plans_569.csv vendored Normal file
View file

@ -0,0 +1,11 @@
id,name,portfolio_id,property_id,scenario_id,created_at,is_default,valuation_increase_lower_bound,valuation_increase_upper_bound,valuation_increase_average,plan_type,post_sap_points,post_epc_rating,post_co2_emissions,co2_savings,post_energy_bill,energy_bill_savings,post_energy_consumption,energy_consumption_savings,valuation_post_retrofit,valuation_increase,cost_of_works,contingency_cost
1604277,,569,660478,1060,2026-02-19 16:14:45.560816,True,0.0302,0.07,0.048226666,,71.5,Epc.C,4.1813498,0.71865046,1447.5204,691.6662,15303.688,3276.7622,,,6984.568,1003.9568
1604448,,569,660529,1060,2026-02-19 16:14:52.052740,True,0.0302,0.07,0.048226666,,70.0,Epc.C,7.32816,1.5818402,2978.734,2314.7651,16558.295,1837.0155,,,13528.6,2844.636
1604367,,569,660538,1060,2026-02-19 16:14:48.517937,True,0.02,0.03,0.025,,71.0,Epc.C,5.003036,0.43696404,1933.2236,521.5316,19190.531,1883.4657,,,5520.0,828.0
1604577,,569,660688,1060,2026-02-19 16:15:04.461456,True,0.02,0.03,0.025,,70.0,Epc.C,3.6019807,0.20801921,1610.3181,248.27809,13746.731,896.6345,,,5100.0,765.0
1604579,,569,660690,1060,2026-02-19 16:15:04.461456,True,0.02,0.03,0.025,,70.0,Epc.C,4.7473392,0.5326607,1867.537,699.7881,18730.615,2527.2231,,,5469.0,825.74
1605110,,569,660598,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,70.0,Epc.C,1.89,0.0,1125.7338,0.0,7268.866,0.0,,,0.0,0.0
1605111,,569,660599,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,68.7,Epc.D,2.02,1.1,1174.9326,319.18213,7748.233,3924.9,,,1218.584,124.0984
1605080,,569,660448,1069,2026-02-19 16:18:57.581528,True,0.0,0.0,0.0,,71.0,Epc.C,1.79,0.0,1101.9677,0.0,6821.7285,0.0,,,0.0,0.0
1605112,,569,660600,1069,2026-02-19 16:18:57.606337,True,0.0,0.0,0.0,,64.9,Epc.D,1.89,0.8,1131.3535,172.0886,7241.062,2466.7,,,3885.834,716.7084
1605404,,569,660652,1069,2026-02-19 16:19:28.383096,True,0.0,0.0,0.0,,71.0,Epc.C,3.18,0.0,1757.515,0.0,11929.814,0.0,,,0.0,0.0
1 id name portfolio_id property_id scenario_id created_at is_default valuation_increase_lower_bound valuation_increase_upper_bound valuation_increase_average plan_type post_sap_points post_epc_rating post_co2_emissions co2_savings post_energy_bill energy_bill_savings post_energy_consumption energy_consumption_savings valuation_post_retrofit valuation_increase cost_of_works contingency_cost
2 1604277 569 660478 1060 2026-02-19 16:14:45.560816 True 0.0302 0.07 0.048226666 71.5 Epc.C 4.1813498 0.71865046 1447.5204 691.6662 15303.688 3276.7622 6984.568 1003.9568
3 1604448 569 660529 1060 2026-02-19 16:14:52.052740 True 0.0302 0.07 0.048226666 70.0 Epc.C 7.32816 1.5818402 2978.734 2314.7651 16558.295 1837.0155 13528.6 2844.636
4 1604367 569 660538 1060 2026-02-19 16:14:48.517937 True 0.02 0.03 0.025 71.0 Epc.C 5.003036 0.43696404 1933.2236 521.5316 19190.531 1883.4657 5520.0 828.0
5 1604577 569 660688 1060 2026-02-19 16:15:04.461456 True 0.02 0.03 0.025 70.0 Epc.C 3.6019807 0.20801921 1610.3181 248.27809 13746.731 896.6345 5100.0 765.0
6 1604579 569 660690 1060 2026-02-19 16:15:04.461456 True 0.02 0.03 0.025 70.0 Epc.C 4.7473392 0.5326607 1867.537 699.7881 18730.615 2527.2231 5469.0 825.74
7 1605110 569 660598 1069 2026-02-19 16:18:57.606337 True 0.0 0.0 0.0 70.0 Epc.C 1.89 0.0 1125.7338 0.0 7268.866 0.0 0.0 0.0
8 1605111 569 660599 1069 2026-02-19 16:18:57.606337 True 0.0 0.0 0.0 68.7 Epc.D 2.02 1.1 1174.9326 319.18213 7748.233 3924.9 1218.584 124.0984
9 1605080 569 660448 1069 2026-02-19 16:18:57.581528 True 0.0 0.0 0.0 71.0 Epc.C 1.79 0.0 1101.9677 0.0 6821.7285 0.0 0.0 0.0
10 1605112 569 660600 1069 2026-02-19 16:18:57.606337 True 0.0 0.0 0.0 64.9 Epc.D 1.89 0.8 1131.3535 172.0886 7241.062 2466.7 3885.834 716.7084
11 1605404 569 660652 1069 2026-02-19 16:19:28.383096 True 0.0 0.0 0.0 71.0 Epc.C 3.18 0.0 1757.515 0.0 11929.814 0.0 0.0 0.0

2
backend/export/tests/fixtures/portfolio_569.csv vendored Normal file
View file

@ -0,0 +1,2 @@
id,name,budget,status,goal,cost,number_of_properties,co2_equivalent_savings,energy_savings,energy_cost_savings,property_valuation_increase,rental_yield_increase,total_work_hours,labour_days,created_at,updated_at,epc_breakdown_pre_retrofit,epc_breakdown_post_retrofit,n_units_to_retrofit,co2_per_unit_pre_retrofit,co2_per_unit_post_retrofit,energy_bill_per_unit_pre_retrofit,energy_bill_per_unit_post_retrofit,energy_consumption_per_unit_pre_retrofit,energy_consumption_per_unit_post_retrofit,valuation_improvement_per_unit,cost_per_unit,cost_per_co2_saved,cost_per_sap_point,valuation_return_on_investment
569,Lifespace Rentals - Sample Retrofit Plans,,PortfolioStatus.SCOPING,PortfolioGoal.NONE,,,,,,,,,,2026-02-12 21:23:37.862000+00:00,2026-02-12 21:23:37.862000+00:00,,,,,,,,,,,,,,
1 id name budget status goal cost number_of_properties co2_equivalent_savings energy_savings energy_cost_savings property_valuation_increase rental_yield_increase total_work_hours labour_days created_at updated_at epc_breakdown_pre_retrofit epc_breakdown_post_retrofit n_units_to_retrofit co2_per_unit_pre_retrofit co2_per_unit_post_retrofit energy_bill_per_unit_pre_retrofit energy_bill_per_unit_post_retrofit energy_consumption_per_unit_pre_retrofit energy_consumption_per_unit_post_retrofit valuation_improvement_per_unit cost_per_unit cost_per_co2_saved cost_per_sap_point valuation_return_on_investment
2 569 Lifespace Rentals - Sample Retrofit Plans PortfolioStatus.SCOPING PortfolioGoal.NONE 2026-02-12 21:23:37.862000+00:00 2026-02-12 21:23:37.862000+00:00

11
backend/export/tests/fixtures/properties_569.csv vendored Normal file
View file

@ -0,0 +1,11 @@
,id,portfolio_id,creation_status,uprn,landlord_property_id,building_reference_number,status,address,postcode,has_pre_condition_report,has_recommendations,created_at,updated_at,property_type,built_form,local_authority,constituency,number_of_rooms,year_built,tenure,current_epc_rating,current_sap_points,current_valuation,installed_measures_sap_point_adjustment,is_sap_points_adjusted_for_installed_measures,original_sap_points
0,660478,569,PropertyCreationStatus.READY,100090438731.0,BARR052,3460742868.0,PortfolioStatus.ASSESSMENT,"52, Barrack Street",CO1 2LR,True,True,2026-02-12 21:59:02.744427,2026-02-19 16:18:57.941443,House,End-Terrace,Colchester,Colchester,4.0,1900.0,rental (private),Epc.E,53.0,0.0,0.0,False,53.0
1,660448,569,PropertyCreationStatus.READY,100090678548.0,BOUR110A,10002385993.0,PortfolioStatus.ASSESSMENT,Upper 110a Bournemouth Park Road,SS2 5LS,True,True,2026-02-12 21:59:02.388473,2026-02-19 16:18:57.578330,Flat,Detached,Southend-on-Sea,Rochford and Southend East,2.0,1950.0,Rented (private),Epc.C,71.0,0.0,0.0,False,71.0
2,660538,569,PropertyCreationStatus.READY,10033423541.0,CHUR099,8188570968.0,PortfolioStatus.ASSESSMENT,"99, Church Road",RM3 0SH,True,True,2026-02-12 21:59:03.203854,2026-02-19 16:19:03.748571,House,Mid-Terrace,Havering,Hornchurch and Upminster,5.0,1900.0,rental (private),Epc.D,58.0,0.0,0.0,False,58.0
3,660529,569,PropertyCreationStatus.READY,100091596678.0,CHER003,8961772668.0,PortfolioStatus.ASSESSMENT,"3, Brickfield Cottages",SS4 1PP,True,True,2026-02-12 21:59:02.935502,2026-02-19 16:18:55.971569,House,Mid-Terrace,Rochford,Rochford and Southend East,4.0,1900.0,rental (private),Epc.E,41.0,0.0,0.0,False,41.0
4,660598,569,PropertyCreationStatus.READY,100090663644.0,FLEM049B,10006705876.0,PortfolioStatus.ASSESSMENT,49b Flemming Crescent,SS9 4HR,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,Semi-Detached,Southend-on-Sea,,2.0,1930.0,Rented (social),Epc.C,70.0,0.0,0.0,False,70.0
5,660599,569,PropertyCreationStatus.READY,10012149765.0,FORE003A,9740118668.0,PortfolioStatus.ASSESSMENT,"3a, Forest Avenue",SS1 2HU,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,End-Terrace,Southend-on-Sea,Rochford and Southend East,2.0,1930.0,rental (private),Epc.D,56.0,0.0,0.0,False,56.0
6,660600,569,PropertyCreationStatus.READY,10012149797.0,FORE003GFF,1436818568.0,PortfolioStatus.ASSESSMENT,"3, Forest Avenue",SS1 2HU,True,True,2026-02-12 21:59:04.732965,2026-02-19 16:18:57.601893,Flat,End-Terrace,Southend-on-Sea,Rochford and Southend East,2.0,1900.0,rental (private),Epc.D,59.0,0.0,0.0,False,59.0
7,660652,569,PropertyCreationStatus.READY,100022668838.0,MANT061,10000429573.0,PortfolioStatus.ASSESSMENT,61 MANTILLA ROAD,SW17 8DY,True,True,2026-02-12 21:59:04.711717,2026-02-19 16:19:28.379512,Flat,Mid-Terrace,Wandsworth,Tooting,4.0,1900.0,Owner-occupied,Epc.C,71.0,0.0,0.0,False,71.0
8,660690,569,PropertyCreationStatus.READY,100021987220.0,MERR008,9050743578.0,PortfolioStatus.ASSESSMENT,"8, Merritt Road",SE4 1DY,True,True,2026-02-12 21:59:09.459245,2026-02-19 16:19:32.826638,House,Mid-Terrace,Lewisham,"Lewisham, Deptford",6.0,1900.0,owner-occupied,Epc.D,58.0,0.0,0.0,False,58.0
9,660688,569,PropertyCreationStatus.READY,207158120.0,MEDC048,208210678.0,PortfolioStatus.ASSESSMENT,"48, Medcalf Road",EN3 6HL,True,True,2026-02-12 21:59:09.459245,2026-02-19 16:19:32.826638,House,Mid-Terrace,Enfield,Enfield North,4.0,1900.0,rental (private),Epc.D,61.0,0.0,0.0,False,61.0
1 id portfolio_id creation_status uprn landlord_property_id building_reference_number status address postcode has_pre_condition_report has_recommendations created_at updated_at property_type built_form local_authority constituency number_of_rooms year_built tenure current_epc_rating current_sap_points current_valuation installed_measures_sap_point_adjustment is_sap_points_adjusted_for_installed_measures original_sap_points
2 0 660478 569 PropertyCreationStatus.READY 100090438731.0 BARR052 3460742868.0 PortfolioStatus.ASSESSMENT 52, Barrack Street CO1 2LR True True 2026-02-12 21:59:02.744427 2026-02-19 16:18:57.941443 House End-Terrace Colchester Colchester 4.0 1900.0 rental (private) Epc.E 53.0 0.0 0.0 False 53.0
3 1 660448 569 PropertyCreationStatus.READY 100090678548.0 BOUR110A 10002385993.0 PortfolioStatus.ASSESSMENT Upper 110a Bournemouth Park Road SS2 5LS True True 2026-02-12 21:59:02.388473 2026-02-19 16:18:57.578330 Flat Detached Southend-on-Sea Rochford and Southend East 2.0 1950.0 Rented (private) Epc.C 71.0 0.0 0.0 False 71.0
4 2 660538 569 PropertyCreationStatus.READY 10033423541.0 CHUR099 8188570968.0 PortfolioStatus.ASSESSMENT 99, Church Road RM3 0SH True True 2026-02-12 21:59:03.203854 2026-02-19 16:19:03.748571 House Mid-Terrace Havering Hornchurch and Upminster 5.0 1900.0 rental (private) Epc.D 58.0 0.0 0.0 False 58.0
5 3 660529 569 PropertyCreationStatus.READY 100091596678.0 CHER003 8961772668.0 PortfolioStatus.ASSESSMENT 3, Brickfield Cottages SS4 1PP True True 2026-02-12 21:59:02.935502 2026-02-19 16:18:55.971569 House Mid-Terrace Rochford Rochford and Southend East 4.0 1900.0 rental (private) Epc.E 41.0 0.0 0.0 False 41.0
6 4 660598 569 PropertyCreationStatus.READY 100090663644.0 FLEM049B 10006705876.0 PortfolioStatus.ASSESSMENT 49b Flemming Crescent SS9 4HR True True 2026-02-12 21:59:04.732965 2026-02-19 16:18:57.601893 Flat Semi-Detached Southend-on-Sea 2.0 1930.0 Rented (social) Epc.C 70.0 0.0 0.0 False 70.0
7 5 660599 569 PropertyCreationStatus.READY 10012149765.0 FORE003A 9740118668.0 PortfolioStatus.ASSESSMENT 3a, Forest Avenue SS1 2HU True True 2026-02-12 21:59:04.732965 2026-02-19 16:18:57.601893 Flat End-Terrace Southend-on-Sea Rochford and Southend East 2.0 1930.0 rental (private) Epc.D 56.0 0.0 0.0 False 56.0
8 6 660600 569 PropertyCreationStatus.READY 10012149797.0 FORE003GFF 1436818568.0 PortfolioStatus.ASSESSMENT 3, Forest Avenue SS1 2HU True True 2026-02-12 21:59:04.732965 2026-02-19 16:18:57.601893 Flat End-Terrace Southend-on-Sea Rochford and Southend East 2.0 1900.0 rental (private) Epc.D 59.0 0.0 0.0 False 59.0
9 7 660652 569 PropertyCreationStatus.READY 100022668838.0 MANT061 10000429573.0 PortfolioStatus.ASSESSMENT 61 MANTILLA ROAD SW17 8DY True True 2026-02-12 21:59:04.711717 2026-02-19 16:19:28.379512 Flat Mid-Terrace Wandsworth Tooting 4.0 1900.0 Owner-occupied Epc.C 71.0 0.0 0.0 False 71.0
10 8 660690 569 PropertyCreationStatus.READY 100021987220.0 MERR008 9050743578.0 PortfolioStatus.ASSESSMENT 8, Merritt Road SE4 1DY True True 2026-02-12 21:59:09.459245 2026-02-19 16:19:32.826638 House Mid-Terrace Lewisham Lewisham, Deptford 6.0 1900.0 owner-occupied Epc.D 58.0 0.0 0.0 False 58.0
11 9 660688 569 PropertyCreationStatus.READY 207158120.0 MEDC048 208210678.0 PortfolioStatus.ASSESSMENT 48, Medcalf Road EN3 6HL True True 2026-02-12 21:59:09.459245 2026-02-19 16:19:32.826638 House Mid-Terrace Enfield Enfield North 4.0 1900.0 rental (private) Epc.D 61.0 0.0 0.0 False 61.0

11
backend/export/tests/fixtures/property_details_epc_569.csv vendored Normal file
View file

@ -0,0 +1,11 @@
,id,property_id,portfolio_id,full_address,lodgement_date,is_expired,total_floor_area,walls,walls_rating,roof,roof_rating,floor,floor_rating,windows,windows_rating,heating,heating_rating,heating_controls,heating_controls_rating,hot_water,hot_water_rating,lighting,lighting_rating,mainfuel,ventilation,solar_pv,solar_hot_water,wind_turbine,floor_height,number_heated_rooms,heat_loss_corridor,unheated_corridor_length,number_of_open_fireplaces,number_of_extensions,number_of_storeys,mains_gas,energy_tariff,primary_energy_consumption,co2_emissions,current_energy_demand,current_energy_demand_heating_hotwater,estimated,sap_05_overwritten,sap_05_score,sap_05_epc_rating,heating_cost_current,hot_water_cost_current,lighting_cost_current,appliances_cost_current,gas_standing_charge,electricity_standing_charge,original_co2_emissions,original_primary_energy_consumption,original_current_energy_demand,original_current_energy_demand_heating_hotwater,installed_measures_co2_adjustment,installed_measures_energy_demand_adjustment,installed_measures_total_energy_bill_adjustment,installed_measures_heat_demand_adjustment,is_epc_adjusted_for_installed_measures
44,1534934,660688,569,"48, Medcalf Road",2018-09-05,False,68.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Solid, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.55,,False,,0,0,,True,Single,278.0,3.81,14643.366,12185.6,False,False,,,711.0628,139.06198,70.770935,609.7844,128.0785,199.8375,3.81,278.0,14643.366,12185.6,0.0,0.0,0.0,0.0,False
53,1534816,660600,569,"3, Forest Avenue",2020-02-27,False,35.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Suspended, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 83% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.64,,False,,0,0,,True,Single,389.0,2.69,9707.762,8267.8,False,False,,,466.75378,110.046844,53.1057,345.6198,128.0785,199.8375,2.69,389.0,9707.762,8267.8,0.0,0.0,0.0,0.0,False
292,1534754,660478,569,"52, Barrack Street",2019-09-11,False,67.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Solid, no insulation",,Partial double glazing,2,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,Low energy lighting in 78% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.36,,False,,0,1,,True,Single,374.0,4.9,18580.451,16094.1,False,False,,,980.4243,142.37581,86.25319,602.2173,128.0785,199.8375,4.9,374.0,18580.451,16094.1,0.0,0.0,0.0,0.0,False
295,1534868,660652,569,"61 MANTILLA ROAD, LONDON",2020-12-10,False,79.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Solid, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.63,,False,,0,0,,True,off-peak 7 hour,184.0,3.18,11929.814,9046.1,False,False,,,487.25763,143.84087,110.2875,688.2131,128.0785,199.8375,3.18,184.0,11929.814,9046.1,0.0,0.0,0.0,0.0,False
310,1534964,660448,569,Upper 110a Bournemouth Park Road,2022-02-22,False,35.0,"Solid brick, as built, no insulation",1,"Pitched, 100 mm loft insulation",3.0,(another dwelling below),,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 80% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.41,,False,,0,0,,True,Unknown,238.0,1.79,6821.7285,5382.4,False,False,,,272.55676,102.9448,52.930252,345.6198,128.0785,199.8375,1.79,238.0,6821.7285,5382.4,0.0,0.0,0.0,0.0,False
344,1534936,660690,569,"8, Merritt Road",2017-08-15,False,101.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Suspended, no insulation",,Fully double glazed,3,"Boiler and radiators, mains gas",4,"Programmer, room thermostat and trvs",4,From main system,4,No low energy lighting,1,Mains gas not community,natural,0.0,False,0.0,2.6,,False,,0,1,,True,Unknown,260.0,5.28,21257.838,17606.3,False,False,,,1074.1602,154.13814,194.25749,816.8532,128.0785,199.8375,5.28,260.0,21257.838,17606.3,0.0,0.0,0.0,0.0,False
460,1535385,660529,569,"3, Brickfield Cottages, Cherry Orchard Lane",2020-04-09,False,85.0,"Solid brick, as built, no insulation",2,"Pitched, 200 mm loft insulation",4.0,"Suspended, no insulation",,Fully double glazed,3,Electric storage heaters,3,Manual charge control,2,"Electric immersion, off-peak",3,Low energy lighting in 58% of fixed outlets,4,Electricity not community,natural,0.0,False,0.0,2.45,,False,,0,1,,True,dual,577.0,8.91,18395.31,15230.1,False,False,,,3550.6333,666.58136,149.46556,726.9812,0.0,199.8375,8.91,577.0,18395.31,15230.1,0.0,0.0,0.0,0.0,False
485,1534784,660538,569,"99, Church Road, Harold Wood",2019-09-03,False,92.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,"Suspended, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 80% of fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.52,,False,,0,1,,True,Single,297.0,5.44,21073.996,17904.0,False,False,,,1092.4246,156.6427,109.16419,768.6077,128.0785,199.8375,5.44,297.0,21073.996,17904.0,0.0,0.0,0.0,0.0,False
494,1534814,660598,569,49b Flemming Crescent,2024-10-03,False,35.0,"Solid brick, as built, no insulation",1,(another dwelling above),,"Suspended, no insulation",,Fully double glazed,4,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in all fixed outlets,5,Mains gas not community,natural,0.0,False,0.0,2.42,,False,,0,0,,True,Single,261.0,1.89,7268.866,5865.4,False,False,,,304.39737,104.800545,43.0,345.6198,128.0785,199.8375,1.89,261.0,7268.866,5865.4,0.0,0.0,0.0,0.0,False
741,1534815,660599,569,"3a, Forest Avenue",2020-06-05,False,40.0,"Solid brick, as built, no insulation",1,"Pitched, no insulation",1.0,(another dwelling below),,Fully double glazed,3,"Boiler and radiators, mains gas",4,Programmer and room thermostat,3,From main system,4,Low energy lighting in 38% of fixed outlets,3,Mains gas not community,natural,0.0,False,0.0,2.58,,False,,0,0,,True,Unknown,396.0,3.12,11673.133,9974.6,False,False,,,587.73975,108.13529,85.62337,384.70035,128.0785,199.8375,3.12,396.0,11673.133,9974.6,0.0,0.0,0.0,0.0,False
1 id property_id portfolio_id full_address lodgement_date is_expired total_floor_area walls walls_rating roof roof_rating floor floor_rating windows windows_rating heating heating_rating heating_controls heating_controls_rating hot_water hot_water_rating lighting lighting_rating mainfuel ventilation solar_pv solar_hot_water wind_turbine floor_height number_heated_rooms heat_loss_corridor unheated_corridor_length number_of_open_fireplaces number_of_extensions number_of_storeys mains_gas energy_tariff primary_energy_consumption co2_emissions current_energy_demand current_energy_demand_heating_hotwater estimated sap_05_overwritten sap_05_score sap_05_epc_rating heating_cost_current hot_water_cost_current lighting_cost_current appliances_cost_current gas_standing_charge electricity_standing_charge original_co2_emissions original_primary_energy_consumption original_current_energy_demand original_current_energy_demand_heating_hotwater installed_measures_co2_adjustment installed_measures_energy_demand_adjustment installed_measures_total_energy_bill_adjustment installed_measures_heat_demand_adjustment is_epc_adjusted_for_installed_measures
2 44 1534934 660688 569 48, Medcalf Road 2018-09-05 False 68.0 Solid brick, as built, no insulation 1 Pitched, no insulation 1.0 Solid, no insulation Fully double glazed 4 Boiler and radiators, mains gas 4 Programmer, room thermostat and trvs 4 From main system 4 Low energy lighting in all fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.55 False 0 0 True Single 278.0 3.81 14643.366 12185.6 False False 711.0628 139.06198 70.770935 609.7844 128.0785 199.8375 3.81 278.0 14643.366 12185.6 0.0 0.0 0.0 0.0 False
3 53 1534816 660600 569 3, Forest Avenue 2020-02-27 False 35.0 Solid brick, as built, no insulation 1 (another dwelling above) Suspended, no insulation Fully double glazed 3 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in 83% of fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.64 False 0 0 True Single 389.0 2.69 9707.762 8267.8 False False 466.75378 110.046844 53.1057 345.6198 128.0785 199.8375 2.69 389.0 9707.762 8267.8 0.0 0.0 0.0 0.0 False
4 292 1534754 660478 569 52, Barrack Street 2019-09-11 False 67.0 Solid brick, as built, no insulation 1 Pitched, no insulation 1.0 Solid, no insulation Partial double glazing 2 Boiler and radiators, mains gas 4 Programmer, room thermostat and trvs 4 From main system 4 Low energy lighting in 78% of fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.36 False 0 1 True Single 374.0 4.9 18580.451 16094.1 False False 980.4243 142.37581 86.25319 602.2173 128.0785 199.8375 4.9 374.0 18580.451 16094.1 0.0 0.0 0.0 0.0 False
5 295 1534868 660652 569 61 MANTILLA ROAD, LONDON 2020-12-10 False 79.0 Solid brick, as built, no insulation 1 (another dwelling above) Solid, no insulation Fully double glazed 3 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in all fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.63 False 0 0 True off-peak 7 hour 184.0 3.18 11929.814 9046.1 False False 487.25763 143.84087 110.2875 688.2131 128.0785 199.8375 3.18 184.0 11929.814 9046.1 0.0 0.0 0.0 0.0 False
6 310 1534964 660448 569 Upper 110a Bournemouth Park Road 2022-02-22 False 35.0 Solid brick, as built, no insulation 1 Pitched, 100 mm loft insulation 3.0 (another dwelling below) Fully double glazed 3 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in 80% of fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.41 False 0 0 True Unknown 238.0 1.79 6821.7285 5382.4 False False 272.55676 102.9448 52.930252 345.6198 128.0785 199.8375 1.79 238.0 6821.7285 5382.4 0.0 0.0 0.0 0.0 False
7 344 1534936 660690 569 8, Merritt Road 2017-08-15 False 101.0 Solid brick, as built, no insulation 1 Pitched, no insulation 1.0 Suspended, no insulation Fully double glazed 3 Boiler and radiators, mains gas 4 Programmer, room thermostat and trvs 4 From main system 4 No low energy lighting 1 Mains gas not community natural 0.0 False 0.0 2.6 False 0 1 True Unknown 260.0 5.28 21257.838 17606.3 False False 1074.1602 154.13814 194.25749 816.8532 128.0785 199.8375 5.28 260.0 21257.838 17606.3 0.0 0.0 0.0 0.0 False
8 460 1535385 660529 569 3, Brickfield Cottages, Cherry Orchard Lane 2020-04-09 False 85.0 Solid brick, as built, no insulation 2 Pitched, 200 mm loft insulation 4.0 Suspended, no insulation Fully double glazed 3 Electric storage heaters 3 Manual charge control 2 Electric immersion, off-peak 3 Low energy lighting in 58% of fixed outlets 4 Electricity not community natural 0.0 False 0.0 2.45 False 0 1 True dual 577.0 8.91 18395.31 15230.1 False False 3550.6333 666.58136 149.46556 726.9812 0.0 199.8375 8.91 577.0 18395.31 15230.1 0.0 0.0 0.0 0.0 False
9 485 1534784 660538 569 99, Church Road, Harold Wood 2019-09-03 False 92.0 Solid brick, as built, no insulation 1 Pitched, no insulation 1.0 Suspended, no insulation Fully double glazed 4 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in 80% of fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.52 False 0 1 True Single 297.0 5.44 21073.996 17904.0 False False 1092.4246 156.6427 109.16419 768.6077 128.0785 199.8375 5.44 297.0 21073.996 17904.0 0.0 0.0 0.0 0.0 False
10 494 1534814 660598 569 49b Flemming Crescent 2024-10-03 False 35.0 Solid brick, as built, no insulation 1 (another dwelling above) Suspended, no insulation Fully double glazed 4 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in all fixed outlets 5 Mains gas not community natural 0.0 False 0.0 2.42 False 0 0 True Single 261.0 1.89 7268.866 5865.4 False False 304.39737 104.800545 43.0 345.6198 128.0785 199.8375 1.89 261.0 7268.866 5865.4 0.0 0.0 0.0 0.0 False
11 741 1534815 660599 569 3a, Forest Avenue 2020-06-05 False 40.0 Solid brick, as built, no insulation 1 Pitched, no insulation 1.0 (another dwelling below) Fully double glazed 3 Boiler and radiators, mains gas 4 Programmer and room thermostat 3 From main system 4 Low energy lighting in 38% of fixed outlets 3 Mains gas not community natural 0.0 False 0.0 2.58 False 0 0 True Unknown 396.0 3.12 11673.133 9974.6 False False 587.73975 108.13529 85.62337 384.70035 128.0785 199.8375 3.12 396.0 11673.133 9974.6 0.0 0.0 0.0 0.0 False

14
backend/export/tests/fixtures/recommendations_569.csv vendored Normal file
View file

@ -0,0 +1,14 @@
Unnamed: 0,id,property_id,created_at,type,measure_type,description,estimated_cost,default,starting_u_value,new_u_value,sap_points,heat_demand,kwh_savings,co2_equivalent_savings,energy_savings,energy_cost_savings,property_valuation_increase,rental_yield_increase,total_work_hours,labour_days,already_installed,plan_name
49705,24798968,660478,2026-02-19 16:14:45.560816,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",874.568,True,,,1.5,14.9,1041.2,0.2,14.9,72.639015,,,4.16,1.0,False,whatever
49709,24798972,660478,2026-02-19 16:14:45.560816,solar_pv,solar_pv,"8 panel system, 400W solar panels, 5.8kw Growatt battery - 3.2 kWp system",6110.0,True,,,17.0,79.1,2235.5623,0.5186504,79.1,619.02716,,,48.0,2.0,False,whatever
51133,24800396,660538,2026-02-19 16:14:48.517937,solar_pv,solar_pv,"10 panel system, 400W solar panels - 4.0 kWp system",5520.0,True,,,13.0,58.5,1883.4657,0.43696404,58.5,521.5316,,,48.0,2.0,False,whatever
52686,24801949,660529,2026-02-19 16:14:52.052740,heating,boiler_upgrade,"Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs",8008.6,True,,,12.9,132.9,0.0,1.1556525,132.9,1806.0955,,,26.5,4.0,False,whatever
52707,24801970,660529,2026-02-19 16:14:52.052740,solar_pv,solar_pv,"10 panel system, 400W solar panels - 4.0 kWp system",5520.0,True,,,16.1,68.8,1837.0155,0.4261876,68.8,508.6696,,,48.0,2.0,False,whatever
55310,24804573,660688,2026-02-19 16:15:04.461456,solar_pv,solar_pv,"5 panel system, 400W solar panels - 2.0 kWp system",5100.0,True,,,9.0,41.4,896.6345,0.20801921,41.4,248.27809,,,48.0,2.0,False,whatever
55380,24804643,660690,2026-02-19 16:15:04.461456,low_energy_lighting,low_energy_lighting,Install low energy lighting in 14 outlets,49.0,True,,,2.0,18.2,766.5,0.124173,18.2,212.24385,,,1.0,0.125,False,whatever
55384,24804647,660690,2026-02-19 16:15:04.461456,solar_pv,solar_pv,"9 panel system, 400W solar panels - 3.6 kWp system",5420.0,True,,,10.0,43.9,1760.723,0.40848774,43.9,487.54422,,,48.0,2.0,False,whatever
62983,24812246,660599,2026-02-19 16:18:57.606337,loft_insulation,loft_insulation,Install 300mm of Knauf Loft Roll 44 glass fibre roll in your loft,600.0,True,2.3,2.3,8.4,102.8,3178.2,0.9,102.8,221.72618,,,8.0,1.0,False,whatever
62985,24812248,660599,2026-02-19 16:18:57.606337,low_energy_lighting,low_energy_lighting,Install low energy lighting in 4 outlets,14.0,True,,,1.0,14.2,219.0,0.0,14.2,60.6411,,,1.0,0.125,False,whatever
62987,24812250,660599,2026-02-19 16:18:57.606337,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",604.584,True,,,3.3,18.4,527.7,0.2,18.4,36.814835,,,3.08,1.0,False,whatever
62989,24812252,660600,2026-02-19 16:18:57.606337,suspended_floor_insulation,suspended_floor_insulation,Install 75mm Q-bot underfloor insulation insulation in suspended floor,3281.25,True,0.87,0.22,4.0,99.2,1816.6,0.6,99.2,126.734566,,,57.05,2.3770833,False,whatever
62992,24812255,660600,2026-02-19 16:18:57.606337,heating,time_temperature_zone_control,"Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control)",604.584,True,,,1.9,17.7,650.1,0.2,17.7,45.354034,,,3.08,1.0,False,whatever
1 Unnamed: 0 id property_id created_at type measure_type description estimated_cost default starting_u_value new_u_value sap_points heat_demand kwh_savings co2_equivalent_savings energy_savings energy_cost_savings property_valuation_increase rental_yield_increase total_work_hours labour_days already_installed plan_name
2 49705 24798968 660478 2026-02-19 16:14:45.560816 heating time_temperature_zone_control Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control) 874.568 True 1.5 14.9 1041.2 0.2 14.9 72.639015 4.16 1.0 False whatever
3 49709 24798972 660478 2026-02-19 16:14:45.560816 solar_pv solar_pv 8 panel system, 400W solar panels, 5.8kw Growatt battery - 3.2 kWp system 6110.0 True 17.0 79.1 2235.5623 0.5186504 79.1 619.02716 48.0 2.0 False whatever
4 51133 24800396 660538 2026-02-19 16:14:48.517937 solar_pv solar_pv 10 panel system, 400W solar panels - 4.0 kWp system 5520.0 True 13.0 58.5 1883.4657 0.43696404 58.5 521.5316 48.0 2.0 False whatever
5 52686 24801949 660529 2026-02-19 16:14:52.052740 heating boiler_upgrade Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs 8008.6 True 12.9 132.9 0.0 1.1556525 132.9 1806.0955 26.5 4.0 False whatever
6 52707 24801970 660529 2026-02-19 16:14:52.052740 solar_pv solar_pv 10 panel system, 400W solar panels - 4.0 kWp system 5520.0 True 16.1 68.8 1837.0155 0.4261876 68.8 508.6696 48.0 2.0 False whatever
7 55310 24804573 660688 2026-02-19 16:15:04.461456 solar_pv solar_pv 5 panel system, 400W solar panels - 2.0 kWp system 5100.0 True 9.0 41.4 896.6345 0.20801921 41.4 248.27809 48.0 2.0 False whatever
8 55380 24804643 660690 2026-02-19 16:15:04.461456 low_energy_lighting low_energy_lighting Install low energy lighting in 14 outlets 49.0 True 2.0 18.2 766.5 0.124173 18.2 212.24385 1.0 0.125 False whatever
9 55384 24804647 660690 2026-02-19 16:15:04.461456 solar_pv solar_pv 9 panel system, 400W solar panels - 3.6 kWp system 5420.0 True 10.0 43.9 1760.723 0.40848774 43.9 487.54422 48.0 2.0 False whatever
10 62983 24812246 660599 2026-02-19 16:18:57.606337 loft_insulation loft_insulation Install 300mm of Knauf Loft Roll 44 glass fibre roll in your loft 600.0 True 2.3 2.3 8.4 102.8 3178.2 0.9 102.8 221.72618 8.0 1.0 False whatever
11 62985 24812248 660599 2026-02-19 16:18:57.606337 low_energy_lighting low_energy_lighting Install low energy lighting in 4 outlets 14.0 True 1.0 14.2 219.0 0.0 14.2 60.6411 1.0 0.125 False whatever
12 62987 24812250 660599 2026-02-19 16:18:57.606337 heating time_temperature_zone_control Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control) 604.584 True 3.3 18.4 527.7 0.2 18.4 36.814835 3.08 1.0 False whatever
13 62989 24812252 660600 2026-02-19 16:18:57.606337 suspended_floor_insulation suspended_floor_insulation Install 75mm Q-bot underfloor insulation insulation in suspended floor 3281.25 True 0.87 0.22 4.0 99.2 1816.6 0.6 99.2 126.734566 57.05 2.3770833 False whatever
14 62992 24812255 660600 2026-02-19 16:18:57.606337 heating time_temperature_zone_control Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & temperature zone control) 604.584 True 1.9 17.7 650.1 0.2 17.7 45.354034 3.08 1.0 False whatever

540
backend/export/tests/test_export.py Normal file
View file

@ -0,0 +1,540 @@
import pandas as pd
import numpy as np
from pathlib import Path
import time
from backend.export.property_scenarios.main import process_export
from backend.export.property_scenarios.input_schema import ExportRequest
from backend.app.db.models.portfolio import PropertyModel, Epc, Portfolio, PortfolioStatus, PortfolioGoal, \
PropertyCreationStatus, PropertyDetailsEpcModel
from backend.app.db.models.recommendations import PlanModel, Recommendation, PlanRecommendations, \
RecommendationMaterials
from backend.app.db.models.materials import Material
from utils.logger import setup_logger
FIXTURE_PATH = Path("backend/export/tests/fixtures")
logger = setup_logger()
def load_csv(name: str) -> pd.DataFrame:
df = pd.read_csv(FIXTURE_PATH / name)
df = df.replace({np.nan: None})
return df
def test_default_export_integration(db_session):
# ----------------------------------------
# 1) Load csvs
# ----------------------------------------
t0 = time.perf_counter()
portfolio_df = load_csv("portfolio_569.csv")
properties_df = load_csv("properties_569.csv")
property_details_epc_df = load_csv("property_details_epc_569.csv")
plans_df = load_csv("plans_569.csv")
plan_recs_df = load_csv("plan_recs_569.csv")
recommendations_df = load_csv("recommendations_569.csv")
logger.info(
"Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s",
time.perf_counter() - t0,
len(properties_df),
len(plans_df),
len(recommendations_df),
)
logger.info("Starting database load")
db_load_t0 = time.perf_counter()
# ----------------------------------------
# 2) Insert test portfolio
# ----------------------------------------
portfolios = []
for row in portfolio_df.itertuples(index=False):
portfolios.append(
Portfolio(
id=row.id,
name=row.name,
status=PortfolioStatus[row.status.split(".")[-1]],
goal=PortfolioGoal[row.goal.split(".")[-1]] if row.goal else None,
)
)
db_session.bulk_save_objects(portfolios)
db_session.flush()
# ----------------------------------------
# 3) Insert test property
# ----------------------------------------
properties = []
for row in properties_df.itertuples(index=False):
row_dict = row._asdict()
row_dict["uprn"] = int(row_dict["uprn"]) if row_dict.get("uprn") else None
row_dict["building_reference_number"] = (
int(row_dict["building_reference_number"])
if row_dict.get("building_reference_number")
else None
)
prop = PropertyModel(**{
col: row_dict[col]
for col in PropertyModel.__table__.columns.keys()
if col in row_dict
})
prop.creation_status = PropertyCreationStatus[
row_dict["creation_status"].split(".")[-1]
]
prop.status = PortfolioStatus[row_dict["status"].split(".")[-1]]
if row_dict.get("current_epc_rating"):
prop.current_epc_rating = Epc[
row_dict["current_epc_rating"].split(".")[-1]
]
properties.append(prop)
db_session.bulk_save_objects(properties)
db_session.flush()
# ----------------------------------------
# 4) Insert property details - EPC
# ----------------------------------------
epc_rows = []
for row in property_details_epc_df.itertuples(index=False):
row_dict = row._asdict()
# Build only fields that exist on the model
epc_data = {
col.name: row_dict[col.name]
for col in PropertyDetailsEpcModel.__table__.columns.values()
if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"]
}
epc = PropertyDetailsEpcModel(
property_id=row.property_id,
portfolio_id=row.portfolio_id,
**epc_data,
)
epc_rows.append(epc)
db_session.bulk_save_objects(epc_rows)
db_session.flush()
# ----------------------------------------
# 4) Insert default plan
# ----------------------------------------
plans = []
for row in plans_df.itertuples(index=False):
row_dict = row._asdict()
if row_dict.get("post_epc_rating"):
row_dict["post_epc_rating"] = Epc[
row_dict["post_epc_rating"].split(".")[-1]
]
row_dict["scenario_id"] = None
plan = PlanModel(**{
col: row_dict[col]
for col in PlanModel.__table__.columns.keys()
if col in row_dict
})
plans.append(plan)
db_session.bulk_save_objects(plans)
db_session.flush()
# ----------------------------------------
# 5) Insert recommendation
# ----------------------------------------
recs = [
Recommendation(**{
col: row[col]
for col in Recommendation.__table__.columns.keys()
if col in row
})
for _, row in recommendations_df.iterrows()
]
db_session.bulk_save_objects(recs)
db_session.flush()
# ----------------------------------------
# 6) Insert PlanRecommendations
# ----------------------------------------
links = [
PlanRecommendations(
plan_id=row.plan_id,
recommendation_id=row.recommendation_id,
)
for row in plan_recs_df.itertuples(index=False)
]
db_session.bulk_save_objects(links)
db_session.commit()
logger.info("Inserted all data in %.2f seconds", time.perf_counter() - db_load_t0)
# ----------------------------------------
# 6) Build payload
# ----------------------------------------
body_dict = {
"task_id": "test",
"subtask_id": "test",
"portfolio_id": 569,
"scenario_ids": [],
"default_plans_only": True,
}
payload = ExportRequest.model_validate(body_dict)
# ----------------------------------------
# 7) Call process_export
# ----------------------------------------
logger.info(
"Recommendation count in DB: %s",
db_session.query(Recommendation).count()
)
logger.info(
"Property count in DB: %s",
db_session.query(PropertyModel).count()
)
logger.info(
"Property EPC in DB: %s",
db_session.query(PropertyDetailsEpcModel).count()
)
logger.info(
"Plan count in DB: %s",
db_session.query(PlanModel).count()
)
logger.info(
"PlanRecommendatons count in DB: %s",
db_session.query(PlanModel).count()
)
logger.info("Starting process_export")
process_t0 = time.perf_counter()
result = process_export(payload, session=db_session)
logger.info("process_export finished in %.2f seconds", time.perf_counter() - process_t0)
# ----------------------------------------
# 8) Assertions
# ----------------------------------------
assert "default_plans" in result, "Expected 'default_plans' in export result, got {}".format(result.keys())
df = result["default_plans"]
assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0])
failed = df[df["predicted_post_works_sap"] < 69]
failed_property_types = failed["property_type"].value_counts().to_dict()
assert failed_property_types["Flat"] == 2
# Check the houses
assert failed.shape[0]
assert df["total_retrofit_cost"].sum() == 41706.585999999996, (
"Expected total retrofit cost to be 10000, got {}".format(df["total_retrofit_cost"].sum())
)
assert df["predicted_post_works_sap"].sum() == 698.1, (
"Expected total predicted post works SAP to be 698.1, got {}".format(df["predicted_post_works_sap"].sum())
)
assert df["sap_points"].sum() == 100.10000000000001, (
"Expected total SAP points increase to be 100.10000000000001, got {}".format(df["sap_points"].sum())
)
assert df.shape == (10, 95), "Expected dataframe shape to be (10, 11), got {}".format(df.shape)
def test_solar_with_battery_example(db_session):
test_portfolio_id = 1
test_property_id = 1
portfolio_df = pd.DataFrame(
[{'id': test_portfolio_id, 'name': 'Example', 'budget': None,
'status': 'PortfolioStatus.SCOPING', 'goal': 'PortfolioGoal.NONE', 'cost': None, 'number_of_properties': None,
'co2_equivalent_savings': None, 'energy_savings': None, 'energy_cost_savings': None,
'property_valuation_increase': None, 'rental_yield_increase': None, 'total_work_hours': None,
'labour_days': None, 'created_at': '2026-02-12 21:23:37.862000+00:00',
'updated_at': '2026-02-12 21:23:37.862000+00:00', 'epc_breakdown_pre_retrofit': None,
'epc_breakdown_post_retrofit': None, 'n_units_to_retrofit': None, 'co2_per_unit_pre_retrofit': None,
'co2_per_unit_post_retrofit': None, 'energy_bill_per_unit_pre_retrofit': None,
'energy_bill_per_unit_post_retrofit': None, 'energy_consumption_per_unit_pre_retrofit': None,
'energy_consumption_per_unit_post_retrofit': None, 'valuation_improvement_per_unit': None,
'cost_per_unit': None, 'cost_per_co2_saved': None, 'cost_per_sap_point': None,
'valuation_return_on_investment': None}]
)
properties_df = pd.DataFrame(
[{'id': test_property_id, 'portfolio_id': test_portfolio_id, 'creation_status': 'PropertyCreationStatus.READY',
'uprn': 100090438731, 'landlord_property_id': 'BARR052', 'building_reference_number': 3460742868.0,
'status': 'PortfolioStatus.ASSESSMENT', 'address': '52, Barrack Street', 'postcode': 'CO1 2LR',
'has_pre_condition_report': True, 'has_recommendations': True, 'created_at': '2026-02-12 21:59:02.744427',
'updated_at': '2026-02-19 16:18:57.941443', 'property_type': 'House', 'built_form': 'End-Terrace',
'local_authority': 'Colchester', 'constituency': 'Colchester', 'number_of_rooms': 4.0, 'year_built': 1900.0,
'tenure': 'rental (private)', 'current_epc_rating': 'Epc.E', 'current_sap_points': 53.0,
'current_valuation': 0.0, 'installed_measures_sap_point_adjustment': 0.0,
'is_sap_points_adjusted_for_installed_measures': False, 'original_sap_points': 53.0}]
)
property_details_epc_df = pd.DataFrame(
[
{'id': 1534934, 'property_id': test_property_id, 'portfolio_id': test_portfolio_id,
'full_address': '48, Medcalf Road', 'lodgement_date': '2018-09-05', 'is_expired': False,
'total_floor_area': 68.0, 'walls': 'Solid brick, as built, no insulation', 'walls_rating': 1,
'roof': 'Pitched, no insulation', 'roof_rating': 1.0, 'floor': 'Solid, no insulation',
'floor_rating': None,
'windows': 'Fully double glazed', 'windows_rating': 4, 'heating': 'Boiler and radiators, mains gas',
'heating_rating': 4, 'heating_controls': 'Programmer, room thermostat and trvs',
'heating_controls_rating': 4,
'hot_water': 'From main system', 'hot_water_rating': 4,
'lighting': 'Low energy lighting in all fixed outlets', 'lighting_rating': 5,
'mainfuel': 'Mains gas not community', 'ventilation': 'natural', 'solar_pv': 0.0, 'solar_hot_water': False,
'wind_turbine': 0.0, 'floor_height': 2.55, 'number_heated_rooms': None, 'heat_loss_corridor': False,
'unheated_corridor_length': None, 'number_of_open_fireplaces': 0, 'number_of_extensions': 0,
'number_of_storeys': None, 'mains_gas': True, 'energy_tariff': 'Single',
'primary_energy_consumption': 278.0,
'co2_emissions': 3.81, 'current_energy_demand': 14643.366,
'current_energy_demand_heating_hotwater': 12185.6,
'estimated': False, 'sap_05_overwritten': False, 'sap_05_score': None, 'sap_05_epc_rating': None,
'heating_cost_current': 711.0628, 'hot_water_cost_current': 139.06198, 'lighting_cost_current': 70.770935,
'appliances_cost_current': 609.7844, 'gas_standing_charge': 128.0785,
'electricity_standing_charge': 199.8375,
'original_co2_emissions': 3.81, 'original_primary_energy_consumption': 278.0,
'original_current_energy_demand': 14643.366, 'original_current_energy_demand_heating_hotwater': 12185.6,
'installed_measures_co2_adjustment': 0.0, 'installed_measures_energy_demand_adjustment': 0.0,
'installed_measures_total_energy_bill_adjustment': 0.0, 'installed_measures_heat_demand_adjustment': 0.0,
'is_epc_adjusted_for_installed_measures': False}
]
)
plans_df = pd.DataFrame(
[
{'id': 0, 'name': None, 'portfolio_id': test_portfolio_id, 'property_id': test_property_id,
'scenario_id': 1060, 'created_at': '2026-02-19 16:14:45.560816', 'is_default': True,
'valuation_increase_lower_bound': 0.0302,
'valuation_increase_upper_bound': 0.07, 'valuation_increase_average': 0.048226666, 'plan_type': None,
'post_sap_points': 71.5, 'post_epc_rating': 'Epc.C', 'post_co2_emissions': 4.1813498,
'co2_savings': 0.71865046, 'post_energy_bill': 1447.5204, 'energy_bill_savings': 691.6662,
'post_energy_consumption': 15303.688, 'energy_consumption_savings': 3276.7622,
'valuation_post_retrofit': None, 'valuation_increase': None, 'cost_of_works': 6984.568,
'contingency_cost': 1003.9568}
]
)
plan_recs_df = pd.DataFrame(
[{'id': 0, 'plan_id': 0, 'recommendation_id': 0}]
)
recommendations_df = pd.DataFrame(
[{'id': 0, 'property_id': test_property_id, 'created_at': '2026-02-19 16:14:45.560816',
'type': 'solar_pv', 'measure_type': 'solar_pv',
'description': 'Fit solar',
'estimated_cost': 10000, 'default': True, 'starting_u_value': None, 'new_u_value': None, 'sap_points': 1.5,
'heat_demand': 14.9, 'kwh_savings': 1041.2, 'co2_equivalent_savings': 0.2, 'energy_savings': 14.9,
'energy_cost_savings': 72.639015, 'property_valuation_increase': None, 'rental_yield_increase': None,
'total_work_hours': 4.16, 'labour_days': 1.0, 'already_installed': False, 'plan_name': 'whatever'}
]
)
recommendations_materials_df = pd.DataFrame(
[
{
"id": 0, "recommendation_id": 0, "material_id": 0, "depth": None, "quantity": 1.0,
"quantity_unit": "part",
"estimated_cost": 10000, "created_at": '2026-02-19 16:14:45.560816',
"updated_at": '2026-02-19 16:14:45.560816',
}
]
)
materials_df = pd.DataFrame(
[
{'id': 0, 'type': 'solar_pv', 'description': 'Some solar product',
'depth': 75.0,
'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'Test',
'created_at': "'2026-02-19 16:14:45.560816", 'is_active': True,
'prime_material_cost': None,
'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0,
'total_cost': 10000,
'notes': None, 'is_installer_quote': True, 'innovation_rate': 0.25, 'size': None, 'size_unit': None,
'includes_scaffolding': True, 'includes_battery': True, 'battery_size': 5.8}
]
)
# Load into db
# -------------------------------------------------
# Insert Portfolio
# -------------------------------------------------
for row in portfolio_df.itertuples(index=False):
db_session.add(
Portfolio(
id=row.id,
name=row.name,
status=PortfolioStatus[row.status.split(".")[-1]],
goal=PortfolioGoal[row.goal.split(".")[-1]],
)
)
db_session.flush()
# -------------------------------------------------
# Insert Property
# -------------------------------------------------
for row in properties_df.itertuples(index=False):
prop = PropertyModel(
id=row.id,
portfolio_id=row.portfolio_id,
creation_status=PropertyCreationStatus[row.creation_status.split(".")[-1]],
status=PortfolioStatus[row.status.split(".")[-1]],
uprn=row.uprn,
property_type=row.property_type,
current_sap_points=row.current_sap_points,
current_epc_rating=Epc[row.current_epc_rating.split(".")[-1]],
)
db_session.add(prop)
db_session.flush()
# -------------------------------------------------
# Insert EPC Details
# -------------------------------------------------
for row in property_details_epc_df.itertuples(index=False):
epc = PropertyDetailsEpcModel(
property_id=row.property_id,
portfolio_id=row.portfolio_id,
full_address=row.full_address,
total_floor_area=row.total_floor_area,
walls=row.walls,
roof=row.roof,
windows=row.windows,
heating=row.heating,
solar_pv=row.solar_pv,
)
db_session.add(epc)
db_session.flush()
# -------------------------------------------------
# Insert Plan (default)
# -------------------------------------------------
for row in plans_df.itertuples(index=False):
plan = PlanModel(
id=row.id,
portfolio_id=row.portfolio_id,
property_id=row.property_id,
scenario_id=None, # default mode
is_default=row.is_default,
)
db_session.add(plan)
db_session.flush()
# -------------------------------------------------
# IMPORTANT: Force recommendation to be solar_pv
# -------------------------------------------------
recommendations_df.loc[0, "measure_type"] = "solar_pv"
for row in recommendations_df.itertuples(index=False):
rec = Recommendation(
id=row.id,
property_id=row.property_id,
measure_type=row.measure_type,
estimated_cost=row.estimated_cost,
default=row.default,
already_installed=row.already_installed,
sap_points=row.sap_points,
type=row.type,
description=row.description
)
db_session.add(rec)
db_session.flush()
# -------------------------------------------------
# Link Plan -> Recommendation
# -------------------------------------------------
for row in plan_recs_df.itertuples(index=False):
db_session.add(
PlanRecommendations(
plan_id=row.plan_id,
recommendation_id=row.recommendation_id,
)
)
db_session.flush()
# -------------------------------------------------
# Insert Material (includes_battery=True)
# -------------------------------------------------
for row in materials_df.itertuples(index=False):
material = Material(
id=row.id,
type=row.type,
description=row.description,
depth_unit=row.depth_unit,
cost_unit=row.cost_unit,
r_value_unit=row.r_value_unit,
thermal_conductivity_unit=row.thermal_conductivity_unit,
includes_battery=row.includes_battery,
is_active=row.is_active,
)
db_session.add(material)
db_session.flush()
# -------------------------------------------------
# Link Recommendation -> Material
# -------------------------------------------------
for row in recommendations_materials_df.itertuples(index=False):
db_session.add(
RecommendationMaterials(
recommendation_id=row.recommendation_id,
material_id=row.material_id,
depth=row.depth or 0.0,
quantity=row.quantity,
quantity_unit=row.quantity_unit,
estimated_cost=row.estimated_cost,
)
)
db_session.commit()
payload = ExportRequest.model_validate({
"task_id": "test",
"subtask_id": "test",
"portfolio_id": test_portfolio_id,
"scenario_ids": [],
"default_plans_only": True,
})
result = process_export(payload, session=db_session)
assert "default_plans" in result
df = result["default_plans"]
assert "solar_pv_with_battery" in df.columns
# solar_pv should NOT exist
assert "solar_pv" not in df.columns
assert df.shape[0] == 1, "Expected 1 property in the export, got {}".format(df.shape[0])
# Cost should land in correct column
assert df["solar_pv_with_battery"].iloc[0] == 10000

2
infrastructure/terraform/lambda/_template/main.tf
View file

@ -36,6 +36,8 @@ module "lambda" {
# Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
maximum_concurrency = var.maximum_concurrency
batch_size = var.batch_size
environment = {
STAGE = var.stage
LOG_LEVEL = "info"

5
infrastructure/terraform/lambda/_template/variables.tf
View file

@ -23,6 +23,11 @@ variable "maximum_concurrency" {
description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
}
variable "batch_size" {
type = number
default = 1
}
locals {
image_uri = "${var.ecr_repo_url}@${var.image_digest}"
}

4
infrastructure/terraform/lambda/categorisation/main.tf
View file

@ -20,9 +20,11 @@ module "lambda" {
name = "categorisation"
stage = var.stage
image_uri = local.image_uri
maximum_concurrency = var.maximum_concurrency
batch_size = var.batch_size
timeout = 120
environment = merge(
{

5
infrastructure/terraform/lambda/categorisation/variables.tf
View file

@ -23,6 +23,11 @@ variable "maximum_concurrency" {
description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
}
variable "batch_size" {
type = number
default = 2
}
locals {
image_uri = "${var.ecr_repo_url}@${var.image_digest}"
}

76
infrastructure/terraform/lambda/engine/main.tf Normal file
View file

@ -0,0 +1,76 @@
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
data "aws_secretsmanager_secret_version" "db_credentials" {
secret_id = "${var.stage}/assessment_model/db_credentials"
}
locals {
db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
}
module "lambda" {
source = "../modules/lambda_with_sqs"
name = "engine"
stage = var.stage
image_uri = local.image_uri
maximum_concurrency = var.maximum_concurrency
batch_size = var.batch_size
timeout = var.timeout
memory_size = var.memory_size
environment = merge(
{
STAGE = var.stage
LOG_LEVEL = "info"
# DB from Secrets Manager
DB_USERNAME = local.db_credentials.db_assessment_model_username
DB_PASSWORD = local.db_credentials.db_assessment_model_password
# Secrets from GitHub
DB_HOST = var.db_host
DB_NAME = var.db_name
DB_PORT = var.db_port
API_KEY = var.api_key
SECRET_KEY = var.secret_key
DOMAIN_NAME = var.domain_name
EPC_AUTH_TOKEN = var.epc_auth_token
GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
# Buckets - from terraform state
PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
HEATING_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
# SQS
ENGINE_SQS_URL = "test" # Not actually needed by engine, only to satisfy Settings
# Deployment
ECR_URI = var.ecr_repo_url
GITHUB_SHA = var.image_digest
}
)
}
### Policies and IAM
# S3
resource "aws_iam_role_policy_attachment" "engine_s3_read_and_write" {
role = module.lambda.role_name
policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn
}

16
infrastructure/terraform/lambda/engine/provider.tf Normal file
View file

@ -0,0 +1,16 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
}
}
backend "s3" {
bucket = "ara-engine-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
required_version = ">= 1.2.0"
}

88
infrastructure/terraform/lambda/engine/variables.tf Normal file
View file

@ -0,0 +1,88 @@
variable "lambda_name" {
type = string
description = "Logical name of the lambda (e.g. address2uprn)"
}
variable "stage" {
description = "Deployment stage (e.g. dev, prod)"
type = string
}
variable "ecr_repo_url" {
type = string
description = "ECR repository URL (no tag, no digest)"
}
variable "image_digest" {
type = string
description = "Image digest (sha256:...)"
}
variable "maximum_concurrency" {
type = number
default = 12
description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
}
variable "batch_size" {
type = number
default = 1
}
variable "timeout" {
type = number
default = 900
description = "Lambda timeout in seconds"
}
variable "memory_size" {
type = number
default = 3008
description = "Lambda memory size in MB"
}
variable "db_host" {
type = string
sensitive = true
}
variable "db_name" {
type = string
sensitive = true
}
variable "db_port" {
type = string
sensitive = true
}
variable "api_key" {
type = string
sensitive = true
}
variable "secret_key" {
type = string
sensitive = true
}
variable "domain_name" {
type = string
}
variable "epc_auth_token" {
type = string
sensitive = true
}
variable "google_solar_api_key" {
type = string
sensitive = true
}
locals {
image_uri = "${var.ecr_repo_url}@${var.image_digest}"
}
output "resolved_image_uri" {
value = local.image_uri
}

37
infrastructure/terraform/modules/ses/main.tf
View file

@ -12,6 +12,35 @@ resource "aws_iam_user" "ses_user" {
name = "${var.stage}-ses-user"
}
# SES configuration set for tracking events
resource "aws_ses_configuration_set" "this" {
name = "${var.stage}-ses-config"
}
# SNS topic for SES event notifications
resource "aws_sns_topic" "ses_events" {
name = "${var.stage}-ses-events"
}
# SES event destination for debugging
resource "aws_ses_event_destination" "sns" {
name = "ses-event-destination"
configuration_set_name = aws_ses_configuration_set.this.name
enabled = true
matching_types = [
"send",
"bounce",
"reject",
"complaint",
"delivery"
]
sns_destination {
topic_arn = aws_sns_topic.ses_events.arn
}
}
resource "aws_iam_user_policy" "ses_send_policy" {
name = "AllowSESSendEmail"
user = aws_iam_user.ses_user.name
@ -20,8 +49,8 @@ resource "aws_iam_user_policy" "ses_send_policy" {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
Effect = "Allow"
Action = [
"ses:SendEmail",
"ses:SendRawEmail"
]
@ -42,9 +71,9 @@ resource "aws_secretsmanager_secret" "ses_smtp" {
}
resource "aws_secretsmanager_secret_version" "ses_smtp" {
secret_id = aws_secretsmanager_secret.ses_smtp.id
secret_id = aws_secretsmanager_secret.ses_smtp.id
secret_string = jsonencode({
username = aws_iam_access_key.ses_user.id
password = aws_iam_access_key.ses_user.ses_smtp_password_v4
})
}
}

1
infrastructure/terraform/modules/ses/variables.tf
View file

@ -7,3 +7,4 @@ variable "stage" {
description = "Deployment stage (e.g. dev, prod)"
type = string
}

72
infrastructure/terraform/shared/main.tf
View file

@ -102,6 +102,11 @@ module "s3_presignable_bucket" {
allowed_origins = var.allowed_origins
}
output "retrofit_plan_trigger_bucket_name" {
value = module.s3_presignable_bucket.bucket_name
description = "Name of the retrofit plan trigger bucket"
}
module "s3_due_considerations_bucket" {
source = "../modules/s3_presignable_bucket"
bucketname = "retrofit-due-considerations-${var.stage}"
@ -134,6 +139,11 @@ module "retrofit_sap_predictions" {
allowed_origins = var.allowed_origins
}
output "retrofit_sap_predictions_bucket_name" {
value = module.retrofit_sap_predictions.bucket_name
description = "Name of the retrofit SAP predictions bucket"
}
module "retrofit_sap_data" {
source = "../modules/s3"
bucketname = "retrofit-data-${var.stage}"
@ -151,12 +161,22 @@ module "retrofit_carbon_predictions" {
allowed_origins = var.allowed_origins
}
output "retrofit_carbon_predictions_bucket_name" {
value = module.retrofit_carbon_predictions.bucket_name
description = "Name of the retrofit carbon predictions bucket"
}
module "retrofit_heat_predictions" {
source = "../modules/s3"
bucketname = "retrofit-heat-predictions-${var.stage}"
allowed_origins = var.allowed_origins
}
output "retrofit_heat_predictions_bucket_name" {
value = module.retrofit_heat_predictions.bucket_name
description = "Name of the retrofit heat predictions bucket"
}
module "retrofit_lighting_cost_predictions" {
source = "../modules/s3"
bucketname = "retrofit-lighting-cost-predictions-${var.stage}"
@ -181,12 +201,22 @@ module "retrofit_heating_kwh_predictions" {
allowed_origins = var.allowed_origins
}
output "retrofit_heating_kwh_predictions_bucket_name" {
value = module.retrofit_heating_kwh_predictions.bucket_name
description = "Name of the retrofit heating kWh predictions bucket"
}
module "retrofit_hotwater_kwh_predictions" {
source = "../modules/s3"
bucketname = "retrofit-hotwater-kwh-predictions-${var.stage}"
allowed_origins = var.allowed_origins
}
output "retrofit_hotwater_kwh_predictions_bucket_name" {
value = module.retrofit_hotwater_kwh_predictions.bucket_name
description = "Name of the retrofit hotwater kWh predictions bucket"
}
module "retrofit_sap_baseline_predictions" {
source = "../modules/s3"
bucketname = "retrofit-sap-baseline-predictions-${var.stage}"
@ -201,6 +231,11 @@ module "retrofit_energy_assessments" {
environment = var.stage
}
output "retrofit_energy_assessments_bucket_name" {
value = module.retrofit_energy_assessments.bucket_name
description = "Name of the retrofit energy assessments bucket"
}
# Set up the route53 record for the API
module "route53" {
source = "../modules/route53"
@ -446,4 +481,41 @@ module "ordnance_s3_read_and_write" {
output "ordnance_s3_read_and_write_arn" {
value = module.ordnance_s3_read_and_write.policy_arn
################################################
# Engine – Lambda ECR
################################################
module "engine_state_bucket" {
source = "../modules/tf_state_bucket"
bucket_name = "ara-engine-terraform-state"
}
module "engine_registry" {
source = "../modules/container_registry"
name = "engine"
stage = var.stage
}
# S3 policy for Engine to read and write from various S3 buckets
module "engine_s3_read_and_write" {
source = "../modules/s3_iam_policy"
policy_name = "EngineReadandWriteS3"
policy_description = "Allow Engine Lambda to read from and write to various S3 buckets"
bucket_arns = [
"arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}",
"arn:aws:s3:::${module.retrofit_sap_data.bucket_name}",
"arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
]
actions = ["s3:*"]
resource_paths = ["/*"]
}
output "engine_s3_read_and_write_arn" {
value = module.engine_s3_read_and_write.policy_arn
}

3
pyproject.toml Normal file
View file

@ -0,0 +1,3 @@
[tool.pyright]
reportUnknownMemberType = false
reportUnknownVariableType = false

8
pyrightconfig.json Normal file
View file

@ -0,0 +1,8 @@
{
"typeCheckingMode": "strict",
"venvPath": "/Users/khalimconn-kowlessar/opt/anaconda3/envs/",
"venv": "Fastapi-backend",
"include": [
"."
]
}

4
pytest.ini
View file

@ -1,4 +1,6 @@
[pytest]
pythonpath = .
log_cli = true
log_cli_level = INFO
addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial
testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests
testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests

1
recommendations/optimiser/StrategicOptimiser.py
View file

@ -165,7 +165,6 @@ class StrategicOptimiser:
min_gain=self.target_gain,
verbose=self.verbose
)
opt.setup()
opt.solve()

18
recommendations/optimiser/funding_optimiser.py
View file

@ -14,10 +14,9 @@ from typing import Mapping, Union
from itertools import product
from backend.app.plan.schemas import (
WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES, ECO4_ELIGIBILE_FABRIC_MEASURES
WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES, ECO4_ELIGIBILE_FABRIC_MEASURES,
WALL_INSULATION_WITH_VENTILATION_MEASURES
)
from recommendations.optimiser.CostOptimiser import CostOptimiser
from recommendations.optimiser.GainOptimiser import GainOptimiser
from recommendations.optimiser.StrategicOptimiser import StrategicOptimiser
from utils.logger import setup_logger
from backend.Funding import Funding
@ -689,9 +688,10 @@ def optimise_with_scenarios(
# - Only once the fabric has been upgraded, do we consider heating upgrades
# This should be wall insulation, roof insulation, floor insulation and windows
fabric_measures = WALL_INSULATION_MEASURES + ROOF_INSULATION_MEASURES + ECO4_ELIGIBILE_FABRIC_MEASURES + [
"internal_wall_insulation+mechanical_ventilation", "external_wall_insulation+mechanical_ventilation"
]
fabric_measures = (
WALL_INSULATION_MEASURES + ROOF_INSULATION_MEASURES + ECO4_ELIGIBILE_FABRIC_MEASURES +
WALL_INSULATION_WITH_VENTILATION_MEASURES
)
fabric_only_measures = [
[opt for opt in group if opt["type"] in fabric_measures] for group in optimisation_measures
@ -751,11 +751,9 @@ def optimise_with_scenarios(
# Scenario 1: Air source heat pump with required insulation
# ------------------------------------------------------------------
if enforce_heat_pump_insulation:
# Wall measures could be IWI or EWI
# Wall measures could be IWI, EWI or CWI
remaining_wall_measures = [
x for x in all_measure_types if x in WALL_INSULATION_MEASURES + [
"internal_wall_insulation+mechanical_ventilation", "external_wall_insulation+mechanical_ventilation"
]
x for x in all_measure_types if x in WALL_INSULATION_MEASURES + WALL_INSULATION_WITH_VENTILATION_MEASURES
]
remaining_roof_measures = [x for x in all_measure_types if x in ROOF_INSULATION_MEASURES]

475
recommendations/tests/test_optimiser_functions.py
View file

@ -588,248 +588,30 @@ class TestCheckNeedsVentilation:
class TestOptimiseWithScenarios:
def test_zero_gain(self, property_instance):
input_measures = [[{'id': '0_phase=0', 'cost': 16901.01977922431, 'gain': np.float64(2.0),
'type': 'internal_wall_insulation+mechanical_ventilation', 'innovation_uplift': 0,
'cost_minus_uplift': 16901.01977922431, 'raw_cost': 16341.019779224309,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '1_phase=1', 'cost': 1197.0, 'gain': 0, 'type': 'loft_insulation',
'innovation_uplift': 0, 'cost_minus_uplift': 1197.0, 'raw_cost': 1197.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '2_phase=1', 'cost': 1026.0, 'gain': 0, 'type': 'loft_insulation',
'innovation_uplift': 0, 'cost_minus_uplift': 1026.0, 'raw_cost': 1026.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '3_phase=1', 'cost': 855.0, 'gain': 0, 'type': 'loft_insulation',
'innovation_uplift': 0, 'cost_minus_uplift': 855.0, 'raw_cost': 855.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '5_phase=3', 'cost': 5343.75, 'gain': 1, 'type': 'suspended_floor_insulation',
'innovation_uplift': 0, 'cost_minus_uplift': 5343.75, 'raw_cost': 5343.75,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '6_phase=4', 'cost': 1009.5600000000001, 'gain': np.float64(0.9000000000000057),
'type': 'time_temperature_zone_control', 'innovation_uplift': 0,
'cost_minus_uplift': 1009.5600000000001, 'raw_cost': 1009.5600000000001,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '7_phase=4', 'cost': 18979.9, 'gain': np.float64(6.9), 'type': 'air_source_heat_pump',
'innovation_uplift': 0, 'cost_minus_uplift': 18979.9, 'raw_cost': 18979.9,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '8_phase=5', 'cost': 5420.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5420.0, 'raw_cost': 5420.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.6},
{'id': '9_phase=5', 'cost': 6210.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6210.0, 'raw_cost': 6210.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.6},
{'id': '10_phase=5', 'cost': 6820.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6820.0, 'raw_cost': 6820.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.6},
{'id': '11_phase=5', 'cost': 7202.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7202.0, 'raw_cost': 7202.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.915},
{'id': '12_phase=5', 'cost': 6495.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6495.0, 'raw_cost': 6495.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.92},
{'id': '13_phase=5', 'cost': 7285.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7285.0, 'raw_cost': 7285.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.92},
{'id': '14_phase=5', 'cost': 7895.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7895.0, 'raw_cost': 7895.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.92},
{'id': '15_phase=5', 'cost': 5520.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5520.0, 'raw_cost': 5520.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.0},
{'id': '16_phase=5', 'cost': 6310.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6310.0, 'raw_cost': 6310.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '17_phase=5', 'cost': 6920.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6920.0, 'raw_cost': 6920.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '18_phase=5', 'cost': 5840.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5840.0, 'raw_cost': 5840.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.2},
{'id': '19_phase=5', 'cost': 6630.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6630.0, 'raw_cost': 6630.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.2},
{'id': '20_phase=5', 'cost': 7240.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7240.0, 'raw_cost': 7240.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.2},
{'id': '21_phase=5', 'cost': 8630.0, 'gain': np.float64(14.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8630.0, 'raw_cost': 8630.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.655},
{'id': '22_phase=5', 'cost': 7660.0, 'gain': np.float64(14.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7660.0, 'raw_cost': 7660.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.66},
{'id': '23_phase=5', 'cost': 8470.0, 'gain': np.float64(14.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8470.0, 'raw_cost': 8470.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.66},
{'id': '24_phase=5', 'cost': 9090.0, 'gain': np.float64(14.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 9090.0, 'raw_cost': 9090.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.66},
{'id': '25_phase=5', 'cost': 7240.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7240.0, 'raw_cost': 7240.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.79},
{'id': '26_phase=5', 'cost': 8050.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8050.0, 'raw_cost': 8050.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.79},
{'id': '27_phase=5', 'cost': 8660.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8660.0, 'raw_cost': 8660.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.79},
{'id': '28_phase=5', 'cost': 5740.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5740.0, 'raw_cost': 5740.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.8},
{'id': '29_phase=5', 'cost': 6530.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6530.0, 'raw_cost': 6530.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.8},
{'id': '30_phase=5', 'cost': 7140.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7140.0, 'raw_cost': 7140.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.8},
{'id': '31_phase=5', 'cost': 8360.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8360.0, 'raw_cost': 8360.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.22},
{'id': '32_phase=5', 'cost': 7470.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7470.0, 'raw_cost': 7470.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.22},
{'id': '33_phase=5', 'cost': 8280.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8280.0, 'raw_cost': 8280.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.22},
{'id': '34_phase=5', 'cost': 8890.0, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8890.0, 'raw_cost': 8890.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 5.22},
{'id': '35_phase=5', 'cost': 5892.21, 'gain': np.float64(13.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5892.21, 'raw_cost': 5892.21,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 5.34},
{'id': '36_phase=5', 'cost': 5320.0, 'gain': np.float64(8.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5320.0, 'raw_cost': 5320.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.2},
{'id': '37_phase=5', 'cost': 6110.0, 'gain': np.float64(8.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6110.0, 'raw_cost': 6110.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.2},
{'id': '38_phase=5', 'cost': 6720.0, 'gain': np.float64(8.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6720.0, 'raw_cost': 6720.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.2},
{'id': '39_phase=5', 'cost': 6932.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6932.0, 'raw_cost': 6932.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.48},
{'id': '40_phase=5', 'cost': 6295.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6295.0, 'raw_cost': 6295.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 3.48},
{'id': '41_phase=5', 'cost': 7085.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7085.0, 'raw_cost': 7085.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.48},
{'id': '42_phase=5', 'cost': 7695.0, 'gain': np.float64(9.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7695.0, 'raw_cost': 7695.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 3.48},
{'id': '43_phase=5', 'cost': 5640.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5640.0, 'raw_cost': 5640.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.4},
{'id': '44_phase=5', 'cost': 6430.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6430.0, 'raw_cost': 6430.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.4},
{'id': '45_phase=5', 'cost': 7040.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7040.0, 'raw_cost': 7040.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.4},
{'id': '46_phase=5', 'cost': 8090.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8090.0, 'raw_cost': 8090.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.785},
{'id': '47_phase=5', 'cost': 7240.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7240.0, 'raw_cost': 7240.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.79},
{'id': '48_phase=5', 'cost': 8050.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8050.0, 'raw_cost': 8050.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.79},
{'id': '49_phase=5', 'cost': 8660.0, 'gain': np.float64(12.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8660.0, 'raw_cost': 8660.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.79},
{'id': '50_phase=5', 'cost': 5520.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5520.0, 'raw_cost': 5520.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.0},
{'id': '51_phase=5', 'cost': 6310.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6310.0, 'raw_cost': 6310.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '52_phase=5', 'cost': 6920.0, 'gain': np.float64(10.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6920.0, 'raw_cost': 6920.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '53_phase=5', 'cost': 7820.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7820.0, 'raw_cost': 7820.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.35},
{'id': '54_phase=5', 'cost': 6675.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6675.0, 'raw_cost': 6675.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.35},
{'id': '55_phase=5', 'cost': 7485.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7485.0, 'raw_cost': 7485.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.35},
{'id': '56_phase=5', 'cost': 8095.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 8095.0, 'raw_cost': 8095.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.35},
{'id': '57_phase=5', 'cost': 5640.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5640.0, 'raw_cost': 5640.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.4},
{'id': '58_phase=5', 'cost': 6430.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 6430.0, 'raw_cost': 6430.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.4},
{'id': '59_phase=5', 'cost': 7040.0, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 7040.0, 'raw_cost': 7040.0,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': True, 'array_size': 4.4},
{'id': '60_phase=5', 'cost': 5692.21, 'gain': np.float64(11.0), 'type': 'solar_pv',
'innovation_uplift': 0, 'cost_minus_uplift': 5692.21, 'raw_cost': 5692.21,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 4.45}]]
input_measures = [
[
{'already_installed': False, 'id': '0_phase=0',
'type': 'internal_wall_insulation+mechanical_ventilation',
'gain': np.float64(2.0), 'cost': 16901.01977922431}
],
[
{'already_installed': False, 'id': '1_phase=1', 'type': 'loft_insulation', 'gain': 0, 'cost': 1197.0},
],
[
{'already_installed': False, 'id': '5_phase=3', 'type': 'suspended_floor_insulation', 'gain': 1,
'cost': 5343.75}],
[
{'already_installed': False, 'id': '6_phase=4', 'type': 'time_temperature_zone_control',
'gain': np.float64(0.9000000000000057), 'cost': 1009.5600000000001},
{'already_installed': False, 'id': '7_phase=4', 'type': 'air_source_heat_pump', 'gain': np.float64(6.9),
'cost': 18979.9}],
[
{'already_installed': False, 'id': '8_phase=5', 'type': 'solar_pv', 'gain': np.float64(9.0),
'cost': 5420.0, "has_battery": False},
{'already_installed': False, 'id': '9_phase=5', 'type': 'solar_pv', 'gain': np.float64(9.0),
'cost': 6210.0, "has_battery": False},
]
]
solutions = optimise_with_scenarios(
p=property_instance,
@ -842,3 +624,212 @@ class TestOptimiseWithScenarios:
)
assert solutions.empty
def test_ashp_needing_cwi_first(self, property_instance):
input_measures = [
[
{'id': '0_phase=0', 'cost': 1653.5495595376553, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'already_installed': False},
{'id': '1_phase=0', 'cost': 1535.3279855335845, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'already_installed': False},
{'id': '2_phase=0', 'cost': 1801.326527042744, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'already_installed': False},
{'id': '3_phase=0', 'cost': 1505.7725920325668, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'already_installed': False}
],
[
{'id': '4_phase=1', 'cost': 766.5, 'gain': 0, 'type': 'loft_insulation', 'already_installed': False},
{'id': '5_phase=1', 'cost': 657.0, 'gain': 0, 'type': 'loft_insulation', 'already_installed': False},
{'id': '6_phase=1', 'cost': 547.5, 'gain': 0, 'type': 'loft_insulation', 'already_installed': False}
],
[
{'id': '8_phase=3', 'cost': 7.0, 'gain': 0, 'type': 'low_energy_lighting', 'already_installed': False}
],
[
{'id': '9_phase=4', 'cost': 1009.5600000000001, 'gain': np.float64(0.3),
'type': 'time_temperature_zone_control', 'already_installed': False},
{'id': '10_phase=4', 'cost': 18979.9, 'gain': np.float64(7.5), 'type': 'air_source_heat_pump',
'already_installed': False}
],
[
{'id': '11_phase=5', 'cost': 150.0, 'gain': np.float64(3.3), 'type': 'secondary_heating',
'already_installed': False}
],
[
{'id': '12_phase=6', 'cost': 5420.0, 'gain': np.float64(15.4), 'type': 'solar_pv',
'already_installed': False, "has_battery": False},
{'id': '13_phase=6', 'cost': 6210.0, 'gain': np.float64(15.4), 'type': 'solar_pv',
'already_installed': False, "has_battery": False}
]
]
solutions = optimise_with_scenarios(
p=property_instance,
input_measures=input_measures,
budget=None,
target_gain=7.5,
enforce_heat_pump_insulation=True,
enforce_fabric_first=False,
already_installed_sap=0, # To be passed to output
)
# heat pump solutions
heat_pump_solutions = solutions[solutions["scenario"] == "heat_pump_with_insulation"]
assert len(heat_pump_solutions) == 12
for x in heat_pump_solutions["items"].values:
res = [y["type"] for y in x]
# All results should include loft & CWI
assert "loft_insulation" in res
assert "cavity_wall_insulation+mechanical_ventilation" in res
def test_fabric_first(self, property_instance):
input_measures = [
[{'id': '0_phase=0', 'cost': 1653.5495595376553, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'innovation_uplift': 0,
'cost_minus_uplift': 1653.5495595376553, 'raw_cost': 1093.5495595376553, 'partial_project_funding': 0,
'partial_project_score': 0, 'uplift_project_score': 0, 'already_installed': False, 'has_battery': False,
'array_size': 0},
{'id': '1_phase=0', 'cost': 1535.3279855335845, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'innovation_uplift': 0,
'cost_minus_uplift': 1535.3279855335845, 'raw_cost': 975.3279855335845,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '2_phase=0', 'cost': 1801.326527042744, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'innovation_uplift': 0,
'cost_minus_uplift': 1801.326527042744, 'raw_cost': 1241.326527042744, 'partial_project_funding': 0,
'partial_project_score': 0, 'uplift_project_score': 0, 'already_installed': False, 'has_battery': False,
'array_size': 0},
{'id': '3_phase=0', 'cost': 1505.7725920325668, 'gain': 1,
'type': 'cavity_wall_insulation+mechanical_ventilation', 'innovation_uplift': 0,
'cost_minus_uplift': 1505.7725920325668, 'raw_cost': 945.7725920325668,
'partial_project_funding': 0, 'partial_project_score': 0, 'uplift_project_score': 0,
'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '4_phase=1', 'cost': 766.5, 'gain': 1, 'type': 'loft_insulation', 'innovation_uplift': 0,
'cost_minus_uplift': 766.5, 'raw_cost': 766.5, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '5_phase=1', 'cost': 657.0, 'gain': 1, 'type': 'loft_insulation', 'innovation_uplift': 0,
'cost_minus_uplift': 657.0, 'raw_cost': 657.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '6_phase=1', 'cost': 547.5, 'gain': 1, 'type': 'loft_insulation', 'innovation_uplift': 0,
'cost_minus_uplift': 547.5, 'raw_cost': 547.5, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '8_phase=3', 'cost': 7.0, 'gain': 1, 'type': 'low_energy_lighting', 'innovation_uplift': 0,
'cost_minus_uplift': 7.0, 'raw_cost': 7.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 0}],
[{'id': '9_phase=4', 'cost': 1009.5600000000001, 'gain': np.float64(0.3),
'type': 'time_temperature_zone_control', 'innovation_uplift': 0, 'cost_minus_uplift': 1009.5600000000001,
'raw_cost': 1009.5600000000001, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 0},
{'id': '10_phase=4', 'cost': 18979.9, 'gain': np.float64(7.5), 'type': 'air_source_heat_pump',
'innovation_uplift': 0, 'cost_minus_uplift': 18979.9, 'raw_cost': 18979.9, 'partial_project_funding': 0,
'partial_project_score': 0, 'uplift_project_score': 0, 'already_installed': False, 'has_battery': False,
'array_size': 0}],
[{'id': '11_phase=5', 'cost': 150.0, 'gain': np.float64(3.3), 'type': 'secondary_heating',
'innovation_uplift': 0, 'cost_minus_uplift': 150.0, 'raw_cost': 150.0, 'partial_project_funding': 0,
'partial_project_score': 0, 'uplift_project_score': 0, 'already_installed': False, 'has_battery': False,
'array_size': 0}],
[{'id': '12_phase=6', 'cost': 5420.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5420.0, 'raw_cost': 5420.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.6},
{'id': '13_phase=6', 'cost': 6210.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6210.0, 'raw_cost': 6210.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.6},
{'id': '14_phase=6', 'cost': 6820.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6820.0, 'raw_cost': 6820.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.6},
{'id': '15_phase=6', 'cost': 7202.0, 'gain': np.float64(15.9), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 7202.0, 'raw_cost': 7202.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.915},
{'id': '16_phase=6', 'cost': 6495.0, 'gain': np.float64(15.9), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6495.0, 'raw_cost': 6495.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.92},
{'id': '17_phase=6', 'cost': 7285.0, 'gain': np.float64(15.9), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 7285.0, 'raw_cost': 7285.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.92},
{'id': '18_phase=6', 'cost': 7895.0, 'gain': np.float64(15.9), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 7895.0, 'raw_cost': 7895.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.92},
{'id': '19_phase=6', 'cost': 5520.0, 'gain': np.float64(16.7), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5520.0, 'raw_cost': 5520.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 4.0},
{'id': '20_phase=6', 'cost': 6310.0, 'gain': np.float64(16.7), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6310.0, 'raw_cost': 6310.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '21_phase=6', 'cost': 6920.0, 'gain': np.float64(16.7), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6920.0, 'raw_cost': 6920.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 4.0},
{'id': '22_phase=6', 'cost': 5320.0, 'gain': np.float64(13.6), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5320.0, 'raw_cost': 5320.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.2},
{'id': '23_phase=6', 'cost': 6110.0, 'gain': np.float64(13.6), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6110.0, 'raw_cost': 6110.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.2},
{'id': '24_phase=6', 'cost': 6720.0, 'gain': np.float64(13.6), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6720.0, 'raw_cost': 6720.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.2},
{'id': '25_phase=6', 'cost': 6932.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6932.0, 'raw_cost': 6932.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.48},
{'id': '26_phase=6', 'cost': 6295.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6295.0, 'raw_cost': 6295.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.48},
{'id': '27_phase=6', 'cost': 7085.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 7085.0, 'raw_cost': 7085.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.48},
{'id': '28_phase=6', 'cost': 7695.0, 'gain': np.float64(15.4), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 7695.0, 'raw_cost': 7695.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': True, 'array_size': 3.48},
{'id': '29_phase=6', 'cost': 5220.0, 'gain': np.float64(12.2), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5220.0, 'raw_cost': 5220.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.8},
{'id': '30_phase=6', 'cost': 6662.0, 'gain': np.float64(12.8), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6662.0, 'raw_cost': 6662.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.045},
{'id': '31_phase=6', 'cost': 6095.0, 'gain': np.float64(12.8), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6095.0, 'raw_cost': 6095.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 3.05},
{'id': '32_phase=6', 'cost': 5160.0, 'gain': np.float64(10.1), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5160.0, 'raw_cost': 5160.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.4},
{'id': '33_phase=6', 'cost': 6392.0, 'gain': np.float64(10.1), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6392.0, 'raw_cost': 6392.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.61},
{'id': '34_phase=6', 'cost': 5910.0, 'gain': np.float64(10.1), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5910.0, 'raw_cost': 5910.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.61},
{'id': '35_phase=6', 'cost': 5100.0, 'gain': np.float64(8.0), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5100.0, 'raw_cost': 5100.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.0},
{'id': '36_phase=6', 'cost': 6098.0, 'gain': np.float64(9.1), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 6098.0, 'raw_cost': 6098.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.175},
{'id': '37_phase=6', 'cost': 5725.0, 'gain': np.float64(9.1), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5725.0, 'raw_cost': 5725.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 2.18},
{'id': '38_phase=6', 'cost': 5040.0, 'gain': np.float64(7.0), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5040.0, 'raw_cost': 5040.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 1.6},
{'id': '39_phase=6', 'cost': 5828.0, 'gain': np.float64(7.0), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5828.0, 'raw_cost': 5828.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 1.74},
{'id': '40_phase=6', 'cost': 5540.0, 'gain': np.float64(7.0), 'type': 'solar_pv', 'innovation_uplift': 0,
'cost_minus_uplift': 5540.0, 'raw_cost': 5540.0, 'partial_project_funding': 0, 'partial_project_score': 0,
'uplift_project_score': 0, 'already_installed': False, 'has_battery': False, 'array_size': 1.74}]
]
solutions = optimise_with_scenarios(
p=property_instance,
input_measures=input_measures,
budget=None,
target_gain=7.5,
enforce_heat_pump_insulation=True,
enforce_fabric_first=True,
already_installed_sap=0, # To be passed to output
)
assert solutions.shape[0] == 1
items = solutions["items"].values[0]
types = [x["type"] for x in items]
assert types == ['cavity_wall_insulation+mechanical_ventilation', 'loft_insulation', 'solar_pv']

3
serverless.yml
View file

@ -30,6 +30,8 @@ provider:
GOOGLE_SOLAR_API_KEY: ${env:GOOGLE_SOLAR_API_KEY}
ENGINE_SQS_URL:
Ref: EngineQueue
# hardcode the categorisation queue for now as it's created in terraform
CATEGORISATION_SQS_URL: "https://sqs.eu-west-2.amazonaws.com/337213553626/categorisation-queue-dev"
plugins:
- serverless-python-requirements
@ -106,6 +108,7 @@ resources:
- sqs:SendMessage
Resource:
- Fn::GetAtt: [ EngineQueue, Arn ]
- "arn:aws:sqs:eu-west-2:337213553626:categorisation-queue-dev"
- Effect: Allow
Action:
- s3:GetObject

10
sfr/principal_pitch/2_export_data.py
View file

@ -28,15 +28,13 @@ from sqlalchemy import func
# PORTFOLIO_ID = 206
# SCENARIOS = [389]
PORTFOLIO_ID = 404
SCENARIOS = [819, 829, 872]
PORTFOLIO_ID = 581
SCENARIOS = [1124]
scenario_names = {
819: "EPC C",
829: "EPC C - no solid floor",
872: "EPC C - no solid floor, refresh",
1124: "EPC C - Solar Focused",
}
project_name = "lincs_rural"
project_name = "WCHG EPC D rated properties"
def get_data(portfolio_id, scenario_ids):

4
test.requirements.txt
View file

@ -2,4 +2,6 @@ pytest
mock
pytest-cov
pytest-mock
dotenv
dotenv
psycopg[binary]
pytest-postgresql