From 63bafda6ad7fc8604488800a96fac9bcbe05d353 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 12 Jan 2026 16:30:57 +0000 Subject: [PATCH] save --- .devcontainer/Dockerfile | 32 +- asset_list/AssetList.py | 3 +- asset_list/DataMapper.py | 2 +- asset_list/app.py | 1000 +------------------------------------- 4 files changed, 27 insertions(+), 1010 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4d898973..d7021f4c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.12-bullseye +FROM python:3.11.10-slim-bullseye + ARG USER=vscode ARG DEBIAN_FRONTEND=noninteractive @@ -9,24 +10,31 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential pkg-config automake autoconf libtool \ && rm -rf /var/lib/apt/lists/* -# 2) Build and install libpostal from source -RUN git clone --depth 1 https://github.com/openvenues/libpostal /tmp/libpostal \ - && cd /tmp/libpostal \ - && ./bootstrap.sh \ - && ./configure --datadir=/usr/local/share/libpostal \ - && make -j"$(nproc)" \ - && make install \ - && ldconfig \ - && rm -rf /tmp/libpostal +# # 2) Build and install libpostal from source +# RUN git clone --depth 1 https://github.com/openvenues/libpostal /tmp/libpostal \ +# && cd /tmp/libpostal \ +# && ./bootstrap.sh \ +# && ./configure --datadir=/usr/local/share/libpostal \ +# && make -j"$(nproc)" \ +# && make install \ +# && ldconfig \ +# && rm -rf /tmp/libpostal # 3) Create the user and grant sudo privileges RUN useradd -m -s /usr/bin/bash ${USER} \ && echo "${USER} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/${USER} \ && chmod 0440 /etc/sudoers.d/${USER} -# 4) Python deps +# # 4) Python deps - if you want to run assest list +# ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 +# ADD asset_list/requirements.txt requirements.txt +# RUN pip install -r requirements.txt + +# ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 -ADD asset_list/requirements.txt requirements.txt +ADD backend/engine/requirements.txt requirements1.txt +ADD backend/app/requirements/requirements.txt requirements2.txt +RUN cat requirements1.txt requirements2.txt > requirements.txt RUN pip install -r requirements.txt # 5) Workdir diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 9569afe8..248eb8ea 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -34,7 +34,8 @@ from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes logger = setup_logger() # OpenAI API Key (set this in your environment variables for security) -OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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") + class DataRemapper: diff --git a/asset_list/DataMapper.py b/asset_list/DataMapper.py index ac1b8db3..0751a7cf 100644 --- a/asset_list/DataMapper.py +++ b/asset_list/DataMapper.py @@ -1,5 +1,5 @@ # OpenAI API Key (set this in your environment variables for security) -OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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") class DataRemapper: diff --git a/asset_list/app.py b/asset_list/app.py index 6b820bc5..b78e968f 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -57,47 +57,13 @@ def app(): EPC recommendations Property UPRN """ - # Adhoc script for Nick Sout - data_folder = "../../home/Downloads" - data_filename = "nick.xlsx" - sheet_name = "not_on_SAL" - postcode_column = 'Postcode' - address1_column = "Address 1" - address1_method = None - fulladdress_column = None - address_cols_to_concat = ["Address 1"] - missing_postcodes_method = None - landlord_year_built = None - landlord_os_uprn = None - landlord_property_type = None - landlord_built_form = None - landlord_wall_construction = None - landlord_roof_construction = None - landlord_heating_system = None - landlord_existing_pv = None - landlord_property_id = "Property Sequence Number" - landlord_sap = None - outcomes_filename = None - outcomes_sheetname = None - outcomes_postcode = None - outcomes_houseno = None - outcomes_id = None - outcomes_address = None - master_filepaths = [] - master_id_colnames = [] - master_to_asset_list_filepath = None - phase = False - ecosurv_landlords = None - asset_list_header = 0 - landlord_block_reference = None - data_folder = ("/workspaces/model/asset_list") data_filename = "assets.xlsx" - sheet_name = "Standardised Asset List" - postcode_column = 'domna_postcode' + sheet_name = "Fabric Programme" + postcode_column = 'Postcode' address1_column = None address1_method = 'house_number_extraction' - fulladdress_column = 'domna_full_address' + fulladdress_column = 'Address' address_cols_to_concat = None missing_postcodes_method = None landlord_year_built = None @@ -108,7 +74,7 @@ def app(): landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "landlord_property_id" + landlord_property_id = "UPRN" landlord_sap = None outcomes_filename = None outcomes_sheetname = None @@ -125,964 +91,6 @@ def app(): landlord_block_reference = None - # # Lambeth - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lambeth" - # data_filename = "LAMBETH Asset List ( Incomplete).xlsx" - # sheet_name = "Green properties" - # postcode_column = 'SX3 Postcode' - # address1_column = "SX3 Short Address" - # address1_method = None - # fulladdress_column = None - # address_cols_to_concat = ["SX3 Short Address"] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = "Property Type" - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "row_id" - # landlord_sap = None - # outcomes_filename = None - # outcomes_sheetname = None - # outcomes_postcode = None - # outcomes_houseno = None - # outcomes_id = None - # outcomes_address = None - # master_filepaths = [] - # master_id_colnames = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = None - - # # Colchester - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester/Aug2025 202 inspections" - # data_filename = "Colchester Borough Homes - Inspections - Additional 202 Addresses JW 280725 copy.xlsx" - # sheet_name = "Extra 202 Colchester Addresses" - # postcode_column = 'domna_postcode' - # address1_column = "domna_address_1" - # address1_method = None - # fulladdress_column = "domna_full_address" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = "landlord_property_type" - # landlord_built_form = "landlord_built_form" - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "landlord_property_id" - # landlord_sap = None - # outcomes_filename = None - # outcomes_sheetname = None - # outcomes_postcode = None - # outcomes_houseno = None - # outcomes_id = None - # outcomes_address = None - # master_filepaths = [] - # master_id_colnames = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = "landlord_block_reference" - - # # Abri - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections" - # data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx" - # sheet_name = "Reviewed List" - # postcode_column = 'domna_postcode' - # address1_column = "domna_address_1" - # address1_method = None - # fulladdress_column = "domna_full_address" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "landlord_year_built" - # landlord_os_uprn = None - # landlord_property_type = "PropertyType_original_from_landlord" - # landlord_built_form = "BuildForm_original_from_landlord" - # landlord_wall_construction = "Wall Construction_original_from_landlord" - # landlord_roof_construction = None - # landlord_heating_system = "HeatingType_original_from_landlord" - # landlord_existing_pv = None - # landlord_property_id = "landlord_property_id" - # landlord_sap = None - # outcomes_filename = None - # outcomes_sheetname = None - # outcomes_postcode = None - # outcomes_houseno = None - # outcomes_id = None - # outcomes_address = None - # master_filepaths = [] - # master_id_colnames = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = None - - # Freebridge - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge/Aug2025 programme" - # data_filename = "Domna - FCH property data May 25 copy.xlsx" - # sheet_name = "EPC Data" - # postcode_column = 'Post Code' - # address1_column = "Address 1" - # address1_method = None - # fulladdress_column = None - # address_cols_to_concat = ["Address 1", "Address 4"] - # missing_postcodes_method = None - # landlord_year_built = "Build Date" - # landlord_os_uprn = None - # landlord_property_type = "Property Type" - # landlord_built_form = None - # landlord_wall_construction = "Walls Description" - # landlord_heating_system = "Heating Type" - # landlord_existing_pv = None - # landlord_property_id = "Place Ref" - # landlord_roof_construction = "Roof Description" - # landlord_sap = "Current SAP" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_address = [] - # outcomes_id = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [] - # phase = False # Inspections not complete, produce a partial view - # ecosurv_landlords = None - - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands" - # data_filename = "Broadlands Asset List.xlsx" - # sheet_name = "Assets" - # postcode_column = 'POSTCODE' - # fulladdress_column = None - # address1_column = "Address1" - # address1_method = None - # address_cols_to_concat = ["Address1"] - # missing_postcodes_method = None - # landlord_year_built = "DATEBUILT" - # landlord_os_uprn = None - # landlord_property_type = "PropertyType" - # landlord_built_form = "PropertyType" - # landlord_wall_construction = None - # landlord_heating_system = "Heating Fuel" - # landlord_existing_pv = None - # landlord_property_id = "Row ID" - # outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")] - # outcomes_sheetname = ["Sheet1"] - # outcomes_postcode = ["Postcode"] - # outcomes_houseno = ["No."] - # outcomes_address = ["Address"] - # outcomes_id = [None] - # master_filepaths = [ - # os.path.join(data_folder, "eco3 submissions.csv"), - # os.path.join(data_folder, "eco4 submissions.csv"), - # ] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None, None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = "broadland" - # # - # - # # Community: - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme" - # data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'POSTCODE' - # fulladdress_column = "ADDRESS" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "BUILD DATE" - # landlord_os_uprn = None - # landlord_property_type = "PROPERTY TYPE" - # landlord_built_form = "Archetype" # Using the inspections archetype - # landlord_wall_construction = "CONSTRUCTION TYPE" - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # landlord_sap = None - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 1 - # landlord_block_reference = None - # master_id_colnames = [] - # - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis" - # data_filename = "EalingProjectRebuildJW210725.xlsx" - # sheet_name = "Refine & Houses" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = None # Using the inspections property type - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "Property ref" - # landlord_sap = None - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = "Block Reference" - # master_id_colnames = [] - # - # # TODO: Delete me - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/" - # data_filename = "20250716 Asset List.xlsx" - # sheet_name = "Sheet 1" - # postcode_column = 'Postcode' - # fulladdress_column = "Full Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = None - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "Row ID" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_address = [] - # outcomes_id = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = None - # - # # Southend - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme" - # data_filename = "SOUTHEND - RYAN.xlsx" - # sheet_name = "July 2025 Surveys" - # postcode_column = 'Postcode' - # fulladdress_column = "Full postal address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Property age" - # landlord_os_uprn = None - # landlord_property_type = "Property type" - # landlord_built_form = "Property type" - # landlord_wall_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "ID" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_address = [] - # outcomes_id = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = None - # - # # For Rooftop - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop" - # data_filename = "Rooftop Asset List - July 2025.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'post_code' - # fulladdress_column = None - # address1_column = "add_1" - # address1_method = None - # address_cols_to_concat = [ - # "add_1", "add_2", "add_3", "add_4" - # ] - # missing_postcodes_method = None - # landlord_year_built = "date_built" - # landlord_os_uprn = None - # landlord_property_type = "ConstructionStyle" - # landlord_built_form = "ConstructionStyle" - # landlord_wall_construction = None - # landlord_heating_system = "Description" - # landlord_existing_pv = None - # landlord_property_id = "PropertyCode" - # outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")] - # outcomes_sheetname = ["OUTCOMES"] - # outcomes_postcode = ["POSTCODE"] - # outcomes_houseno = ["NO"] - # outcomes_address = ["ADDRESS"] - # outcomes_id = [None] - # master_filepaths = [os.path.join(data_folder, "Master.csv")] - # master_to_asset_list_filepath = None - # asset_list_header = 1 - # landlord_block_reference = "bl_rec_ref" - # master_id_colnames = [None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = "rooftop" - # - # # For Housing - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025" - # data_filename = "FOR HOUSING Asset List (Combined).xlsx" - # sheet_name = "Asset List" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = "Type" - # landlord_built_form = "Type" - # landlord_wall_construction = None - # landlord_heating_system = "Heating - full" - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")] - # outcomes_sheetname = ["Sheet1"] - # outcomes_postcode = ["POSTCODE"] - # outcomes_houseno = ["NO"] - # outcomes_address = ["ADDRESS"] - # outcomes_id = [None] - # master_filepaths = [os.path.join(data_folder, "submissions.csv")] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = "SAP" - # ecosurv_landlords = "for housing" - # - # # CDS - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS" - # data_filename = "Founder Estates - Asset List.xlsx" - # sheet_name = "Combined" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = None - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_heating_system = "Heating Type" - # landlord_existing_pv = None - # landlord_property_id = "Row ID" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_address = [] - # outcomes_id = [] - # master_filepaths = [os.path.join(data_folder, "submissions.csv")] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = "cds" - # - # # Plus Dane - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/" - # data_filename = "20250711 Plus Dane Asset List.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Property Age" - # landlord_os_uprn = None - # landlord_property_type = "Property Type" - # landlord_built_form = "Built Form" - # landlord_wall_construction = "Wall Construction" - # landlord_heating_system = "Full Heating System" - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # outcomes_filename = [ - # os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"), - # os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"), - # os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"), - # ] - # outcomes_sheetname = [ - # "CWI & LI - 2024", "2025 - CWI", "PV - 2025", - # ] - # outcomes_postcode = ["Postcode", "Postcode", "Postcode"] - # outcomes_houseno = ["No.", "No", "No"] - # outcomes_address = ["Address", "Address", "Address"] - # outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"] - # master_filepaths = [ - # os.path.join(data_folder, "submissions/JJC-Table 1.csv"), - # os.path.join(data_folder, "submissions/SCIS-Table 1.csv") - # ] - # master_to_asset_list_filepath = None - # asset_list_header = 1 - # landlord_block_reference = None - # master_id_colnames = [None, None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = "SAP Rating" - # ecosurv_landlords = "plus dane" - - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme" - # data_filename = "20250710 Asset List Brentwood.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'Postcode' - # fulladdress_column = None - # address1_column = "House Number" - # address1_method = None - # address_cols_to_concat = ["House Number", "Address Line 1", "Address Line 2", "Address Line 3"] - # missing_postcodes_method = None - # landlord_year_built = "Year Built" - # landlord_os_uprn = None - # landlord_property_type = "Dwelling" - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_heating_system = "Heating" - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # outcomes_filename = [os.path.join(data_folder, "Brentwood - outcomes for analysis.xlsx")] - # outcomes_sheetname = ["OUTCOMES"] - # outcomes_postcode = ["POSTCODE"] - # outcomes_houseno = [None] - # outcomes_address = ["ADDRESS"] - # outcomes_id = [None] - # master_filepaths = [os.path.join(data_folder, "Submissions.csv")] - # master_to_asset_list_filepath = None - # asset_list_header = 1 - # landlord_block_reference = None - # master_id_colnames = [None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = "brentwood" - - # Brentwood - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme" - # data_filename = "20250710 Asset List Brentwood.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'Postcode' - # fulladdress_column = None - # address1_column = "House Number" - # address1_method = None - # address_cols_to_concat = ["House Number", "Address Line 1", "Address Line 2", "Address Line 3"] - # missing_postcodes_method = None - # landlord_year_built = "Year Built" - # landlord_os_uprn = None - # landlord_property_type = "Dwelling" - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_heating_system = "Heating" - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # outcomes_filename = [os.path.join(data_folder, "Brentwood - outcomes for analysis.xlsx")] - # outcomes_sheetname = ["OUTCOMES"] - # outcomes_postcode = ["POSTCODE"] - # outcomes_houseno = [None] - # outcomes_address = ["ADDRESS"] - # outcomes_id = [None] - # master_filepaths = [os.path.join(data_folder, "Submissions.csv")] - # master_to_asset_list_filepath = None - # asset_list_header = 1 - # landlord_block_reference = None - # master_id_colnames = [None] - # landlord_roof_construction = None - # phase = False - # landlord_sap = None - # ecosurv_landlords = "brentwood" - # - # # Eastlight - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Eastlight/New Programme" - # data_filename = "INSPECTIONS MASTER Non Tech.xlsx" - # sheet_name = "EASTLIGHT CW" - # postcode_column = 'Postcode' - # fulladdress_column = None - # address1_column = "HouseName" - # address1_method = None - # address_cols_to_concat = ["HouseName", "Block", "Address1", "Address2", "Address3"] - # missing_postcodes_method = None - # landlord_year_built = "Built In Year" - # landlord_os_uprn = None - # landlord_property_type = "AssetType" - # landlord_built_form = "Archetype" # Using inspections archetype - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = "Main Heating Source" - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # landlord_sap = "SAP Score" - # outcomes_filename = [ - # os.path.join(data_folder, "Eastlight_CWI_JJC_2025.xlsx"), - # os.path.join(data_folder, "Eastlight_CWI_SCIS_2025.xlsx"), - # ] - # outcomes_sheetname = ["Outcomes", "Feedback"] - # outcomes_postcode = ["Postcode", "Postcode"] - # outcomes_houseno = ["No", "No."] - # outcomes_id = [None, None] - # outcomes_address = ["Address", "Address"] - # master_filepaths = [ - # os.path.join(data_folder, "ECO 3-Table 1.csv"), - # os.path.join(data_folder, "ECO 4-Table 1.csv"), - # ] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = "eastlight" - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None, None] - # landlord_sap = None - - # Pickering and Ferens - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Pickering & Ferens" - # data_filename = "SAP 9 vs SAP 10 Sava Intelligent Energy - Property List (190625).xlsx" - # sheet_name = "Sava Intelligent Energy - Prope" - # postcode_column = 'Postcode' - # fulladdress_column = 'Address' - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = "Property Type" # Using the inspections property type - # landlord_built_form = "Archetype 2" - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "UPRN" - # landlord_sap = "SAP Rating (RdSAP 10)" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [ - # os.path.join(data_folder, "PICKERING & FERENS ROLLING MASTER SHEET HEDGEFUND - 26.7.24 - K.csv"), - # os.path.join(data_folder, "PICKERING & FERENS NEW MASTER GBIS UPDATED 21.8.24 - M - For Analysis.csv"), - # ] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = "pickering" - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None, None] - - # Colchester - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester" - # data_filename = "Warmfront data- Colchester Borough Homes (Complete).xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'Full Address.1' - # fulladdress_column = "Full Address" - # address1_column = None - # address1_method = "first_word" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Build Date" - # landlord_os_uprn = None - # landlord_property_type = "Property Type" - # landlord_wall_construction = "Wallinsul" - # landlord_heating_system = "HeatSorc" - # landlord_existing_pv = None - # landlord_property_id = "Property Reference" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # asset_list_header = 0 - # landlord_built_form = None - # landlord_roof_construction = None - # landlord_sap = None - # landlord_block_reference = None - # phase = False - # ecosurv_landlords = None - # master_id_colnames = [] - - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Hubspot" - # data_filename = "EalingFlats.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = None # Using the inspections property type - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "Property ref" - # landlord_sap = None - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = "Block Ref" - # master_id_colnames = [] - - # Southern - Jan list - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/New Programme/Jan 2025 List" - # data_filename = "SOUTHERN ASSETS January 2025 Additions Query 21.03.2025.xlsx" - # sheet_name = "Jan 2025 additions" - # postcode_column = 'Post Code' - # fulladdress_column = None - # address1_column = "NO." - # address1_method = None - # address_cols_to_concat = ["NO.", "Street / Block Name", "Town/Area"] - # missing_postcodes_method = None - # landlord_year_built = None - # landlord_os_uprn = None - # landlord_property_type = None # Using the inspections property type - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_roof_construction = None - # landlord_heating_system = None - # landlord_existing_pv = None - # landlord_property_id = "SH Property Reference" - # landlord_sap = None - # outcomes_filename = [ - # os.path.join(data_folder, "RT - Southern Housing Group - JJC.xlsx"), - # os.path.join(data_folder, "RT - SOUTHERN OUTCOMES - SCIS Merged.xlsx"), - # ] - # outcomes_sheetname = ["Feedback", "Collated"] - # outcomes_postcode = ["Poscode", "Postcode"] - # outcomes_houseno = ["No.", "No"] - # outcomes_id = ["UPRNs", None] - # outcomes_address = ["Address", "Address"] - # master_filepaths = [ - # os.path.join(data_folder, "southern_submissions/CAVITY'S - DECEMBER 2018-Table 1.csv"), - # os.path.join(data_folder, "southern_submissions/CAVITY'S 2019-Table 1.csv"), - # os.path.join(data_folder, "southern_submissions/CAVITY'S ECO4-Table 1.csv"), - # os.path.join(data_folder, "southern_submissions/LOFT'S-Table 1.csv"), - # ] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = "southern" - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [None, None, None, None] - - # NCHA - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NCHA" - # data_filename = "Energy Information MASTER June 2025.xlsx" - # sheet_name = "Data" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Build Date (HAR10)" - # landlord_os_uprn = None - # landlord_property_type = "Property Type (HAR10)" - # landlord_built_form = "Build Form (EPC)" - # landlord_wall_construction = "Wall Description" - # landlord_roof_construction = None - # landlord_heating_system = "HEAT Code" - # landlord_existing_pv = None - # landlord_property_id = "Place ref" - # landlord_sap = "EPC SAP" - # outcomes_filename = None - # outcomes_sheetname = None - # outcomes_postcode = None - # outcomes_houseno = None - # outcomes_id = None - # outcomes_address = None - # master_filepaths = [] - # master_to_asset_list_filepath = None - # phase = False - # ecosurv_landlords = None - # asset_list_header = 0 - # landlord_block_reference = None - # master_id_colnames = [] - - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Calico" - # data_filename = "07.04 CALICO - Final List.xlsx" - # asset_list_header = 2 - # sheet_name = "Final List" - # postcode_column = 'Postcode' - # fulladdress_column = None - # address1_column = "Property Number / Name" - # address1_method = None - # address_cols_to_concat = [ - # "Property Number / Name", - # "Street", - # "Town" - # ] - # missing_postcodes_method = None - # landlord_year_built = "NROSH Estimated Build Date" - # landlord_os_uprn = None - # landlord_property_type = "Asset Type" - # landlord_built_form = None - # landlord_wall_construction = "Wall Type" - # landlord_heating_system = "Boiler Type" - # landlord_existing_pv = None - # landlord_property_id = "Asset Reference" - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_id_colnames = [] - # master_to_asset_list_filepath = None - # landlord_roof_construction = None - # landlord_block_reference = None - # landlord_sap = "Current Efficiency Rating - Score" - # phase = None - # ecosurv_landlords = None - - # data_folder = ( - # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/2018 Asset - # List" - # ) - # data_filename = "LIVEWEST STOCK - 23rd October 2018.xlsx" - # sheet_name = "Assets" - # postcode_column = 'Postcode' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Build Year" - # landlord_os_uprn = None - # landlord_property_type = "Property Archetype" - # landlord_built_form = None - # landlord_wall_construction = None - # landlord_heating_system = "Heating Fuel Type" - # landlord_existing_pv = None - # landlord_property_id = "Uprn - DO NOT DELETE" - # outcomes_filename = [ - # os.path.join(data_folder, "RT - LiveWest.xlsx") - # ] - # outcomes_sheetname = ["Feedback"] - # outcomes_postcode = ["Poscode"] - # outcomes_houseno = ["No."] - # outcomes_id = ["UPRN"] - # outcomes_address = ["Address"] - # master_filepaths = [ - # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling - # Master " - # "- redacted for analysis/CAVITY-Table 1.csv" - # ] - # master_id_colnames = [None] - # master_to_asset_list_filepath = None - # landlord_roof_construction = None - # landlord_block_reference = None - # landlord_sap = None - # phase = None - # ecosurv_landlords = "livewest|live west" - - # data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March " - # "2025/Livewest Asset List (Original) - csv") - # data_filename = "Report-Table 1.csv" - # sheet_name = None - # postcode_column = 'Postcode' - # fulladdress_column = "T1_Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Build Yr" - # landlord_os_uprn = None - # landlord_property_type = "T1_AssetType" - # landlord_built_form = "T1_AssetType" - # landlord_wall_construction = "Wall Type Cavity" - # landlord_heating_system = "Heating Fuel" - # landlord_existing_pv = None - # landlord_property_id = "T1_UPRN" - # outcomes_filename = [ - # os.path.join(data_folder, "RT - LiveWest.xlsx") - # ] - # outcomes_address = ["Address"] - # outcomes_sheetname = ["Feedback"] - # outcomes_postcode = ["Poscode"] - # outcomes_houseno = ["No."] - # outcomes_id = ["UPRN"] - # master_filepaths = [ - # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling - # Master " - # "- redacted for analysis/CAVITY-Table 1.csv" - # ] - # master_id_colnames = [None] - # master_to_asset_list_filepath = None - # landlord_roof_construction = None - # landlord_block_reference = None - # landlord_sap = None - # phase = None - # ecosurv_landlords = "livewest|live west" - - # Stori - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Storicymru" - # data_filename = "Asset list - for analysis.xlsx" - # sheet_name = "SAP and Costs Calculations" - # postcode_column = 'Postcode' - # fulladdress_column = "Address1" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Age" - # landlord_os_uprn = None - # landlord_property_type = "TYPE" - # landlord_built_form = "AGE / DETACHMENT" - # landlord_wall_construction = "WALL" - # landlord_roof_construction = "LOFT INSULATION" - # landlord_heating_system = "BOILER" - # landlord_existing_pv = "SOLAR PV" - # landlord_property_id = "UPRN" - # landlord_sap = "Current SAP Rating" - # landlord_block_reference = None - # outcomes_filename = [] - # outcomes_sheetname = [] - # outcomes_postcode = [] - # outcomes_houseno = [] - # outcomes_id = [] - # outcomes_address = [] - # master_filepaths = [] - # master_to_asset_list_filepath = None - # master_id_colnames = [] - # phase = False - # ecosurv_landlords = None - - # Thrive - reconciliation - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation" - # data_filename = "Thrive Asset List - Complete - Updated May 2025.xlsx" - # sheet_name = "Sheet1" - # postcode_column = 'postcode' - # fulladdress_column = "full_address" - # address1_column = "address_line_1" - # address1_method = None - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "age_band_calculated" - # landlord_os_uprn = None - # landlord_property_type = "property_type" - # landlord_built_form = "build_form" - # landlord_wall_construction = None - # landlord_roof_construction = "assumed_loft_insulation_thickness_updated" - # landlord_heating_system = "heating_type_updated" - # landlord_existing_pv = None - # landlord_property_id = "thrive_property_id" - # landlord_sap = "sap_rating_updated" - # landlord_block_reference = "block_reference" - # outcomes_filename = [ - # os.path.join(data_folder, "Thrive - Outcomes - April 24-March25 - Corrected.xlsx") - # ] - # outcomes_sheetname = ["Sheet1"] - # outcomes_postcode = ["postcode"] - # outcomes_houseno = ["No."] - # outcomes_id = ["thrive_property_id"] - # outcomes_address = ["address"] - # master_filepaths = [ - # os.path.join(data_folder, "Thrive Submissions ECO3 - with IDS.csv"), - # os.path.join(data_folder, "Thrive Submissions ECO4 - with IDS.csv"), - # ] - # master_to_asset_list_filepath = None - # master_id_colnames = ["thrive_property_id", "thrive_property_id"] - # phase = False - # ecosurv_landlords = "thrive" - - # Southern Midlands - # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/Midlands Properties - Apr 2025" - # data_filename = "Southern Housing Midlands Property List - combined.xlsx" - # sheet_name = "Sheet 1" - # postcode_column = 'Post Code' - # fulladdress_column = "Address" - # address1_column = None - # address1_method = "house_number_extraction" - # address_cols_to_concat = [] - # missing_postcodes_method = None - # landlord_year_built = "Age_1" - # landlord_os_uprn = None - # landlord_property_type = "Prop_Type" - # landlord_built_form = "Prop_Type" - # landlord_wall_construction = "Walls_P" - # landlord_heating_system = "Heating System" - # landlord_existing_pv = None - # landlord_property_id = "AssetID" - # outcomes_filename = None - # outcomes_sheetname = None - # outcomes_postcode = None - # outcomes_houseno = None - # outcomes_id = None - # outcomes_address = None - # master_filepaths = [] - # master_to_asset_list_filepath = None - # Maps addresses to uprn in problematic cases manual_uprn_map = {}