From 4dbafe3992a30a652480e3a478a061aa8eec8237 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 16 Apr 2026 22:21:54 +0000 Subject: [PATCH] added local devconaitner --- asset_list/app.py | 12 +- backend/export/property_scenarios/main.py | 69 +++++------ .../scripts/combine_address2uprn_outputs.py | 2 + devcontainer.sh | 110 ++++++++++++++++++ etl/hubspot/scripts/scraper/bulk_load.py | 4 +- 5 files changed, 156 insertions(+), 41 deletions(-) create mode 100644 devcontainer.sh diff --git a/asset_list/app.py b/asset_list/app.py index b0030667..25c72bda 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -74,23 +74,23 @@ def app(): """ data_folder = "/workspaces/model/asset_list" - data_filename = "Waverley UPRN Match.xlsx" + data_filename = "foom (2).xlsx" sheet_name = "in" postcode_column = "postcode_clean" - address1_column = "domna_found_address" + address1_column = "address2uprn_address" address1_method = None - fulladdress_column = "domna_found_address" + fulladdress_column = "address2uprn_address" address_cols_to_concat = [] missing_postcodes_method = None landlord_year_built = None - landlord_os_uprn = "domna_found_uprn" - landlord_property_type = "Property Type 1" # Good to include if landlord gave + landlord_os_uprn = "address2uprn_uprn" + landlord_property_type = None # Good to include if landlord gave landlord_built_form = None # Good to include if landlord gave landlord_wall_construction = None landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "WBC Ref" + landlord_property_id = "UPRN" landlord_sap = None outcomes_filename = None outcomes_sheetname = None diff --git a/backend/export/property_scenarios/main.py b/backend/export/property_scenarios/main.py index f3ea0100..64627e01 100644 --- a/backend/export/property_scenarios/main.py +++ b/backend/export/property_scenarios/main.py @@ -26,15 +26,14 @@ def has_solar_with_battery(materials_list: Optional[List[Dict[str, Any]]]) -> bo :return: """ for m in materials_list or []: - if ( - m.get("type") == "solar_pv" - and m.get("includes_battery") is True - ): + if m.get("type") == "solar_pv" and m.get("includes_battery") is True: return True return False -def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]: +def process_export( + payload: ExportRequest, session: Session +) -> Dict[Union[str, int], pd.DataFrame]: export_files: Dict[Union[str, int], pd.DataFrame] = {} db_methods = DbMethods(session) @@ -52,7 +51,9 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, logger.info("Retrieved %s plans for export", len(plans_df)) if plans_df.empty: - logger.info("Empty plans dataframe - no plans to export. Returning empty export.") + logger.info( + "Empty plans dataframe - no plans to export. Returning empty export." + ) return export_files plan_ids: List[int] = plans_df["id"].tolist() recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids) @@ -61,13 +62,12 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, recommendations_df = db_methods.attach_materials(recommendations_df) - recommendations_df["has_solar_with_battery"] = ( - recommendations_df["materials"].apply(has_solar_with_battery) - ) + recommendations_df["has_solar_with_battery"] = recommendations_df[ + "materials" + ].apply(has_solar_with_battery) - _filter = ( - (recommendations_df["measure_type"] == "solar_pv") - & (recommendations_df["has_solar_with_battery"]) + _filter = (recommendations_df["measure_type"] == "solar_pv") & ( + recommendations_df["has_solar_with_battery"] ) recommendations_df.loc[_filter, "measure_type"] = ( @@ -83,10 +83,13 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, else: scenario_recs = recommendations_df[ recommendations_df["scenario_id"] == group_key - ] + ] if scenario_recs.empty: - logger.info("No recommendations found for group_key %s - skipping export for this group", group_key) + logger.info( + "No recommendations found for group_key %s - skipping export for this group", + group_key, + ) continue measures_df: pd.DataFrame = scenario_recs[ @@ -99,14 +102,12 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, values="estimated_cost", ).reset_index() - pivot["total_retrofit_cost"] = ( - pivot.drop(columns=["property_id", "plan_name"]).sum(axis=1) - ) + pivot["total_retrofit_cost"] = pivot.drop( + columns=["property_id", "plan_name"] + ).sum(axis=1) post_sap: pd.DataFrame = ( - scenario_recs.groupby("property_id")[["sap_points"]] - .sum() - .reset_index() + scenario_recs.groupby("property_id")[["sap_points"]].sum().reset_index() ) df: pd.DataFrame = ( @@ -117,7 +118,9 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, df["sap_points"] = df["sap_points"].fillna(0) df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] - df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc) + df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply( + sap_to_epc + ) export_files[group_key] = df @@ -128,22 +131,17 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, # Lambda Handler # ============================================================ -def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Union[int, str]]: + +def handler( + event: Mapping[str, Any], context: Optional[Any] +) -> Mapping[str, Union[int, str]]: """ Example event: body_dict = { "task_id": "test", "subtask_id": "test", - "portfolio_id": 655, - "scenario_ids": [], - "default_plans_only": True, - } - - body_dict = { - "task_id": "test", - "subtask_id": "test", - "portfolio_id": 655, - "scenario_ids": [1174], + "portfolio_id": 682, + "scenario_ids": [1210], "default_plans_only": False, } :param event: Lambda event containing export request details @@ -168,7 +166,12 @@ def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Un exported_files = process_export(payload, session) # TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url - _ = exported_files + output_path = f"/tmp/export_{payload.portfolio_id}.xlsx" + with pd.ExcelWriter(output_path, engine="openpyxl") as writer: + for group_key, df in exported_files.items(): + sheet_name = str(group_key)[:31] # Excel sheet names max 31 chars + df.to_excel(writer, sheet_name=sheet_name, index=False) + logger.info("Exported files written to %s", output_path) return { "statusCode": 200, "body": json.dumps({}), diff --git a/backend/scripts/combine_address2uprn_outputs.py b/backend/scripts/combine_address2uprn_outputs.py index 105b8639..085240a9 100644 --- a/backend/scripts/combine_address2uprn_outputs.py +++ b/backend/scripts/combine_address2uprn_outputs.py @@ -31,6 +31,8 @@ def download_csv(key): def main(task_id, output): + task_id = "3fb9a9b7-ff49-4c11-b9e1-9d00da955a75" + print(f"Scanning task: {task_id}") csv_files = list_csv_files(task_id) diff --git a/devcontainer.sh b/devcontainer.sh new file mode 100644 index 00000000..32908a9e --- /dev/null +++ b/devcontainer.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +# dc.sh — devcontainer helper for this repo +# +# Usage: +# devcontainer.sh +# +# Configs: backend | asset_list +# Commands: up, shell, down, rebuild +# +# `shell` auto-runs `up` first if the container isn't already running, +# so it's safe to call cold. +# +# Examples: +# ./scripts/dc.sh backend shell # up + exec bash +# ./scripts/dc.sh asset_list up +# ./scripts/dc.sh backend rebuild +# ./scripts/dc.sh backend down + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." &>/dev/null && pwd)" + +VALID_CONFIGS=(backend asset_list) +VALID_COMMANDS=(up shell down rebuild) + +# --- helpers --------------------------------------------------------------- + +usage() { + sed -n '3,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +in_list() { + # in_list + local needle="$1" + shift + local item + for item in "$@"; do + [[ "${item}" == "${needle}" ]] && return 0 + done + return 1 +} + +container_id_for() { + # Find a running container for the given config path via devcontainer labels. + local config_path="$1" + docker ps -q \ + --filter "label=devcontainer.local_folder=${REPO_ROOT}" \ + --filter "label=devcontainer.config_file=${config_path}" +} + +# --- argument parsing ------------------------------------------------------ + +[[ $# -eq 2 ]] || usage 1 + +CONFIG_NAME="$1" +COMMAND="$2" + +in_list "${CONFIG_NAME}" "${VALID_CONFIGS[@]}" \ + || die "invalid config '${CONFIG_NAME}' (expected: ${VALID_CONFIGS[*]})" +in_list "${COMMAND}" "${VALID_COMMANDS[@]}" \ + || die "invalid command '${COMMAND}' (expected: ${VALID_COMMANDS[*]})" + +CONFIG_PATH="${REPO_ROOT}/.devcontainer/${CONFIG_NAME}/devcontainer.json" +[[ -f "${CONFIG_PATH}" ]] || die "config not found: ${CONFIG_PATH}" + +DC_ARGS=(--workspace-folder "${REPO_ROOT}" --config "${CONFIG_PATH}") + +# --- dispatch -------------------------------------------------------------- + +case "${COMMAND}" in + up) + echo ">> bringing up '${CONFIG_NAME}'" + devcontainer up "${DC_ARGS[@]}" + ;; + + shell) + # Auto-up if not already running. `devcontainer up` is idempotent — + # it reuses an existing container, so this is cheap on warm starts. + if [[ -z "$(container_id_for "${CONFIG_PATH}")" ]]; then + echo ">> '${CONFIG_NAME}' not running, bringing it up first" + devcontainer up "${DC_ARGS[@]}" + fi + echo ">> attaching shell to '${CONFIG_NAME}'" + devcontainer exec "${DC_ARGS[@]}" bash 2>/dev/null \ + || devcontainer exec "${DC_ARGS[@]}" sh + ;; + + down) + cid="$(container_id_for "${CONFIG_PATH}")" + if [[ -z "${cid}" ]]; then + echo ">> '${CONFIG_NAME}' not running, nothing to stop" + exit 0 + fi + echo ">> stopping '${CONFIG_NAME}'" + docker stop "${cid}" + ;; + + rebuild) + echo ">> rebuilding '${CONFIG_NAME}' from scratch" + devcontainer up "${DC_ARGS[@]}" --remove-existing-container --build-no-cache + ;; +esac diff --git a/etl/hubspot/scripts/scraper/bulk_load.py b/etl/hubspot/scripts/scraper/bulk_load.py index 91aa89e2..f0529905 100644 --- a/etl/hubspot/scripts/scraper/bulk_load.py +++ b/etl/hubspot/scripts/scraper/bulk_load.py @@ -9,8 +9,8 @@ PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value companies = list( [ # Companies.THE_GUINESS_PARTNERSHIP, - # Companies.SOUTHERN_HOUSING_GROUP, - Companies.CALICO_HOMES, + Companies.SOUTHERN_HOUSING_GROUP, + # Companies.CALICO_HOMES, ] )