added local devconaitner

asset_list/app.py

@@ -74,23 +74,23 @@ def app():
     """
 
     data_folder = "/workspaces/model/asset_list"
-    data_filename = "Waverley UPRN Match.xlsx"
+    data_filename = "foom (2).xlsx"
     sheet_name = "in"
     postcode_column = "postcode_clean"
-    address1_column = "domna_found_address"
+    address1_column = "address2uprn_address"
     address1_method = None
-    fulladdress_column = "domna_found_address"
+    fulladdress_column = "address2uprn_address"
     address_cols_to_concat = []
     missing_postcodes_method = None
     landlord_year_built = None
-    landlord_os_uprn = "domna_found_uprn"
-    landlord_property_type = "Property Type 1"  # Good to include if landlord gave
+    landlord_os_uprn = "address2uprn_uprn"
+    landlord_property_type = None  # Good to include if landlord gave
     landlord_built_form = None  # Good to include if landlord gave
     landlord_wall_construction = None
     landlord_roof_construction = None
     landlord_heating_system = None
     landlord_existing_pv = None
-    landlord_property_id = "WBC Ref"
+    landlord_property_id = "UPRN"
     landlord_sap = None
     outcomes_filename = None
     outcomes_sheetname = None

backend/export/property_scenarios/main.py

@@ -26,15 +26,14 @@ def has_solar_with_battery(materials_list: Optional[List[Dict[str, Any]]]) -> bo
     :return:
     """
     for m in materials_list or []:
-        if (
-            m.get("type") == "solar_pv"
-            and m.get("includes_battery") is True
-        ):
+        if m.get("type") == "solar_pv" and m.get("includes_battery") is True:
             return True
     return False
 
 
-def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]:
+def process_export(
+    payload: ExportRequest, session: Session
+) -> Dict[Union[str, int], pd.DataFrame]:
     export_files: Dict[Union[str, int], pd.DataFrame] = {}
 
     db_methods = DbMethods(session)
@@ -52,7 +51,9 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
     logger.info("Retrieved %s plans for export", len(plans_df))
 
     if plans_df.empty:
-        logger.info("Empty plans dataframe - no plans to export. Returning empty export.")
+        logger.info(
+            "Empty plans dataframe - no plans to export. Returning empty export."
+        )
         return export_files
     plan_ids: List[int] = plans_df["id"].tolist()
     recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids)
@@ -61,13 +62,12 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
 
     recommendations_df = db_methods.attach_materials(recommendations_df)
 
-    recommendations_df["has_solar_with_battery"] = (
-        recommendations_df["materials"].apply(has_solar_with_battery)
-    )
+    recommendations_df["has_solar_with_battery"] = recommendations_df[
+        "materials"
+    ].apply(has_solar_with_battery)
 
-    _filter = (
-        (recommendations_df["measure_type"] == "solar_pv")
-        & (recommendations_df["has_solar_with_battery"])
+    _filter = (recommendations_df["measure_type"] == "solar_pv") & (
+        recommendations_df["has_solar_with_battery"]
     )
 
     recommendations_df.loc[_filter, "measure_type"] = (
@@ -83,10 +83,13 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
         else:
             scenario_recs = recommendations_df[
                 recommendations_df["scenario_id"] == group_key
-                ]
+            ]
 
         if scenario_recs.empty:
-            logger.info("No recommendations found for group_key %s - skipping export for this group", group_key)
+            logger.info(
+                "No recommendations found for group_key %s - skipping export for this group",
+                group_key,
+            )
             continue
 
         measures_df: pd.DataFrame = scenario_recs[
@@ -99,14 +102,12 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
             values="estimated_cost",
         ).reset_index()
 
-        pivot["total_retrofit_cost"] = (
-            pivot.drop(columns=["property_id", "plan_name"]).sum(axis=1)
-        )
+        pivot["total_retrofit_cost"] = pivot.drop(
+            columns=["property_id", "plan_name"]
+        ).sum(axis=1)
 
         post_sap: pd.DataFrame = (
-            scenario_recs.groupby("property_id")[["sap_points"]]
-            .sum()
-            .reset_index()
+            scenario_recs.groupby("property_id")[["sap_points"]].sum().reset_index()
         )
 
         df: pd.DataFrame = (
@@ -117,7 +118,9 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
 
         df["sap_points"] = df["sap_points"].fillna(0)
         df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
-        df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc)
+        df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(
+            sap_to_epc
+        )
 
         export_files[group_key] = df
 
@@ -128,22 +131,17 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
 # Lambda Handler
 # ============================================================
 
-def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Union[int, str]]:
+
+def handler(
+    event: Mapping[str, Any], context: Optional[Any]
+) -> Mapping[str, Union[int, str]]:
     """
     Example event:
     body_dict = {
         "task_id": "test",
         "subtask_id": "test",
-        "portfolio_id": 655,
-        "scenario_ids": [],
-        "default_plans_only": True,
-    }
-
-    body_dict = {
-        "task_id": "test",
-        "subtask_id": "test",
-        "portfolio_id": 655,
-        "scenario_ids": [1174],
+        "portfolio_id": 682,
+        "scenario_ids": [1210],
         "default_plans_only": False,
     }
     :param event: Lambda event containing export request details
@@ -168,7 +166,12 @@ def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Un
                 exported_files = process_export(payload, session)
 
             # TODO: Need to handle the exported files - e.g. upload to s3 and email a presigned url
-            _ = exported_files
+            output_path = f"/tmp/export_{payload.portfolio_id}.xlsx"
+            with pd.ExcelWriter(output_path, engine="openpyxl") as writer:
+                for group_key, df in exported_files.items():
+                    sheet_name = str(group_key)[:31]  # Excel sheet names max 31 chars
+                    df.to_excel(writer, sheet_name=sheet_name, index=False)
+            logger.info("Exported files written to %s", output_path)
             return {
                 "statusCode": 200,
                 "body": json.dumps({}),

backend/scripts/combine_address2uprn_outputs.py

@@ -31,6 +31,8 @@ def download_csv(key):
 
 
 def main(task_id, output):
+    task_id = "3fb9a9b7-ff49-4c11-b9e1-9d00da955a75"
+
     print(f"Scanning task: {task_id}")
 
     csv_files = list_csv_files(task_id)

devcontainer.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+#
+# dc.sh — devcontainer helper for this repo
+#
+# Usage:
+#   devcontainer.sh <config> <command>
+#
+# Configs:   backend | asset_list
+# Commands:  up, shell, down, rebuild
+#
+# `shell` auto-runs `up` first if the container isn't already running,
+# so it's safe to call cold.
+#
+# Examples:
+#   ./scripts/dc.sh backend shell       # up + exec bash
+#   ./scripts/dc.sh asset_list up
+#   ./scripts/dc.sh backend rebuild
+#   ./scripts/dc.sh backend down
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." &>/dev/null && pwd)"
+
+VALID_CONFIGS=(backend asset_list)
+VALID_COMMANDS=(up shell down rebuild)
+
+# --- helpers ---------------------------------------------------------------
+
+usage() {
+  sed -n '3,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
+  exit "${1:-0}"
+}
+
+die() {
+  echo "error: $*" >&2
+  exit 1
+}
+
+in_list() {
+  # in_list <needle> <haystack...>
+  local needle="$1"
+  shift
+  local item
+  for item in "$@"; do
+    [[ "${item}" == "${needle}" ]] && return 0
+  done
+  return 1
+}
+
+container_id_for() {
+  # Find a running container for the given config path via devcontainer labels.
+  local config_path="$1"
+  docker ps -q \
+    --filter "label=devcontainer.local_folder=${REPO_ROOT}" \
+    --filter "label=devcontainer.config_file=${config_path}"
+}
+
+# --- argument parsing ------------------------------------------------------
+
+[[ $# -eq 2 ]] || usage 1
+
+CONFIG_NAME="$1"
+COMMAND="$2"
+
+in_list "${CONFIG_NAME}" "${VALID_CONFIGS[@]}" \
+  || die "invalid config '${CONFIG_NAME}' (expected: ${VALID_CONFIGS[*]})"
+in_list "${COMMAND}" "${VALID_COMMANDS[@]}" \
+  || die "invalid command '${COMMAND}' (expected: ${VALID_COMMANDS[*]})"
+
+CONFIG_PATH="${REPO_ROOT}/.devcontainer/${CONFIG_NAME}/devcontainer.json"
+[[ -f "${CONFIG_PATH}" ]] || die "config not found: ${CONFIG_PATH}"
+
+DC_ARGS=(--workspace-folder "${REPO_ROOT}" --config "${CONFIG_PATH}")
+
+# --- dispatch --------------------------------------------------------------
+
+case "${COMMAND}" in
+  up)
+    echo ">> bringing up '${CONFIG_NAME}'"
+    devcontainer up "${DC_ARGS[@]}"
+    ;;
+
+  shell)
+    # Auto-up if not already running. `devcontainer up` is idempotent —
+    # it reuses an existing container, so this is cheap on warm starts.
+    if [[ -z "$(container_id_for "${CONFIG_PATH}")" ]]; then
+      echo ">> '${CONFIG_NAME}' not running, bringing it up first"
+      devcontainer up "${DC_ARGS[@]}"
+    fi
+    echo ">> attaching shell to '${CONFIG_NAME}'"
+    devcontainer exec "${DC_ARGS[@]}" bash 2>/dev/null \
+      || devcontainer exec "${DC_ARGS[@]}" sh
+    ;;
+
+  down)
+    cid="$(container_id_for "${CONFIG_PATH}")"
+    if [[ -z "${cid}" ]]; then
+      echo ">> '${CONFIG_NAME}' not running, nothing to stop"
+      exit 0
+    fi
+    echo ">> stopping '${CONFIG_NAME}'"
+    docker stop "${cid}"
+    ;;
+
+  rebuild)
+    echo ">> rebuilding '${CONFIG_NAME}' from scratch"
+    devcontainer up "${DC_ARGS[@]}" --remove-existing-container --build-no-cache
+    ;;
+esac

etl/hubspot/scripts/scraper/bulk_load.py

@@ -9,8 +9,8 @@ PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
 companies = list(
     [
         # Companies.THE_GUINESS_PARTNERSHIP,
-        # Companies.SOUTHERN_HOUSING_GROUP,
-        Companies.CALICO_HOMES,
+        Companies.SOUTHERN_HOUSING_GROUP,
+        # Companies.CALICO_HOMES,
     ]
 )