Merge branch 'main' into feature/ecmk-to-ara

backend/pashub_fetcher/handler/handler.py

@@ -20,6 +20,7 @@ from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
 )
 from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
 from backend.pashub_fetcher.token_getter import get_token_from_local_storage
+from backend.utils.subtasks import task_handler
 from utils.logger import setup_logger
 from utils.s3 import upload_file_to_s3
 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
@@ -165,6 +166,7 @@ def process_job(
     return job_files
 
 
+@task_handler()
 def handler(event: Mapping[str, Any], context: Any) -> None:
     logger.info("Received message")
     logger.info(f"Number of events: {len(event.get('Records', []))}")

etl/bill_savings/KwhData.py

@@ -77,14 +77,8 @@ class KwhData:
                   'Cheapest tariff (Large legacy suppliers)', 'Cheapest tariff (All suppliers)',
                   'Cheapest tariff (Basket)', 'Default tariff cap level']
 
-        # Extract data rows
-        data_rows = []
-        for row in data[1:]:
-            date = row['\ufeff"']
-            values = row[None]
-            data_rows.append([date] + values)
-
-        self.retail_price_comparison = pd.DataFrame(data_rows, columns=header)
+        self.retail_price_comparison = pd.DataFrame(data)
+        self.retail_price_comparison.columns = header
         self.retail_price_comparison['Date'] = pd.to_datetime(self.retail_price_comparison['Date'], errors='coerce')
 
     @staticmethod

etl/hubspot/scripts/scraper/main.py

@@ -22,7 +22,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
     hubspot_client = HubspotClient()
 
     sqs_client = boto3.client("sqs")
-    PASHUB_TRIGGER_QUEUE_URL = get_settings().PASHUB_TO_ARA_SQS_URL
 
     payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body)
     hubspot_deal_id: str = payload.hubspot_deal_id
@@ -41,7 +40,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
 
     deal_changed = False
     if not db_deal:
-        # New hubspot deal, no diffing to do
         logger.info(f"New HubSpot deal of ID {hubspot_deal_id}. Loading to database...")
         if company:
             company_data: CompanyData = hubspot_client.get_company_information(company)
@@ -49,6 +47,15 @@ def handler(body: dict[str, Any], context: Any) -> None:
             db_client.upsert_organisation(company_data)
 
         db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
+
+        # ==============================
+        # Orchestration of other lambdas
+        # ==============================
+        if hubspot_deal["pashub_link"]:
+            logger.info(
+                f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
+            )
+            _trigger_pashub_fetcher(sqs_client, hubspot_deal)
     else:
         # Deal already in db, check whether anything has changed
         logger.info(
@@ -84,22 +91,7 @@ def handler(body: dict[str, Any], context: Any) -> None:
             logger.info(
                 f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
             )
-            message_body: Dict[str, Optional[str]] = {
-                "pashub_link": hubspot_deal["pashub_link"],
-                "address": None,  # potentially available from Listing, leave as None for now
-                "sharepoint_link": hubspot_deal["sharepoint_link"],
-                "uprn": hubspot_deal["national_uprn"],
-                "landlord_property_id": hubspot_deal["owner_property_id"],
-                "deal_stage": hubspot_deal["deal_stage"],
-            }
-
-            response = sqs_client.send_message(
-                QueueUrl=PASHUB_TRIGGER_QUEUE_URL, MessageBody=json.dumps(message_body)
-            )
-
-            logger.info(
-                f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
-            )
+            _trigger_pashub_fetcher(sqs_client, hubspot_deal)
         else:
             logger.info(
                 f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}"
@@ -108,6 +100,26 @@ def handler(body: dict[str, Any], context: Any) -> None:
     print("done")
 
 
+def _trigger_pashub_fetcher(sqs_client: Any, hubspot_deal: Dict[str, str]) -> None:
+    message_body: Dict[str, Optional[str]] = {
+        "pashub_link": hubspot_deal["pashub_link"],
+        "address": None,  # potentially available from Listing, leave as None for now
+        "sharepoint_link": hubspot_deal["sharepoint_link"],
+        "uprn": hubspot_deal["national_uprn"],
+        "landlord_property_id": hubspot_deal["owner_property_id"],
+        "deal_stage": hubspot_deal["deal_stage"],
+    }
+
+    response = sqs_client.send_message(
+        QueueUrl=get_settings().PASHUB_TO_ARA_SQS_URL,
+        MessageBody=json.dumps(message_body),
+    )
+
+    logger.info(
+        f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
+    )
+
+
 if __name__ == "__main__":
     handler({"hubspot_deal_id": "371470706915"}, "")
     print("beep")

sfr/principal_pitch/2_export_data.py

@@ -230,7 +230,7 @@ for scenario_id in SCENARIOS:
     # Get recs for this scenario
     recommended_measures_df = recommendations_df[
         recommendations_df["scenario_id"] == scenario_id
-    ][["property_id", "measure_type", "estimated_cost", "default"]]
+        ][["property_id", "measure_type", "estimated_cost", "default"]]
     recommended_measures_df = recommended_measures_df[
         recommended_measures_df["default"]
     ]
@@ -238,7 +238,7 @@ for scenario_id in SCENARIOS:
 
     post_install_sap = recommendations_df[
         recommendations_df["scenario_id"] == scenario_id
-    ][["property_id", "default", "sap_points"]]
+        ][["property_id", "default", "sap_points"]]
     post_install_sap = post_install_sap[post_install_sap["default"]]
     # Sum up the sap points by property id
     post_install_sap = (
@@ -282,6 +282,7 @@ for scenario_id in SCENARIOS:
                 "windows",
                 "current_epc_rating",
                 "current_sap_points",
+                "original_sap_points",
                 "total_floor_area",
                 "number_of_rooms",
                 "lodgement_date",
@@ -303,31 +304,6 @@ for scenario_id in SCENARIOS:
     )
     df["uprn"] = df["uprn"].astype(str)
 
-    relevant_plans = plans_df[plans_df["scenario_id"] == scenario_id]
-    df2 = df.merge(
-        relevant_plans[["property_id", "post_sap_points", "post_epc_rating"]],
-        how="left",
-        on="property_id",
-        suffixes=("", "_plan"),
-    )
-    print(df2["predicted_post_works_epc"].value_counts())
-    print(df2["post_epc_rating"].value_counts())
-
-    z = df2[
-        (df2["predicted_post_works_epc"] != "D")
-        & (df2["post_epc_rating"].astype(str) == "Epc.D")
-    ]
-
-    df2["predicted_post_works_epc"].value_counts()
-    df2["post_epc_rating"].astype(str).value_counts()
-
-    df2[df2["total_retrofit_cost"] > 0].shape
-
-    getting_works = df[df["total_retrofit_cost"] > 0]
-    getting_works["predicted_post_works_epc"].value_counts()
-
-    df[df["predicted_post_works_sap"] == ""]
-
     # Expected columns list
     expected_columns = [
         "suspended_floor_insulation",