diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 793b9d70..3689efe9 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -20,6 +20,7 @@ from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( ) from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from backend.utils.subtasks import task_handler from utils.logger import setup_logger from utils.s3 import upload_file_to_s3 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient @@ -165,6 +166,7 @@ def process_job( return job_files +@task_handler() def handler(event: Mapping[str, Any], context: Any) -> None: logger.info("Received message") logger.info(f"Number of events: {len(event.get('Records', []))}") diff --git a/etl/bill_savings/KwhData.py b/etl/bill_savings/KwhData.py index 30e11698..e815a12b 100644 --- a/etl/bill_savings/KwhData.py +++ b/etl/bill_savings/KwhData.py @@ -77,14 +77,8 @@ class KwhData: 'Cheapest tariff (Large legacy suppliers)', 'Cheapest tariff (All suppliers)', 'Cheapest tariff (Basket)', 'Default tariff cap level'] - # Extract data rows - data_rows = [] - for row in data[1:]: - date = row['\ufeff"'] - values = row[None] - data_rows.append([date] + values) - - self.retail_price_comparison = pd.DataFrame(data_rows, columns=header) + self.retail_price_comparison = pd.DataFrame(data) + self.retail_price_comparison.columns = header self.retail_price_comparison['Date'] = pd.to_datetime(self.retail_price_comparison['Date'], errors='coerce') @staticmethod diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index d754cbb1..e6c788ea 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -22,7 +22,6 @@ def handler(body: dict[str, Any], context: Any) -> None: hubspot_client = HubspotClient() sqs_client = boto3.client("sqs") - PASHUB_TRIGGER_QUEUE_URL = get_settings().PASHUB_TO_ARA_SQS_URL payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body) hubspot_deal_id: str = payload.hubspot_deal_id @@ -41,7 +40,6 @@ def handler(body: dict[str, Any], context: Any) -> None: deal_changed = False if not db_deal: - # New hubspot deal, no diffing to do logger.info(f"New HubSpot deal of ID {hubspot_deal_id}. Loading to database...") if company: company_data: CompanyData = hubspot_client.get_company_information(company) @@ -49,6 +47,15 @@ def handler(body: dict[str, Any], context: Any) -> None: db_client.upsert_organisation(company_data) db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) + + # ============================== + # Orchestration of other lambdas + # ============================== + if hubspot_deal["pashub_link"]: + logger.info( + f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_pashub_fetcher(sqs_client, hubspot_deal) else: # Deal already in db, check whether anything has changed logger.info( @@ -84,22 +91,7 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) - message_body: Dict[str, Optional[str]] = { - "pashub_link": hubspot_deal["pashub_link"], - "address": None, # potentially available from Listing, leave as None for now - "sharepoint_link": hubspot_deal["sharepoint_link"], - "uprn": hubspot_deal["national_uprn"], - "landlord_property_id": hubspot_deal["owner_property_id"], - "deal_stage": hubspot_deal["deal_stage"], - } - - response = sqs_client.send_message( - QueueUrl=PASHUB_TRIGGER_QUEUE_URL, MessageBody=json.dumps(message_body) - ) - - logger.info( - f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}" - ) + _trigger_pashub_fetcher(sqs_client, hubspot_deal) else: logger.info( f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}" @@ -108,6 +100,26 @@ def handler(body: dict[str, Any], context: Any) -> None: print("done") +def _trigger_pashub_fetcher(sqs_client: Any, hubspot_deal: Dict[str, str]) -> None: + message_body: Dict[str, Optional[str]] = { + "pashub_link": hubspot_deal["pashub_link"], + "address": None, # potentially available from Listing, leave as None for now + "sharepoint_link": hubspot_deal["sharepoint_link"], + "uprn": hubspot_deal["national_uprn"], + "landlord_property_id": hubspot_deal["owner_property_id"], + "deal_stage": hubspot_deal["deal_stage"], + } + + response = sqs_client.send_message( + QueueUrl=get_settings().PASHUB_TO_ARA_SQS_URL, + MessageBody=json.dumps(message_body), + ) + + logger.info( + f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}" + ) + + if __name__ == "__main__": handler({"hubspot_deal_id": "371470706915"}, "") print("beep") diff --git a/sfr/principal_pitch/2_export_data.py b/sfr/principal_pitch/2_export_data.py index 7c80f4dc..06727f86 100644 --- a/sfr/principal_pitch/2_export_data.py +++ b/sfr/principal_pitch/2_export_data.py @@ -230,7 +230,7 @@ for scenario_id in SCENARIOS: # Get recs for this scenario recommended_measures_df = recommendations_df[ recommendations_df["scenario_id"] == scenario_id - ][["property_id", "measure_type", "estimated_cost", "default"]] + ][["property_id", "measure_type", "estimated_cost", "default"]] recommended_measures_df = recommended_measures_df[ recommended_measures_df["default"] ] @@ -238,7 +238,7 @@ for scenario_id in SCENARIOS: post_install_sap = recommendations_df[ recommendations_df["scenario_id"] == scenario_id - ][["property_id", "default", "sap_points"]] + ][["property_id", "default", "sap_points"]] post_install_sap = post_install_sap[post_install_sap["default"]] # Sum up the sap points by property id post_install_sap = ( @@ -282,6 +282,7 @@ for scenario_id in SCENARIOS: "windows", "current_epc_rating", "current_sap_points", + "original_sap_points", "total_floor_area", "number_of_rooms", "lodgement_date", @@ -303,31 +304,6 @@ for scenario_id in SCENARIOS: ) df["uprn"] = df["uprn"].astype(str) - relevant_plans = plans_df[plans_df["scenario_id"] == scenario_id] - df2 = df.merge( - relevant_plans[["property_id", "post_sap_points", "post_epc_rating"]], - how="left", - on="property_id", - suffixes=("", "_plan"), - ) - print(df2["predicted_post_works_epc"].value_counts()) - print(df2["post_epc_rating"].value_counts()) - - z = df2[ - (df2["predicted_post_works_epc"] != "D") - & (df2["post_epc_rating"].astype(str) == "Epc.D") - ] - - df2["predicted_post_works_epc"].value_counts() - df2["post_epc_rating"].astype(str).value_counts() - - df2[df2["total_retrofit_cost"] > 0].shape - - getting_works = df[df["total_retrofit_cost"] > 0] - getting_works["predicted_post_works_epc"].value_counts() - - df[df["predicted_post_works_sap"] == ""] - # Expected columns list expected_columns = [ "suspended_floor_insulation",