From 05027c332899a7738230db9b0d8c22edfb796ade Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 10:54:35 +0000 Subject: [PATCH 1/6] save --- .devcontainer/docker-compose.yml | 1 - backend/src/dashboard/services/hubspot_client.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index fc12511..ddc1713 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,6 +1,5 @@ version: '3.8' - services: insight: init: true diff --git a/backend/src/dashboard/services/hubspot_client.py b/backend/src/dashboard/services/hubspot_client.py index 20277b2..12dda92 100644 --- a/backend/src/dashboard/services/hubspot_client.py +++ b/backend/src/dashboard/services/hubspot_client.py @@ -129,6 +129,9 @@ class HubSpotClient(): 'design_planned_week', 'retrofit_design_status', 'design_completion_date', + 'item_id__monday_com_', + 'funding_type', + 'coordination_status__stage_1_', ] ) @@ -258,7 +261,7 @@ class HubSpotClient(): "amount", "hs_product_id", "invoice_reference", - "invoiced" + "invoiced", ] ) line_items.append(item.properties) From 1adf7f6950f870091da44a8bb1db65a5c7c2d44e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 12:54:05 +0000 Subject: [PATCH 2/6] wrong code --- backend/src/dashboard/main.py | 26 ++++++++++++------- .../services/hubspot_client_async.py | 3 +++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/backend/src/dashboard/main.py b/backend/src/dashboard/main.py index bc6e8f7..66f3af4 100644 --- a/backend/src/dashboard/main.py +++ b/backend/src/dashboard/main.py @@ -3,15 +3,17 @@ import dash_bootstrap_components as dbc import pandas as pd from datetime import datetime, timedelta from dash import ctx +import json +import os -from backend.src.dashboard.services.file_manager import FileManager -from backend.src.dashboard.services.json_reader import jsonReader -from backend.src.dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday +# from backend.src.dashboard.services.file_manager import FileManager +# from backend.src.dashboard.services.json_reader import jsonReader +# from backend.src.dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday -# from dashboard.services.file_manager import FileManager -# from dashboard.services.json_reader import jsonReader -# from dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday +from dashboard.services.file_manager import FileManager +from dashboard.services.json_reader import jsonReader +from dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday SAFE_DELIM = "\\\\" @@ -27,9 +29,15 @@ def current_week_start(): # ----------------------------------------------------- # Load & Build Master DF # ----------------------------------------------------- -def build_master_df(): - s3 = FileManager() - key, path, data = s3.download_and_read_latest() +def build_master_df(local=False): + local=True + if local is False: + s3 = FileManager() + key, path, data = s3.download_and_read_latest() + else: + file_path = os.path.join(os.path.dirname(__file__), "data.json") + with open(file_path, "r") as f: + data = json.load(f) hubspot_data = jsonReader(data) frames = [] diff --git a/backend/src/dashboard/services/hubspot_client_async.py b/backend/src/dashboard/services/hubspot_client_async.py index ccafdea..e5c972a 100644 --- a/backend/src/dashboard/services/hubspot_client_async.py +++ b/backend/src/dashboard/services/hubspot_client_async.py @@ -124,6 +124,9 @@ class HubSpotClientAsync: 'design_planned_week', 'retrofit_design_status', 'design_completion_date', + 'item_id__monday_com_', + 'funding_type', + 'coordination_status__stage_1_', ] ) From 1f1cb1f262cc4613ecde7bb5371130c1a30c7769 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 14:55:35 +0000 Subject: [PATCH 3/6] do it proper --- .github/workflows/gather_hubspot_data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gather_hubspot_data.yml b/.github/workflows/gather_hubspot_data.yml index 2762151..038e474 100644 --- a/.github/workflows/gather_hubspot_data.yml +++ b/.github/workflows/gather_hubspot_data.yml @@ -9,7 +9,7 @@ jobs: gather_hubspot_data_and_upload_to_s3: runs-on: [self-hosted, mist] - timeout-minutes: 120 # <-- 2 hour timeout + timeout-minutes: 240 steps: From 10512cf8a26ff25f179d813efbf8ad71cb5aebfb Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 17:20:59 +0000 Subject: [PATCH 4/6] no timeout --- .github/workflows/gather_hubspot_data.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/gather_hubspot_data.yml b/.github/workflows/gather_hubspot_data.yml index 038e474..9c24e63 100644 --- a/.github/workflows/gather_hubspot_data.yml +++ b/.github/workflows/gather_hubspot_data.yml @@ -8,10 +8,6 @@ on: jobs: gather_hubspot_data_and_upload_to_s3: runs-on: [self-hosted, mist] - - timeout-minutes: 240 - - steps: - uses: actions/checkout@v4 From c9940609163a7ac2be797e396c613439fb28cc56 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 22:26:35 +0000 Subject: [PATCH 5/6] add code to main --- backend/src/dashboard/services/json_reader.py | 149 ++++++++++-------- 1 file changed, 86 insertions(+), 63 deletions(-) diff --git a/backend/src/dashboard/services/json_reader.py b/backend/src/dashboard/services/json_reader.py index 9eb1539..94f83d3 100644 --- a/backend/src/dashboard/services/json_reader.py +++ b/backend/src/dashboard/services/json_reader.py @@ -61,75 +61,62 @@ class jsonReader: def _return_df_from_deal_info(self, deal, product_type): rows = [] if deal["company_info"]["name"] != "Apple": - if deal["attempts"]: - # Multiple attempts => multiple rows - for attempt in deal["attempts"]: - rows.append({ - "submission_date": self.to_date_only(attempt["submission_date"]), - "hubspot_id": deal["deal_properties"]["deal_id"], - "expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]), - "work_type": product_type, - "price": next( - (item["price"] for item in deal["line_items"] if product_type in item["name"]), - None - ), - "deal_name": deal["deal_properties"]["dealname"], - "company_name": deal["company_info"]["name"], - }) - else: - def historical_ecd_value_processes(timestamp): - if timestamp is None or timestamp == '': - return None - dt = datetime.strptime(timestamp, "%Y-%m-%d") - return dt.strftime("%Y-%m-%d") - history = deal["deal_properties"]["expected_commencement_history"] + if deal["attempts"]: + # Multiple attempts => multiple rows + for attempt in deal["attempts"]: + data = { + "submission_date": self.to_date_only(attempt["submission_date"]), + "hubspot_id": deal["deal_properties"]["deal_id"], + "expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]), + "work_type": product_type, + "price": next( + (item["price"] for item in deal["line_items"] if product_type in item["name"]), + None + ), + "deal_name": deal["deal_properties"]["dealname"], + "company_name": deal["company_info"]["name"], + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + else: + def historical_ecd_value_processes(timestamp): + if timestamp is None or timestamp == '': + return None + dt = datetime.strptime(timestamp, "%Y-%m-%d") + return dt.strftime("%Y-%m-%d") + history = deal["deal_properties"]["expected_commencement_history"] - # ---- SORT HISTORY: latest first ---- - history_sorted = sorted( - history, - key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"), - reverse=True - ) + # ---- SORT HISTORY: latest first ---- + history_sorted = sorted( + history, + key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"), + reverse=True + ) - # Extract latest expected commencement date - if history_sorted: - latest = history_sorted[0] - latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None + # Extract latest expected commencement date + if history_sorted: + latest = history_sorted[0] + latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None - # Convert submission date - raw_submission_date = deal["deal_properties"].get("last_submission_date") - submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None + # Convert submission date + raw_submission_date = deal["deal_properties"].get("last_submission_date") + submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None - # Convert both to datetime for comparison - if submission_date and latest_ecd: - dt_sub = datetime.strptime(submission_date, "%Y-%m-%d") - dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d") + # Convert both to datetime for comparison + if submission_date and latest_ecd: + dt_sub = datetime.strptime(submission_date, "%Y-%m-%d") + dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d") - # Only keep submission date if submission_date > latest ECD - if dt_sub <= dt_ecd: + # Only keep submission date if submission_date > latest ECD + if dt_sub <= dt_ecd: + submission_date = None + else: submission_date = None - else: - submission_date = None - # 1️⃣ Add latest expected commencement date WITH conditional submission date - rows.append({ - "submission_date": submission_date, - "expected_commencement_date": latest_ecd, - "hubspot_id": deal["deal_properties"]["deal_id"], - "work_type": product_type, - "price": next( - (item["price"] for item in deal["line_items"] if product_type in item["name"]), - None - ), - "deal_name": deal["deal_properties"]["dealname"], - "company_name": deal["company_info"]["name"], - }) - - # 2️⃣ Add the remaining history WITHOUT submission date - for attempt in history_sorted[1:]: - rows.append({ - "submission_date": None, - "expected_commencement_date": historical_ecd_value_processes(attempt["value"]), + # 1️⃣ Add latest expected commencement date WITH conditional submission date + data = { + "submission_date": submission_date, + "expected_commencement_date": latest_ecd, "hubspot_id": deal["deal_properties"]["deal_id"], "work_type": product_type, "price": next( @@ -138,10 +125,46 @@ class jsonReader: ), "deal_name": deal["deal_properties"]["dealname"], "company_name": deal["company_info"]["name"], - }) + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + + # 2️⃣ Add the remaining history WITHOUT submission date + for attempt in history_sorted[1:]: + data = { + "submission_date": None, + "expected_commencement_date": historical_ecd_value_processes(attempt["value"]), + "hubspot_id": deal["deal_properties"]["deal_id"], + "work_type": product_type, + "price": next( + (item["price"] for item in deal["line_items"] if product_type in item["name"]), + None + ), + "deal_name": deal["deal_properties"]["dealname"], + "company_name": deal["company_info"]["name"], + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + + + # Return a DataFrame or None return pd.DataFrame(rows) if rows else None + + def _use_different_expected_commencement_data(self, org_data, deal): + work_type = org_data['work_type'].lower() + if "Coordination Stage".lower() in work_type: + org_data.update({ + "expected_commencement_date": self.to_date_only(deal["deal_properties"]["mtp_planned_week"]), + "submission_date": self.to_date_only(deal["deal_properties"]["mtp_completion_date"]), + }) + elif "Design".lower() in work_type: + org_data.update({ + "expected_commencement_date": self.to_date_only(deal["deal_properties"]["design_planned_week"]), + "submission_date": self.to_date_only(deal["deal_properties"]["design_completion_date"]), + }) + return org_data def find_all_job_with_line_item(self): for i, deal in enumerate(self.raw_data): From 26d5357b0854472162feb617ece5e6ecadbf4118 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 3 Dec 2025 22:29:38 +0000 Subject: [PATCH 6/6] add code to main --- backend/src/dashboard/main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/src/dashboard/main.py b/backend/src/dashboard/main.py index 66f3af4..6e51f13 100644 --- a/backend/src/dashboard/main.py +++ b/backend/src/dashboard/main.py @@ -7,13 +7,13 @@ import json import os -# from backend.src.dashboard.services.file_manager import FileManager -# from backend.src.dashboard.services.json_reader import jsonReader -# from backend.src.dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday +from backend.src.dashboard.services.file_manager import FileManager +from backend.src.dashboard.services.json_reader import jsonReader +from backend.src.dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday -from dashboard.services.file_manager import FileManager -from dashboard.services.json_reader import jsonReader -from dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday +# from dashboard.services.file_manager import FileManager +# from dashboard.services.json_reader import jsonReader +# from dashboard.components.pivot_charts import build_pivot_tables_and_charts, week_start_monday SAFE_DELIM = "\\\\" @@ -30,7 +30,6 @@ def current_week_start(): # Load & Build Master DF # ----------------------------------------------------- def build_master_df(local=False): - local=True if local is False: s3 = FileManager() key, path, data = s3.download_and_read_latest()