diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index fc12511..ddc1713 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,6 +1,5 @@ version: '3.8' - services: insight: init: true diff --git a/.github/workflows/gather_hubspot_data.yml b/.github/workflows/gather_hubspot_data.yml index 2762151..9c24e63 100644 --- a/.github/workflows/gather_hubspot_data.yml +++ b/.github/workflows/gather_hubspot_data.yml @@ -8,10 +8,6 @@ on: jobs: gather_hubspot_data_and_upload_to_s3: runs-on: [self-hosted, mist] - - timeout-minutes: 120 # <-- 2 hour timeout - - steps: - uses: actions/checkout@v4 diff --git a/backend/src/dashboard/main.py b/backend/src/dashboard/main.py index bc6e8f7..6e51f13 100644 --- a/backend/src/dashboard/main.py +++ b/backend/src/dashboard/main.py @@ -3,6 +3,8 @@ import dash_bootstrap_components as dbc import pandas as pd from datetime import datetime, timedelta from dash import ctx +import json +import os from backend.src.dashboard.services.file_manager import FileManager @@ -27,9 +29,14 @@ def current_week_start(): # ----------------------------------------------------- # Load & Build Master DF # ----------------------------------------------------- -def build_master_df(): - s3 = FileManager() - key, path, data = s3.download_and_read_latest() +def build_master_df(local=False): + if local is False: + s3 = FileManager() + key, path, data = s3.download_and_read_latest() + else: + file_path = os.path.join(os.path.dirname(__file__), "data.json") + with open(file_path, "r") as f: + data = json.load(f) hubspot_data = jsonReader(data) frames = [] diff --git a/backend/src/dashboard/services/hubspot_client.py b/backend/src/dashboard/services/hubspot_client.py index 20277b2..12dda92 100644 --- a/backend/src/dashboard/services/hubspot_client.py +++ b/backend/src/dashboard/services/hubspot_client.py @@ -129,6 +129,9 @@ class HubSpotClient(): 'design_planned_week', 'retrofit_design_status', 'design_completion_date', + 'item_id__monday_com_', + 'funding_type', + 'coordination_status__stage_1_', ] ) @@ -258,7 +261,7 @@ class HubSpotClient(): "amount", "hs_product_id", "invoice_reference", - "invoiced" + "invoiced", ] ) line_items.append(item.properties) diff --git a/backend/src/dashboard/services/hubspot_client_async.py b/backend/src/dashboard/services/hubspot_client_async.py index ccafdea..e5c972a 100644 --- a/backend/src/dashboard/services/hubspot_client_async.py +++ b/backend/src/dashboard/services/hubspot_client_async.py @@ -124,6 +124,9 @@ class HubSpotClientAsync: 'design_planned_week', 'retrofit_design_status', 'design_completion_date', + 'item_id__monday_com_', + 'funding_type', + 'coordination_status__stage_1_', ] ) diff --git a/backend/src/dashboard/services/json_reader.py b/backend/src/dashboard/services/json_reader.py index 9eb1539..94f83d3 100644 --- a/backend/src/dashboard/services/json_reader.py +++ b/backend/src/dashboard/services/json_reader.py @@ -61,75 +61,62 @@ class jsonReader: def _return_df_from_deal_info(self, deal, product_type): rows = [] if deal["company_info"]["name"] != "Apple": - if deal["attempts"]: - # Multiple attempts => multiple rows - for attempt in deal["attempts"]: - rows.append({ - "submission_date": self.to_date_only(attempt["submission_date"]), - "hubspot_id": deal["deal_properties"]["deal_id"], - "expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]), - "work_type": product_type, - "price": next( - (item["price"] for item in deal["line_items"] if product_type in item["name"]), - None - ), - "deal_name": deal["deal_properties"]["dealname"], - "company_name": deal["company_info"]["name"], - }) - else: - def historical_ecd_value_processes(timestamp): - if timestamp is None or timestamp == '': - return None - dt = datetime.strptime(timestamp, "%Y-%m-%d") - return dt.strftime("%Y-%m-%d") - history = deal["deal_properties"]["expected_commencement_history"] + if deal["attempts"]: + # Multiple attempts => multiple rows + for attempt in deal["attempts"]: + data = { + "submission_date": self.to_date_only(attempt["submission_date"]), + "hubspot_id": deal["deal_properties"]["deal_id"], + "expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]), + "work_type": product_type, + "price": next( + (item["price"] for item in deal["line_items"] if product_type in item["name"]), + None + ), + "deal_name": deal["deal_properties"]["dealname"], + "company_name": deal["company_info"]["name"], + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + else: + def historical_ecd_value_processes(timestamp): + if timestamp is None or timestamp == '': + return None + dt = datetime.strptime(timestamp, "%Y-%m-%d") + return dt.strftime("%Y-%m-%d") + history = deal["deal_properties"]["expected_commencement_history"] - # ---- SORT HISTORY: latest first ---- - history_sorted = sorted( - history, - key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"), - reverse=True - ) + # ---- SORT HISTORY: latest first ---- + history_sorted = sorted( + history, + key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"), + reverse=True + ) - # Extract latest expected commencement date - if history_sorted: - latest = history_sorted[0] - latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None + # Extract latest expected commencement date + if history_sorted: + latest = history_sorted[0] + latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None - # Convert submission date - raw_submission_date = deal["deal_properties"].get("last_submission_date") - submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None + # Convert submission date + raw_submission_date = deal["deal_properties"].get("last_submission_date") + submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None - # Convert both to datetime for comparison - if submission_date and latest_ecd: - dt_sub = datetime.strptime(submission_date, "%Y-%m-%d") - dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d") + # Convert both to datetime for comparison + if submission_date and latest_ecd: + dt_sub = datetime.strptime(submission_date, "%Y-%m-%d") + dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d") - # Only keep submission date if submission_date > latest ECD - if dt_sub <= dt_ecd: + # Only keep submission date if submission_date > latest ECD + if dt_sub <= dt_ecd: + submission_date = None + else: submission_date = None - else: - submission_date = None - # 1️⃣ Add latest expected commencement date WITH conditional submission date - rows.append({ - "submission_date": submission_date, - "expected_commencement_date": latest_ecd, - "hubspot_id": deal["deal_properties"]["deal_id"], - "work_type": product_type, - "price": next( - (item["price"] for item in deal["line_items"] if product_type in item["name"]), - None - ), - "deal_name": deal["deal_properties"]["dealname"], - "company_name": deal["company_info"]["name"], - }) - - # 2️⃣ Add the remaining history WITHOUT submission date - for attempt in history_sorted[1:]: - rows.append({ - "submission_date": None, - "expected_commencement_date": historical_ecd_value_processes(attempt["value"]), + # 1️⃣ Add latest expected commencement date WITH conditional submission date + data = { + "submission_date": submission_date, + "expected_commencement_date": latest_ecd, "hubspot_id": deal["deal_properties"]["deal_id"], "work_type": product_type, "price": next( @@ -138,10 +125,46 @@ class jsonReader: ), "deal_name": deal["deal_properties"]["dealname"], "company_name": deal["company_info"]["name"], - }) + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + + # 2️⃣ Add the remaining history WITHOUT submission date + for attempt in history_sorted[1:]: + data = { + "submission_date": None, + "expected_commencement_date": historical_ecd_value_processes(attempt["value"]), + "hubspot_id": deal["deal_properties"]["deal_id"], + "work_type": product_type, + "price": next( + (item["price"] for item in deal["line_items"] if product_type in item["name"]), + None + ), + "deal_name": deal["deal_properties"]["dealname"], + "company_name": deal["company_info"]["name"], + } + data = self._use_different_expected_commencement_data(data, deal) + rows.append(data) + + + # Return a DataFrame or None return pd.DataFrame(rows) if rows else None + + def _use_different_expected_commencement_data(self, org_data, deal): + work_type = org_data['work_type'].lower() + if "Coordination Stage".lower() in work_type: + org_data.update({ + "expected_commencement_date": self.to_date_only(deal["deal_properties"]["mtp_planned_week"]), + "submission_date": self.to_date_only(deal["deal_properties"]["mtp_completion_date"]), + }) + elif "Design".lower() in work_type: + org_data.update({ + "expected_commencement_date": self.to_date_only(deal["deal_properties"]["design_planned_week"]), + "submission_date": self.to_date_only(deal["deal_properties"]["design_completion_date"]), + }) + return org_data def find_all_job_with_line_item(self): for i, deal in enumerate(self.raw_data):