max timeout is 12 hours

.github/workflows/gather_hubspot_data.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   gather_hubspot_data_and_upload_to_s3:
     runs-on: [self-hosted, mist]
+    timeout-minutes: 720
     steps:
       - uses: actions/checkout@v4
 

backend/src/dashboard/pages/sales_forecast.py

@@ -8,25 +8,39 @@ from datetime import datetime
 from dashboard.services.file_manager import FileManager
 from dashboard.services.json_reader import jsonReader
 from dashboard.components.pivot_charts import week_start_monday
-
+import os
 dash.register_page(__name__, path="/sales-forecast", name="Sales Forecast")
+import json
 
 # -----------------------
 # Load base dataframe
 # -----------------------
-df = pd.DataFrame({
-    "hubspot_id": [1, 2, 3, 4, 5, 6],
-    "product_type": ["Solar", "Cavity", "Solar", "Loft", "Cavity", "Solar"],
-    "price": [5000, 1200, 7000, 800, 2200, 6500],
-    "Planned Week": [
-        "2025-01-06",
-        "2025-01-06",
-        "2025-01-13",
-        "2025-01-13",
-        "2025-01-20",
-        "2025-01-20",
-    ]
-})
+# -----------------------------------------------------
+# Load & Build Master DF
+# -----------------------------------------------------
+def build_master_df(local=False):
+    if local is False:
+        s3 = FileManager()
+        key, path, data = s3.download_and_read_latest(
+            bucket="retrofit-data-dev",
+            prefix="hubspot_insight/sales_forecast/",
+        )
+    else:
+        file_path = os.path.join(os.path.dirname(__file__), "data.json")
+        with open(file_path, "r") as f:
+            data = json.load(f)
+
+    hubspot_data = jsonReader(data)
+    for p in hubspot_data.line_item_names:
+        df = hubspot_data.generate_df_via_product_type(p)
+
+        # show each deal
+
+    return hubspot_data
+
+
+# Load data once (refresh button can rebuild)
+df = build_master_df()
 
 # -----------------------
 # Page Layout

backend/src/dashboard/services/json_reader.py

@@ -60,6 +60,7 @@ class jsonReader:
     
     def _return_df_from_deal_info(self, deal, product_type):
         rows = []
+        print(deal)
         if deal["company_info"]["name"] != "Apple":
                 if deal["attempts"]:
                     # Multiple attempts => multiple rows