junte/insight
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/insight.git synced 2026-06-08 11:17:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #21 from Hestia-Homes/feature/make_it_live_ready

Browse source 
Feature/make it live ready
This commit is contained in:
Jun-te Kim 2025-12-03 22:29:58 +00:00 • committed by GitHub
commit 85248b375d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 103 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.devcontainer/docker-compose.yml
View file

@ -1,6 +1,5 @@
version: '3.8'
services:
insight:
init: true

4
.github/workflows/gather_hubspot_data.yml vendored
View file

@ -8,10 +8,6 @@ on:
jobs:
gather_hubspot_data_and_upload_to_s3:
runs-on: [self-hosted, mist]
timeout-minutes: 120 # <-- 2 hour timeout
steps:
- uses: actions/checkout@v4

13
backend/src/dashboard/main.py
View file

@ -3,6 +3,8 @@ import dash_bootstrap_components as dbc
import pandas as pd
from datetime import datetime, timedelta
from dash import ctx
import json
import os
from backend.src.dashboard.services.file_manager import FileManager
@ -27,9 +29,14 @@ def current_week_start():
# -----------------------------------------------------
# Load & Build Master DF
# -----------------------------------------------------
def build_master_df():
s3 = FileManager()
key, path, data = s3.download_and_read_latest()
def build_master_df(local=False):
if local is False:
s3 = FileManager()
key, path, data = s3.download_and_read_latest()
else:
file_path = os.path.join(os.path.dirname(__file__), "data.json")
with open(file_path, "r") as f:
data = json.load(f)
hubspot_data = jsonReader(data)
frames = []

5
backend/src/dashboard/services/hubspot_client.py
View file

@ -129,6 +129,9 @@ class HubSpotClient():
'design_planned_week',
'retrofit_design_status',
'design_completion_date',
'item_id__monday_com_',
'funding_type',
'coordination_status__stage_1_',
]
)
@ -258,7 +261,7 @@ class HubSpotClient():
"amount",
"hs_product_id",
"invoice_reference",
"invoiced"
"invoiced",
]
)
line_items.append(item.properties)

3
backend/src/dashboard/services/hubspot_client_async.py
View file

@ -124,6 +124,9 @@ class HubSpotClientAsync:
'design_planned_week',
'retrofit_design_status',
'design_completion_date',
'item_id__monday_com_',
'funding_type',
'coordination_status__stage_1_',
]
)

149
backend/src/dashboard/services/json_reader.py
View file

@ -61,75 +61,62 @@ class jsonReader:
def _return_df_from_deal_info(self, deal, product_type):
rows = []
if deal["company_info"]["name"] != "Apple":
if deal["attempts"]:
# Multiple attempts => multiple rows
for attempt in deal["attempts"]:
rows.append({
"submission_date": self.to_date_only(attempt["submission_date"]),
"hubspot_id": deal["deal_properties"]["deal_id"],
"expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]),
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
})
else:
def historical_ecd_value_processes(timestamp):
if timestamp is None or timestamp == '':
return None
dt = datetime.strptime(timestamp, "%Y-%m-%d")
return dt.strftime("%Y-%m-%d")
history = deal["deal_properties"]["expected_commencement_history"]
if deal["attempts"]:
# Multiple attempts => multiple rows
for attempt in deal["attempts"]:
data = {
"submission_date": self.to_date_only(attempt["submission_date"]),
"hubspot_id": deal["deal_properties"]["deal_id"],
"expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]),
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
}
data = self._use_different_expected_commencement_data(data, deal)
rows.append(data)
else:
def historical_ecd_value_processes(timestamp):
if timestamp is None or timestamp == '':
return None
dt = datetime.strptime(timestamp, "%Y-%m-%d")
return dt.strftime("%Y-%m-%d")
history = deal["deal_properties"]["expected_commencement_history"]
# ---- SORT HISTORY: latest first ----
history_sorted = sorted(
history,
key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"),
reverse=True
)
# ---- SORT HISTORY: latest first ----
history_sorted = sorted(
history,
key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"),
reverse=True
)
# Extract latest expected commencement date
if history_sorted:
latest = history_sorted[0]
latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None
# Extract latest expected commencement date
if history_sorted:
latest = history_sorted[0]
latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None
# Convert submission date
raw_submission_date = deal["deal_properties"].get("last_submission_date")
submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None
# Convert submission date
raw_submission_date = deal["deal_properties"].get("last_submission_date")
submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None
# Convert both to datetime for comparison
if submission_date and latest_ecd:
dt_sub = datetime.strptime(submission_date, "%Y-%m-%d")
dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d")
# Convert both to datetime for comparison
if submission_date and latest_ecd:
dt_sub = datetime.strptime(submission_date, "%Y-%m-%d")
dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d")
# Only keep submission date if submission_date > latest ECD
if dt_sub <= dt_ecd:
# Only keep submission date if submission_date > latest ECD
if dt_sub <= dt_ecd:
submission_date = None
else:
submission_date = None
else:
submission_date = None
# 1️⃣ Add latest expected commencement date WITH conditional submission date
rows.append({
"submission_date": submission_date,
"expected_commencement_date": latest_ecd,
"hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
})
# 2️⃣ Add the remaining history WITHOUT submission date
for attempt in history_sorted[1:]:
rows.append({
"submission_date": None,
"expected_commencement_date": historical_ecd_value_processes(attempt["value"]),
# 1️⃣ Add latest expected commencement date WITH conditional submission date
data = {
"submission_date": submission_date,
"expected_commencement_date": latest_ecd,
"hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type,
"price": next(
@ -138,10 +125,46 @@ class jsonReader:
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
})
}
data = self._use_different_expected_commencement_data(data, deal)
rows.append(data)
# 2️⃣ Add the remaining history WITHOUT submission date
for attempt in history_sorted[1:]:
data = {
"submission_date": None,
"expected_commencement_date": historical_ecd_value_processes(attempt["value"]),
"hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
}
data = self._use_different_expected_commencement_data(data, deal)
rows.append(data)
# Return a DataFrame or None
return pd.DataFrame(rows) if rows else None
def _use_different_expected_commencement_data(self, org_data, deal):
work_type = org_data['work_type'].lower()
if "Coordination Stage".lower() in work_type:
org_data.update({
"expected_commencement_date": self.to_date_only(deal["deal_properties"]["mtp_planned_week"]),
"submission_date": self.to_date_only(deal["deal_properties"]["mtp_completion_date"]),
})
elif "Design".lower() in work_type:
org_data.update({
"expected_commencement_date": self.to_date_only(deal["deal_properties"]["design_planned_week"]),
"submission_date": self.to_date_only(deal["deal_properties"]["design_completion_date"]),
})
return org_data
def find_all_job_with_line_item(self):
for i, deal in enumerate(self.raw_data):