junte/insight
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/insight.git synced 2026-07-12 13:28:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #21 from Hestia-Homes/feature/make_it_live_ready

Browse source 
Feature/make it live ready
This commit is contained in:
Jun-te Kim 2025-12-03 22:29:58 +00:00 • committed by GitHub
commit 85248b375d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 103 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.devcontainer/docker-compose.yml
View file

@ -1,6 +1,5 @@
version: '3.8' version: '3.8'
services: services:
insight: insight:
init: true init: true

4
.github/workflows/gather_hubspot_data.yml vendored
View file

@ -8,10 +8,6 @@ on:
jobs: jobs:
gather_hubspot_data_and_upload_to_s3: gather_hubspot_data_and_upload_to_s3:
runs-on: [self-hosted, mist] runs-on: [self-hosted, mist]
timeout-minutes: 120 # <-- 2 hour timeout
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4

13
backend/src/dashboard/main.py
View file

@ -3,6 +3,8 @@ import dash_bootstrap_components as dbc
import pandas as pd import pandas as pd
from datetime import datetime, timedelta from datetime import datetime, timedelta
from dash import ctx from dash import ctx
import json
import os
from backend.src.dashboard.services.file_manager import FileManager from backend.src.dashboard.services.file_manager import FileManager
@ -27,9 +29,14 @@ def current_week_start():
# ----------------------------------------------------- # -----------------------------------------------------
# Load & Build Master DF # Load & Build Master DF
# ----------------------------------------------------- # -----------------------------------------------------
def build_master_df(): def build_master_df(local=False):
s3 = FileManager() if local is False:
key, path, data = s3.download_and_read_latest() s3 = FileManager()
key, path, data = s3.download_and_read_latest()
else:
file_path = os.path.join(os.path.dirname(__file__), "data.json")
with open(file_path, "r") as f:
data = json.load(f)
hubspot_data = jsonReader(data) hubspot_data = jsonReader(data)
frames = [] frames = []

5
backend/src/dashboard/services/hubspot_client.py
View file

@ -129,6 +129,9 @@ class HubSpotClient():
'design_planned_week', 'design_planned_week',
'retrofit_design_status', 'retrofit_design_status',
'design_completion_date', 'design_completion_date',
'item_id__monday_com_',
'funding_type',
'coordination_status__stage_1_',
] ]
) )
@ -258,7 +261,7 @@ class HubSpotClient():
"amount", "amount",
"hs_product_id", "hs_product_id",
"invoice_reference", "invoice_reference",
"invoiced" "invoiced",
] ]
) )
line_items.append(item.properties) line_items.append(item.properties)

3
backend/src/dashboard/services/hubspot_client_async.py
View file

@ -124,6 +124,9 @@ class HubSpotClientAsync:
'design_planned_week', 'design_planned_week',
'retrofit_design_status', 'retrofit_design_status',
'design_completion_date', 'design_completion_date',
'item_id__monday_com_',
'funding_type',
'coordination_status__stage_1_',
] ]
) )

149
backend/src/dashboard/services/json_reader.py
View file

@ -61,75 +61,62 @@ class jsonReader:
def _return_df_from_deal_info(self, deal, product_type): def _return_df_from_deal_info(self, deal, product_type):
rows = [] rows = []
if deal["company_info"]["name"] != "Apple": if deal["company_info"]["name"] != "Apple":
if deal["attempts"]: if deal["attempts"]:
# Multiple attempts => multiple rows # Multiple attempts => multiple rows
for attempt in deal["attempts"]: for attempt in deal["attempts"]:
rows.append({ data = {
"submission_date": self.to_date_only(attempt["submission_date"]), "submission_date": self.to_date_only(attempt["submission_date"]),
"hubspot_id": deal["deal_properties"]["deal_id"], "hubspot_id": deal["deal_properties"]["deal_id"],
"expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]), "expected_commencement_date": self.to_date_only(attempt["expected_commencement_date"]),
"work_type": product_type, "work_type": product_type,
"price": next( "price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]), (item["price"] for item in deal["line_items"] if product_type in item["name"]),
None None
), ),
"deal_name": deal["deal_properties"]["dealname"], "deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"], "company_name": deal["company_info"]["name"],
}) }
else: data = self._use_different_expected_commencement_data(data, deal)
def historical_ecd_value_processes(timestamp): rows.append(data)
if timestamp is None or timestamp == '': else:
return None def historical_ecd_value_processes(timestamp):
dt = datetime.strptime(timestamp, "%Y-%m-%d") if timestamp is None or timestamp == '':
return dt.strftime("%Y-%m-%d") return None
history = deal["deal_properties"]["expected_commencement_history"] dt = datetime.strptime(timestamp, "%Y-%m-%d")
return dt.strftime("%Y-%m-%d")
history = deal["deal_properties"]["expected_commencement_history"]
# ---- SORT HISTORY: latest first ---- # ---- SORT HISTORY: latest first ----
history_sorted = sorted( history_sorted = sorted(
history, history,
key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"), key=lambda h: datetime.strptime(h["timestamp"].split("T")[0], "%Y-%m-%d"),
reverse=True reverse=True
) )
# Extract latest expected commencement date # Extract latest expected commencement date
if history_sorted: if history_sorted:
latest = history_sorted[0] latest = history_sorted[0]
latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None latest_ecd = historical_ecd_value_processes(latest["value"]) # returns YYYY-MM-DD or None
# Convert submission date # Convert submission date
raw_submission_date = deal["deal_properties"].get("last_submission_date") raw_submission_date = deal["deal_properties"].get("last_submission_date")
submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None submission_date = self.to_date_only(raw_submission_date) if raw_submission_date else None
# Convert both to datetime for comparison # Convert both to datetime for comparison
if submission_date and latest_ecd: if submission_date and latest_ecd:
dt_sub = datetime.strptime(submission_date, "%Y-%m-%d") dt_sub = datetime.strptime(submission_date, "%Y-%m-%d")
dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d") dt_ecd = datetime.strptime(latest_ecd, "%Y-%m-%d")
# Only keep submission date if submission_date > latest ECD # Only keep submission date if submission_date > latest ECD
if dt_sub <= dt_ecd: if dt_sub <= dt_ecd:
submission_date = None
else:
submission_date = None submission_date = None
else:
submission_date = None
# 1️⃣ Add latest expected commencement date WITH conditional submission date # 1️⃣ Add latest expected commencement date WITH conditional submission date
rows.append({ data = {
"submission_date": submission_date, "submission_date": submission_date,
"expected_commencement_date": latest_ecd, "expected_commencement_date": latest_ecd,
"hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
})
# 2️⃣ Add the remaining history WITHOUT submission date
for attempt in history_sorted[1:]:
rows.append({
"submission_date": None,
"expected_commencement_date": historical_ecd_value_processes(attempt["value"]),
"hubspot_id": deal["deal_properties"]["deal_id"], "hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type, "work_type": product_type,
"price": next( "price": next(
@ -138,10 +125,46 @@ class jsonReader:
), ),
"deal_name": deal["deal_properties"]["dealname"], "deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"], "company_name": deal["company_info"]["name"],
}) }
data = self._use_different_expected_commencement_data(data, deal)
rows.append(data)
# 2️⃣ Add the remaining history WITHOUT submission date
for attempt in history_sorted[1:]:
data = {
"submission_date": None,
"expected_commencement_date": historical_ecd_value_processes(attempt["value"]),
"hubspot_id": deal["deal_properties"]["deal_id"],
"work_type": product_type,
"price": next(
(item["price"] for item in deal["line_items"] if product_type in item["name"]),
None
),
"deal_name": deal["deal_properties"]["dealname"],
"company_name": deal["company_info"]["name"],
}
data = self._use_different_expected_commencement_data(data, deal)
rows.append(data)
# Return a DataFrame or None # Return a DataFrame or None
return pd.DataFrame(rows) if rows else None return pd.DataFrame(rows) if rows else None
def _use_different_expected_commencement_data(self, org_data, deal):
work_type = org_data['work_type'].lower()
if "Coordination Stage".lower() in work_type:
org_data.update({
"expected_commencement_date": self.to_date_only(deal["deal_properties"]["mtp_planned_week"]),
"submission_date": self.to_date_only(deal["deal_properties"]["mtp_completion_date"]),
})
elif "Design".lower() in work_type:
org_data.update({
"expected_commencement_date": self.to_date_only(deal["deal_properties"]["design_planned_week"]),
"submission_date": self.to_date_only(deal["deal_properties"]["design_completion_date"]),
})
return org_data
def find_all_job_with_line_item(self): def find_all_job_with_line_item(self):
for i, deal in enumerate(self.raw_data): for i, deal in enumerate(self.raw_data):