diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py
index 269862c..27e6903 100644
--- a/etl/hubSpotClient/hubspot.py
+++ b/etl/hubSpotClient/hubspot.py
@@ -18,7 +18,7 @@ class DealStage(Enum):
SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
CUSTOMER_CONTACTED = "888730834"
SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
- NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000"
+ FILES_MISSING_FROM_ASSESSOR = "1887736000"
class HubSpotClient():
def __init__(self):
@@ -226,9 +226,6 @@ class HubSpotClient():
domna_id = domna_id,
uprn = uprn,
))
- if i > 1:
- # Break after just for speed
- break
except Exception as e:
def format_error_note(e):
note_text = "⚠️ Automated Verification Failed:
"
@@ -266,10 +263,8 @@ class HubSpotClient():
self.add_note_to_deal(deal_id, format_error_note(e))
else:
self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True)
-
-
self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'")
- self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)
+ self.move_deals_to_different_stage([deal_id], DealStage.FILES_MISSING_FROM_ASSESSOR.value)
return all_deals
def print_all_pipeline_ids(self):
diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py
index c042d6d..35afaa0 100644
--- a/etl/hubSpotClient/types.py
+++ b/etl/hubSpotClient/types.py
@@ -60,14 +60,20 @@ class SubmissionInfoFromDeal(BaseModel):
raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}")
try:
- # Check if sharepoint link is reachable and has any contents
- files = sp.get_folders_in_path(path)
- if "value" in files and len(files["value"]) > 0:
- pass
- else:
- raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}")
+ try:
+ files = sp.get_folders_in_path(path)
+ if files.get("value"):
+ pass
+ except Exception as e:
+ print("Trying SGEC")
+ sp = SharePointScraper(SharePointInstaller.SGEC)
+ files = sp.get_folders_in_path(path)
+ if files.get("value"):
+ pass
+ else:
+ raise ValueError(f"[SharePoint Folder Empty] Folder has no contents after multiple attempts: {self.submission_folder_path}")
except Exception as e:
- raise ValueError(str(e))
+ raise ValueError(f"[Folder Access Error] {str(e)}")
# download files in url and check files are there:
try:
@@ -80,7 +86,7 @@ class SubmissionInfoFromDeal(BaseModel):
if sdp.condition_report is None:
missing_items.append("Condition Report")
- if sdp.epr_summary_information is None:
+ if sdp.epr_with_data is None:
missing_items.append("EPR Energy report with data")
if sdp.rd_sap_xml is None:
diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py
index 7794d07..6b5f48c 100644
--- a/etl/hubspot_surveyed_needs_sign_off.py
+++ b/etl/hubspot_surveyed_needs_sign_off.py
@@ -10,6 +10,8 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
+os.environ["SGEC_SERVICE_SHAREPOINT_ID"] = "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9"
+
from etl.hubSpotClient.hubspot import DealStage, HubSpotClient
# Local development
@@ -18,7 +20,7 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post
hubspotClient = HubSpotClient()
# files missing from assessor column
-deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR)
+deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASSESSOR)
for deal in deals:
diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py
index 5d208e2..7dca011 100644
--- a/etl/month_end_automation_wave_2_layout.py
+++ b/etl/month_end_automation_wave_2_layout.py
@@ -17,13 +17,13 @@ rate_card_data = {
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Revision"
],
"rate": [
207.65, 101, 186.4, 98, 98,
450, 150, 163, 135, 120,
- "Mariaane Please tell me", 45, 90.5, 40,
- 25, 25, 25
+ "60 - Needs to be verified (Post EPR)", 45, 90.5, 40,
+ 25, 25, 25, "check with Kevin"
]
}
@@ -124,7 +124,7 @@ filtered_dfs.append(v3)
cors2 = df[
df["rc stg. 2"].str.lower().isin(["to invoice"])
].copy()
-cors2["joby_type"] = "Coordination Stage 2"
+cors2["job_type"] = "Coordination Stage 2"
filtered_dfs.append(cors2)
# Design type archietype
@@ -143,9 +143,9 @@ filtered_dfs.append(design1)
# Design stage revisions
design2 = df[
- df["revision 2 design invoice"].str.lower().isin(["to invoice"])
+ df["design revision invoice status"].str.lower().isin(["to invoice"])
].copy()
-design2["job_type"] = "Design Repetitive"
+design2["job_type"] = "Design Revision"
filtered_dfs.append(design2)
# Lodgement Phase 1
@@ -164,7 +164,7 @@ filtered_dfs.append(lodg2)
# POST EPC
post_epc = df[
- df["post-epc status"].str.lower().isin(["epc pics uploaded"])
+ df["post-epc status"].str.lower().isin(["epc files uploaded"])
].copy()
post_epc["job_type"] = "Post EPC"
filtered_dfs.append(post_epc)
diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py
index 962eb46..7fa4c1a 100644
--- a/etl/month_end_automation_wave_2_no_12.py
+++ b/etl/month_end_automation_wave_2_no_12.py
@@ -11,18 +11,20 @@ monday = MondayClient(monday_key)
# Shropshire Council HUG2
board_ids = ["4718185486"]
+empty = "nothing on rate card"
+
rate_card_data = {
"job_type": [
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Sign off"
],
"rate": [
- 259, 125, 280, 125, 125,
- 650, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125,
+ empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135,
+ 120, "(60) Post EPR, please verify", 65, 115, 60,
+ 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne"
]
}
@@ -117,7 +119,7 @@ filtered_dfs.append(v2)
# Coordination stage 2 Please complete
cors2 = df[
df["rc stage 2 invoice"].str.lower().isin(["to invoice"])
-]
+].copy()
cors2["joby_type"] = "Coordination Stage 2"
filtered_dfs.append(cors2)
@@ -134,6 +136,9 @@ filtered_dfs.append(cors2)
# ], "Design Revision")
# filtered_dfs.append(design2)
+# Design sign off
+design_sign_pff = get_df(df, "design payment step 3", ["ready to invoice"], "design sign off")
+
# Lodgement Phase 1
lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1")
filtered_dfs.append(lodg1)
diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py
index 9203f2b..4a644ee 100644
--- a/etl/month_end_automation_wave_2_no_14.py
+++ b/etl/month_end_automation_wave_2_no_14.py
@@ -11,6 +11,24 @@ monday = MondayClient(monday_key)
# Decent Homes Stonewater - Operations
board_ids = ["9319118237"]
+empty = "Rate card info missing"
+rate_card_data_example = {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1",
+ "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show"
+ ],
+ "rate": [
+ 259, 125, 280, 125,
+ 260, 126, 281, 126,
+ 262, 127, 282, 127,
+ ]
+}
+
+
+rate_card_df = pd.DataFrame(rate_card_data_example)
+
for board in tqdm(board_ids):
board_data = monday.boards.fetch_boards_by_id(board)
columns = board_data["data"]["boards"][0]["columns"]
@@ -79,25 +97,25 @@ filtered_dfs.append(ra)
# ATT
-# att = get_df(df, "att", ["completed"], "ATT")
-# filtered_dfs.append(att)
+att = get_df(df, "att", ["completed"], "ATT")
+filtered_dfs.append(att)
# V1 Coordination
v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [
"rc complete",
-], "V1 Coordination")
+], "Coordination Stage 1 v1")
filtered_dfs.append(v1)
-# V2 Coordination
-v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination")
-filtered_dfs.append(v2)
+# # V2 Coordination
+# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination")
+# filtered_dfs.append(v2)
-# # V3 Coordination
-v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination")
-filtered_dfs.append(v3)
+# # # V3 Coordination
+# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination")
+# filtered_dfs.append(v3)
-v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination")
-filtered_dfs.append(v3)
+# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination")
+# filtered_dfs.append(v3)
# Coordination stage 2 Please complete
# cors2 = df[
@@ -107,8 +125,8 @@ filtered_dfs.append(v3)
# filtered_dfs.append(cors2)
# Design stage 1
-design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
-filtered_dfs.append(design1)
+# design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
+# filtered_dfs.append(design1)
# Design revision
# design2 = get_df(df, "design revision invoice", [
@@ -125,7 +143,7 @@ filtered_dfs.append(lodg1)
# Full Lodgement Phase
-full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement")
+full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
filtered_dfs.append(full_lodgement)
# POST EPC
@@ -133,12 +151,12 @@ post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC")
filtered_dfs.append(post_epc)
-# # POST EPR
-# post_epr = df[
-# df["post-epc status"].str.lower().isin(["post epr completed"])
-# ].copy()
-# post_epr["job_type"] = "POST epr"
-# filtered_dfs.append(post_epr)
+# POST EPR
+post_epr = df[
+ df["post epc"].str.lower().isin(["post epr completed"])
+].copy()
+post_epr["job_type"] = "POST epr"
+filtered_dfs.append(post_epr)
# Post ATT
post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT")
diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py
index 3686198..8644db9 100644
--- a/etl/month_end_automation_wave_2_no_15.py
+++ b/etl/month_end_automation_wave_2_no_15.py
@@ -11,18 +11,21 @@ monday = MondayClient(monday_key)
# WCHG SHDF 2.1 Mansard
board_ids = ["5636990610"]
+
+empty = "Rate card was empty"
+
rate_card_data = {
"job_type": [
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Revision"
],
"rate": [
- 259, 125, 280, 125, 125,
- 650, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty,
+ empty, 180, 275, 135,
+ 120, "60 - please verify with Marianne", 45, 45, 40,
+ 25, 25, 25, "Please check price for design revision with Andreas"
]
}
@@ -84,11 +87,12 @@ df = pd.DataFrame(all_records)
filtered_dfs = []
-def get_df(df, column_name, success_critera, job_name):
+def get_df(df, column_name, success_critera, job_name=None):
_ = df[
df[column_name].str.lower().isin(success_critera)
].copy()
- _["job_type"] = job_name
+ if job_name:
+ _["job_type"] = job_name
return _
# RA
@@ -107,8 +111,8 @@ v1 = get_df(df, "coordination status (ioe mtp)".lower(), [
filtered_dfs.append(v1)
# V2 Coordination
-# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination")
-# filtered_dfs.append(v2)
+v2 = get_df(df, "v2 coordination status (ioe mtp)", ["rc complete"], "Coordination Stage 1 v2 remodel")
+filtered_dfs.append(v2)
# # V3 Coordination
# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination")
@@ -125,18 +129,21 @@ cors2 = df[
cors2["joby_type"] = "Coordination Stage 2"
filtered_dfs.append(cors2)
-# Design stage 1
-# design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
-# filtered_dfs.append(design1)
+# Design stage
+design = get_df(df, "design invoice status", ["to invoice"])
+
+# Design archeytpe
+de = get_df(design, "prop type for invoicing", ["archetype"], "Design Archetype")
+filtered_dfs.append(de)
+# Design repetitive
+de = get_df(design, "prop type for invoicing", ["repetitive"], "Design Repetitive")
+filtered_dfs.append(de)
# Design revision
-# design2 = get_df(df, "design revision invoice", [
-# "Rev. A to invoice".lower(),
-# "Rev. B to invoice".lower(),
-# "Rev. C to invoice".lower(),
-# "Rev. D to invoice".lower(),
-# ], "Design Revision")
-# filtered_dfs.append(design2)
+design2 = get_df(df, "design revision invoice status", [
+ "to invoice".lower(),
+], "Design Revision")
+filtered_dfs.append(design2)
# Lodgement Phase 1
lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1")
diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py
index 6e497c5..b349a7a 100644
--- a/etl/month_end_automation_wave_2_no_3.py
+++ b/etl/month_end_automation_wave_2_no_3.py
@@ -17,13 +17,13 @@ rate_card_data = {
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Revision"
],
"rate": [
- 259, 125, 280, 125, 125,
- 650, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ 259, 101, 210, 95, 95,
+ 450, 150, 195, 135,
+ 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4,
+ 25, 25, 25, "Please ask for Design Revision"
]
}
@@ -143,6 +143,11 @@ design1["job_type"] = "Design repetitive"
filtered_dfs.append(design1)
# Design Revision
+design_revision = df[
+ df["design revision invoice status"].str.lower().isin(["to invoice"])
+].copy()
+design_revision["job_type"] = "Design repetitive"
+filtered_dfs.append(design_revision)
# Lodgement Phase 1
lodg1 = df[
diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py
index 1c17c7e..30b949f 100644
--- a/etl/month_end_automation_wave_2_no_4.py
+++ b/etl/month_end_automation_wave_2_no_4.py
@@ -17,13 +17,13 @@ rate_card_data = {
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Revision"
],
"rate": [
- 259, 125, 280, 125, 125,
- 650, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ 165.75, 72.25, 174.25, 174.25, 174.25,
+ 175, 175, 124.25, 135,
+ 120, "(60) - please check with Marianne", 45, 63.75, 34,
+ 25, 25, 25, "Please ask marianne or kev for design revision"
]
}
@@ -107,7 +107,12 @@ v1["job_type"] = "Coordination Stage 1 v1"
filtered_dfs.append(v1)
# V2 Coordination
-_ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy()
+_ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy()
+_["job_type"] = "Coordination Stage 1 v2 remodel"
+filtered_dfs.append(_)
+
+# V2 Coordination
+_ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy()
_["job_type"] = "Coordination Stage 1 v2 remodel"
filtered_dfs.append(_)
@@ -116,6 +121,11 @@ v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
v3["job_type"] = "Coordination Stage 1 v3 remodel"
filtered_dfs.append(_)
+# V3 Coordination
+v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy()
+v3["job_type"] = "Coordination Stage 1 v3 remodel"
+filtered_dfs.append(_)
+
# Coordination stage 2 Please complete
cors2 = df[
df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"])
@@ -139,12 +149,12 @@ design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy(
design1["job_type"] = "Design Repetitive"
filtered_dfs.append(design1)
-# Design revision
-# design2 = df[
-# df[""].str.lower().isin(["to invoice"])
-# ].copy()
-# design2["job_type"] = "Design Revision"
-# filtered_dfs.append(design2)
+# Design Revision
+design_revision = df[
+ df["design revision invoice status"].str.lower().isin(["to invoice"])
+].copy()
+design_revision["job_type"] = "Design repetitive"
+filtered_dfs.append(design_revision)
# Lodgement Phase 1
lodg1 = df[
diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py
index 0cc1749..214c233 100644
--- a/etl/month_end_automation_wave_2_no_7.py
+++ b/etl/month_end_automation_wave_2_no_7.py
@@ -12,7 +12,7 @@ monday = MondayClient(monday_key)
#Home Group Wave 2SP+
board_ids = ["4254419092"]
-rate_card_data = {
+rate_card_data_sp_plus = {
"job_type": [
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
@@ -20,14 +20,48 @@ rate_card_data = {
"RA no show", "ATT no show", "post EPC no show"
],
"rate": [
- 259, 125, 280, 125, 125,
- 650, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ 170, 70, 200, "check with Kevin", "check with Kevin",
+ 470, 155, 165, 135,
+ 120, "60 but check with Kevin as EPR", 45, 70, 40,
+ 30, 30, 30
]
}
-rate_card_df = pd.DataFrame(rate_card_data)
+emp_msg = "was empty in rate card - ask Marianne/Kevin"
+rate_card_data_net_zero = {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
+ "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show"
+ ],
+ "rate": [
+ 170, 70, 200, emp_msg, emp_msg,
+ 325, 140, 165, 135,
+ 120, "60 but check with Kevin as EPR", 45, 70, 40,
+ 30, 30, 30
+ ]
+}
+
+error_message = "Unsure which client this one is - sorry!"
+rate_card_data_error_msg= {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
+ "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show"
+ ],
+ "rate": [
+ error_message, error_message, error_message, error_message, error_message,
+ error_message, error_message, error_message, error_message,
+ error_message, error_message, error_message, error_message, error_message,
+ error_message, error_message, error_message
+ ]
+}
+
+rate_card_df_sp_plus = pd.DataFrame(rate_card_data_sp_plus)
+rate_card_df_net_zero = pd.DataFrame(rate_card_data_net_zero)
+rate_card_df_error_message = pd.DataFrame(rate_card_data_error_msg)
for board in tqdm(board_ids):
@@ -113,9 +147,9 @@ filtered_dfs.append(v1)
v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel")
filtered_dfs.append(v2)
-# # V3 Coordination
-# v3 = get_df(df, "")
-# # filtered_dfs.append(v3)
+# V3 Coordination
+v3 = get_df(df, "v3 rc status", ["rc completed"], "Coordination Stage 1 v3 remodel")
+filtered_dfs.append(v3)
# Coordination stage 2 Please complete
cors2 = df[
@@ -135,13 +169,10 @@ design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design R
filtered_dfs.append(design1)
# Design revision
-# design2 = get_df(df, "design revision invoice", [
-# "Rev. A to invoice".lower(),
-# "Rev. B to invoice".lower(),
-# "Rev. C to invoice".lower(),
-# "Rev. D to invoice".lower(),
-# ], "Design Revision")
-# filtered_dfs.append(design2)
+design2 = get_df(df, "design revision invoice status", [
+ "to invoice"
+], "Design Revision")
+filtered_dfs.append(design2)
# Lodgement Phase 1
lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1")
@@ -164,7 +195,7 @@ post_epr["job_type"] = "POST epr"
filtered_dfs.append(post_epr)
# Post ATT
-post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT")
+post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT")
filtered_dfs.append(post_att)
@@ -201,12 +232,26 @@ filtered_dfs.append(epc_ns)
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
final_df["job_type"] = final_df["job_type"].str.lower()
-rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
+rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower()
+rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower()
+rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower()
# Now perform the merge
-combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
+net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)]
+sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)]
+other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.isin(sp_plus_df.index)]
+
+combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left")
+combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left")
+combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left")
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
attribute = ['address', 'client', 'job_type', 'rate']
-combined_with_rates[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False)
+combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False)
+combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False)
+combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False)
+
+
+# TO DO check everything in excel
+# make logic for seperation
\ No newline at end of file
diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py
index d8f0a35..0aff6fc 100644
--- a/etl/month_end_automation_wave_2_no_8.py
+++ b/etl/month_end_automation_wave_2_no_8.py
@@ -16,13 +16,13 @@ rate_card_data = {
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
- "RA no show", "ATT no show", "post EPC no show"
+ "RA no show", "ATT no show", "post EPC no show", "Design Revision"
],
"rate": [
259, 125, 280, 125, 125,
650, 415, 195, 175, 135,
- 120, "Post EPR Please Marianne", 85, 125, 60,
- 25, 25, 25
+ 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60,
+ 45, 45, 45, "Design Revision check with Kevin/Marianne"
]
}
@@ -139,13 +139,10 @@ design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repet
filtered_dfs.append(design1)
# Design revision
-# design2 = get_df(df, "design revision invoice", [
-# "Rev. A to invoice".lower(),
-# "Rev. B to invoice".lower(),
-# "Rev. C to invoice".lower(),
-# "Rev. D to invoice".lower(),
-# ], "Design Revision")
-# filtered_dfs.append(design2)
+design2 = get_df(df, "design revision invoice status", [
+ "to invoice"
+], "Design Revision")
+filtered_dfs.append(design2)
# Lodgement Phase 1
lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1")
@@ -199,11 +196,11 @@ filtered_dfs.append(att_ns)
# Post visit no show
-# epc_ns = df[
-# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
-# ].copy()
-# epc_ns["job_type"] = "post EPC NO SHOW"
-# filtered_dfs.append(epc_ns)
+epc_ns = df[
+ df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999)
+].copy()
+epc_ns["job_type"] = "post EPC NO SHOW"
+filtered_dfs.append(epc_ns)
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py
index 7e273f2..673ea66 100644
--- a/etl/month_end_automation_wave_3_layout.py
+++ b/etl/month_end_automation_wave_3_layout.py
@@ -10,13 +10,15 @@ import json
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
board_ids = [
- # "9349630181", # WCHG Walkups-Operations
+# "9349630181", # WCHG Walkups-Operations
# "8829428746", # 2502 Accent Housing
- "8830772914", # "L&Q London"
+ # "8830772914", # "L&Q London"
# "9601691730", # Cardo Wales & West - Wave 3
+ "9660895490", # Northumberland County SHDF Wave 3
]
-rate_card_data = {
+empty = "Rate card info missing"
+rate_card_data_example = {
"job_type": [
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
@@ -31,7 +33,59 @@ rate_card_data = {
]
}
-rate_card_df = pd.DataFrame(rate_card_data)
+
+rate_card_data_2502_accent_housing = {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
+ "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
+ ],
+ "rate": [
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, 280, 150
+ ]
+}
+
+rate_card_data_l_and_q_london = {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
+ "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
+ ],
+ "rate": [
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, 280, 150
+ ]
+}
+
+
+rate_card_data_northhumberland_country_shdf_wave_3 = {
+ "job_type": [
+ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
+ "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
+ "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
+ "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
+ ],
+ "rate": [
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, empty, empty,
+ empty, empty, empty, 280, 150
+ ]
+}
+
+
+
+# rate_card_df = pd.DataFrame(rate_card_data_example)
+# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing)
+# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london)
+rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
for board in tqdm(board_ids):
@@ -210,4 +264,4 @@ import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
attribute = ['address', 'client', 'job_type', 'rate']
-combined_with_rates[attribute].to_excel(f'WCHG Walkups-Operations {timestamp}.xlsx', index=False)
+combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False)