added month end automation

.github/workflows/months_end.yml

@@ -0,0 +1,31 @@
+name: Months End
+on:
+  schedule:
+    # - cron: '0 17 * * 1-5'
+  push:
+    branches: [main, feature/month_end_automation_of_all]
+  workflow_dispatch:
+
+jobs:
+  surveyed-needs-sign-off:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-root
+
+      - name: run script
+        run: |
+          pwd
+          ls -la
+          poetry run python etl/month_end_automation_wave_2_layout.py
+        env:
+          PYTHONPATH: ${{ github.workspace }}

etl/MonthEndUploader.py

@@ -3,10 +3,10 @@ os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
 os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
 os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
 from etl.scraper.scraper import SharePointScraper, SharePointInstaller
-import datetime
+from datetime import datetime
 
 
-def upload_to_month_end_folder(file_name_on_sp, local_file_path):
+def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
     sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
 
     parent_folder = "General/Junte Kim/month end"
@@ -16,6 +16,9 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path):
 
     sharepoint.create_dir(formatted_date, parent_folder)
     sharepoint_path = parent_folder + "/" + formatted_date
+    sharepoint.create_dir(add_to_path, sharepoint_path)
+    sharepoint_path += "/" + add_to_path 
+
     print("Uploading to sharepoint...")
     sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
     print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")

etl/month_end_automation_wave_2_layout.py

@@ -7,6 +7,7 @@ from pprint import pprint
 import pandas as pd
 import json
 from MonthEndUploader import upload_to_month_end_folder
+import os
 
 
 monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
@@ -234,5 +235,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
 import datetime
 timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
 
+# Upload to sharepoint
 attribute = ['address', 'client', 'job_type', 'rate']
-combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False)
+master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2"
+file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx"
+combined_with_rates[attribute].to_excel(file_name, index=False)
+
+file_path = os.path.abspath(file_name)
+
+upload_to_month_end_folder(file_name, file_path, master_folder_name)
+
+invoice_name = "rate_card.xlsx"
+file_path = os.path.abspath(invoice_name)
+rate_card_df.to_excel(invoice_name, index=False)
+upload_to_month_end_folder(invoice_name, file_path, master_folder_name)