mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
Merge pull request #80 from Hestia-Homes/feature/month_end_automation_of_all
Feature/month end automation of all
This commit is contained in:
Jun-te Kim
GitHub
commit
745af0e440
22 changed files with 658 additions and 226 deletions
45
.github/workflows/months_end.yml
vendored
Normal file
45
.github/workflows/months_end.yml
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
name: Months End
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
surveyed-needs-sign-off:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install poetry
|
||||
poetry install --no-root
|
||||
|
||||
- name: run script
|
||||
run: |
|
||||
pwd
|
||||
ls -la
|
||||
poetry run python etl/month_end_automation_wave_2_layout.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_3.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_4.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_5.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_6.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_7.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_8.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_9.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_10.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_11.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_12.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_13.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_14.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_15.py
|
||||
poetry run python etl/month_end_automation_wave_2_no_16.py
|
||||
poetry run python etl/month_end_automation_wave_accent_housing.py
|
||||
poetry run python etl/month_end_automation_wave_3_layout.py
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
|
|
@ -57,7 +57,7 @@ def serialize_model(model: Any):
|
|||
else:
|
||||
return model
|
||||
|
||||
def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations):
|
||||
def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations, bepoke_info):
|
||||
# Convert to dict recursively
|
||||
rooms_data = serialize_model(rooms_obj)
|
||||
heating_data = serialize_model(heating_system_obj)
|
||||
|
|
@ -70,6 +70,7 @@ def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevatio
|
|||
"heating_system": heating_data,
|
||||
"occupant_info": occupant_data,
|
||||
"access_and_elevations": access_and_elevations_data,
|
||||
"bespoke_data": bepoke_info
|
||||
}
|
||||
|
||||
# Convert to pretty JSON string
|
||||
|
|
@ -203,18 +204,24 @@ def handler(event, context):
|
|||
local_path = download_private_s3_file(file_uri)
|
||||
|
||||
# Local development of file, please comment out for prod
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf"))
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1 1.pdf"))
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf"))
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf"))
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf"))
|
||||
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/6E-plantagenet-street.pdf"))
|
||||
|
||||
|
||||
|
||||
print("Extracting file...")
|
||||
reader = pdfReaderToText(local_path)
|
||||
# obj2 = WarmHomesConditionReport(reader.text_list, debug=True)
|
||||
obj = WarmHomesConditionReport(reader.text_list)
|
||||
json_ = make_final_json(
|
||||
obj.master_obj[0],
|
||||
obj.master_obj[1],
|
||||
obj.master_obj[2],
|
||||
obj.master_obj[3],
|
||||
{}
|
||||
)
|
||||
print("Extracted completed, made json")
|
||||
|
||||
|
|
|
|||
24
etl/MonthEndUploader.py
Normal file
24
etl/MonthEndUploader.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import os
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
|
||||
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
|
||||
|
||||
parent_folder = "General/Junte Kim/month end"
|
||||
today = datetime.today()
|
||||
# Format as "Month YYYY"
|
||||
formatted_date = today.strftime("%B %Y")
|
||||
|
||||
sharepoint.create_dir(formatted_date, parent_folder)
|
||||
sharepoint_path = parent_folder + "/" + formatted_date
|
||||
sharepoint.create_dir(add_to_path, sharepoint_path)
|
||||
sharepoint_path += "/" + add_to_path
|
||||
|
||||
print("Uploading to sharepoint...")
|
||||
sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
|
||||
print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")
|
||||
|
|
@ -102,12 +102,12 @@ class ECOConditionReport(SiteNotesExtractor):
|
|||
|
||||
|
||||
class WarmHomesConditionReport(SiteNotesExtractor):
|
||||
def __init__(self, data_list):
|
||||
def __init__(self, data_list, debug = False):
|
||||
super().__init__(data_list)
|
||||
self.type = ReportType.WARM_HOMES_CONDITION_REPORT
|
||||
room, heating_system, occupant, access_and_elevation = self.setup_condition_report()
|
||||
|
||||
self.master_obj = room, heating_system, occupant, access_and_elevation
|
||||
if debug is False:
|
||||
room, heating_system, occupant, access_and_elevation = self.setup_condition_report()
|
||||
self.master_obj = room, heating_system, occupant, access_and_elevation
|
||||
|
||||
def setup_condition_report(self):
|
||||
# general_information = self.get_section_1()
|
||||
|
|
@ -292,7 +292,7 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
|
||||
# Gable 2
|
||||
data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding")
|
||||
state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False
|
||||
state = False # TODO quick demo for waltham forest, set to always false
|
||||
if state is False:
|
||||
gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state)
|
||||
else:
|
||||
|
|
@ -347,12 +347,24 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
)
|
||||
|
||||
def get_room_info(self, data):
|
||||
ventilation = VentilationInfo(
|
||||
is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False,
|
||||
are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False,
|
||||
are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"),
|
||||
is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False,
|
||||
)
|
||||
state = True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False
|
||||
if state is False:
|
||||
ventilation = VentilationInfo(
|
||||
is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False,
|
||||
are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False,
|
||||
are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"),
|
||||
is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False,
|
||||
)
|
||||
else:
|
||||
ventilation = VentilationInfo(
|
||||
is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False,
|
||||
are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False,
|
||||
are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"),
|
||||
is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False,
|
||||
location_of_any_damp_or_mould=self.get_next_value(data, "Location of any damp/mould"),
|
||||
what_severity_of_damp_mould_would_you_consider_this_to_be=self.get_next_value(data, "What severity of damp/mould would you consider this to be?"),
|
||||
)
|
||||
|
||||
|
||||
windows_state = True if self.get_next_value(data, "Does the room have any windows?").lower() == "yes" else False
|
||||
if windows_state:
|
||||
|
|
@ -368,20 +380,37 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
does_the_room_have_any_windows=windows_state
|
||||
)
|
||||
|
||||
return RoomInfo(
|
||||
overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"),
|
||||
does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
|
||||
windows_info=windows,
|
||||
ventilation_info=ventilation,
|
||||
)
|
||||
state = True if self.get_next_value(data, "cracking in walls, etc.)").lower()=="yes" else False
|
||||
if state:
|
||||
return RoomInfo(
|
||||
overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"),
|
||||
does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
|
||||
description_of_defect=self.get_next_value(data, "Description of the defects", avoid=["Any additional recommendations?"]),
|
||||
windows_info=windows,
|
||||
ventilation_info=ventilation,
|
||||
)
|
||||
else:
|
||||
return RoomInfo(
|
||||
overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"),
|
||||
does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
|
||||
windows_info=windows,
|
||||
ventilation_info=ventilation,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def get_hallway(self):
|
||||
data = self.get_data_between("Hallway", "Living Room")
|
||||
room = self.get_room_info(data)
|
||||
|
||||
hallway_state = True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False
|
||||
|
||||
if hallway_state:
|
||||
room = self.get_room_info(data)
|
||||
else:
|
||||
room = None
|
||||
|
||||
return Hallway(
|
||||
is_there_a_hallway=True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False,
|
||||
is_there_a_hallway=hallway_state,
|
||||
room_info=room,
|
||||
)
|
||||
|
||||
|
|
@ -494,20 +523,26 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
|
||||
def get_loft_space(self):
|
||||
data = self.get_data_between("Loft Space", "Room in Roof")
|
||||
return LoftSpace(
|
||||
is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"),
|
||||
is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False,
|
||||
overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"),
|
||||
are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"),
|
||||
does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
|
||||
existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"),
|
||||
is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"),
|
||||
is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"),
|
||||
condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"),
|
||||
is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"),
|
||||
is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"),
|
||||
is_it_accessible=self.get_next_value(data, "Is it accessible?"),
|
||||
)
|
||||
state = True if self.get_next_value(data, "Is the main loft space accessible?").lower() == "yes" else False
|
||||
if state:
|
||||
return LoftSpace(
|
||||
is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"),
|
||||
is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False,
|
||||
overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"),
|
||||
are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"),
|
||||
does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"),
|
||||
existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"),
|
||||
is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"),
|
||||
is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"),
|
||||
condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"),
|
||||
is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"),
|
||||
is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"),
|
||||
)
|
||||
else:
|
||||
return LoftSpace(
|
||||
is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"),
|
||||
is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False,
|
||||
)
|
||||
|
||||
def get_room_in_roof(self):
|
||||
data = self.get_data_between("Room in Roof", "4. Heating System")
|
||||
|
|
@ -596,12 +631,12 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
is_there_any_renewable_energy_system_in_place=True if self.get_next_value(data, "Is there any renewable energy system in place?").lower() == "yes" else False,
|
||||
suitable_roof_orientation_for_solar_pv_water=self.get_next_value(data, "Suitable roof orientation for Solar PV/water:"),
|
||||
is_there_a_water_tank=True if self.get_next_value(data, "Is there a Water Tank?").lower() == "yes" else False,
|
||||
type=self.get_next_value(data, "Type"),
|
||||
size=self.get_next_value(data, "Size"),
|
||||
tank_location=self.get_next_value(data, "Tank Location"),
|
||||
is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False,
|
||||
type_of_insulation=self.get_next_value(data, "Type of Insulation"),
|
||||
thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")),
|
||||
# type=self.get_next_value(data, "Type"),
|
||||
# size=self.get_next_value(data, "Size"),
|
||||
# tank_location=self.get_next_value(data, "Tank Location"),
|
||||
# is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False,
|
||||
# type_of_insulation=self.get_next_value(data, "Type of Insulation"),
|
||||
# thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")),
|
||||
)
|
||||
|
||||
def get_general_condition_of_heating_system(self):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
import os
|
||||
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -232,5 +235,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2"
|
||||
file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "NCHA Derbyshire Dales (DDDCC) SHDF"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -198,5 +200,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Northumberland LAD2 & HUG2"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -149,13 +151,13 @@ full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Fu
|
|||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC")
|
||||
post_epc = get_df(df, "post-epc epr status", ["uploaded", "completed"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
df["post-epc epr status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
|
@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Shropshire Council HUG 2"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -196,5 +198,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Stonewater SHDF 2.0 - Operations"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -202,5 +204,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Decent Homes Stonewater - Operations"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -210,5 +212,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False)
|
||||
master_folder_name = "WCHG SHDF 2.1 Mansard"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
|
|||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
import os
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "NCHA SHDF Wave 3 On Hold"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
|
|
@ -230,5 +231,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Platform Housing W2 (in use)"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -237,5 +238,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "Stonewater - (in use)"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -229,5 +230,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "ECO 4 NCHA Almshouse Operation"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
|
|
@ -205,3 +206,19 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
|||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False)
|
||||
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
master_folder_name = "ECO 4 Wates Operations"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -195,7 +196,7 @@ post_epr["job_type"] = "POST epr"
|
|||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT")
|
||||
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
|
|
@ -236,6 +237,7 @@ rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower(
|
|||
rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower()
|
||||
rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower()
|
||||
|
||||
|
||||
# Now perform the merge
|
||||
net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)]
|
||||
sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)]
|
||||
|
|
@ -247,11 +249,41 @@ combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_me
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False)
|
||||
combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False)
|
||||
combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False)
|
||||
|
||||
master_folder_name = "Home Group"
|
||||
|
||||
# TO DO check everything in excel
|
||||
# make logic for seperation
|
||||
# Wave 2SP+
|
||||
file_name = f"{master_folder_name}_Wave 2SP+_{timestamp}.xlsx"
|
||||
combined_with_rates_sp_plus[attribute].to_excel(file_name, index=False)
|
||||
file_path = os.path.abspath(file_name)
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
invoice_name = "wave_2sp_plus_rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df_sp_plus.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
||||
|
||||
# Net Zero
|
||||
file_name = f"{master_folder_name}_NetZero_{timestamp}.xlsx"
|
||||
combined_with_rates_net_zero_df[attribute].to_excel(file_name, index=False)
|
||||
file_path = os.path.abspath(file_name)
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
invoice_name = "net_zero_rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df_net_zero.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
||||
# Unsure
|
||||
file_name = f"{master_folder_name}_unsure_who_to_bill_{timestamp}.xlsx"
|
||||
combined_with_rates_other_from_home_group[attribute].to_excel(file_name, index=False)
|
||||
file_path = os.path.abspath(file_name)
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
invoice_name = "unsure_who_to_bill_rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df_error_message.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -214,5 +216,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "NCHA SHDF 2.1 SBS"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Wave 2's month end automation
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False)
|
||||
master_folder_name = "NCHA Almshouses"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -6,18 +6,37 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
|
|||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
import os
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
from enum import Enum
|
||||
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
||||
class BoardID(Enum):
|
||||
_9349630181 = "WCHG_WALKUPS_OPERATIONS"
|
||||
_8830772914 = "LQ_LONDON"
|
||||
_9601691730 = "CARDO_WALES_WEST_WAVE3"
|
||||
_9660895490 = "NORTHUMBERLAND_COUNTY_SHDF_WAVE3"
|
||||
_9641491000 = "WATFORD_WARM_HOMES"
|
||||
_9671463094 = "SEDDON"
|
||||
|
||||
|
||||
board_ids = [
|
||||
# "9349630181", # WCHG Walkups-Operations
|
||||
# "8830772914", # "L&Q London"
|
||||
# "9601691730", # Cardo Wales & West - Wave 3
|
||||
# "9660895490", # Northumberland County SHDF Wave 3
|
||||
"9641491000", #Watford Warm Homes
|
||||
"9349630181", # WCHG Walkups-Operations x
|
||||
"8830772914", # "L&Q London" x
|
||||
"9601691730", # Cardo Wales & West - Wave 3 x
|
||||
"9660895490", # Northumberland County SHDF Wave 3 x
|
||||
"9641491000", # Watford Warm Homes x
|
||||
"9671463094", # Seddon
|
||||
]
|
||||
|
||||
empty = "Rate card info missing"
|
||||
|
||||
rate_cards = {}
|
||||
|
||||
|
||||
rate_card_data_watford_warm_homes = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
|
|
@ -32,22 +51,9 @@ rate_card_data_watford_warm_homes = {
|
|||
empty, empty, empty, empty, empty,
|
||||
]
|
||||
}
|
||||
rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes)})
|
||||
|
||||
|
||||
rate_card_data_2502_accent_housing = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_data_l_and_q_london = {
|
||||
"job_type": [
|
||||
|
|
@ -64,22 +70,27 @@ rate_card_data_l_and_q_london = {
|
|||
]
|
||||
}
|
||||
|
||||
rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)})
|
||||
|
||||
|
||||
rate_card_data_northhumberland_country_shdf_wave_3 = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 130,
|
||||
259, 125, "280 or 310 depending on SH or LA", 125, 125,
|
||||
650, 415, 195, 225, 175,
|
||||
135, 120, "60 - check", "85 or 90 depending on SH or LA", "110 or 125 depending on SH or LA",
|
||||
60, 45, 45, 45, empty, empty
|
||||
]
|
||||
}
|
||||
|
||||
pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
|
||||
rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)})
|
||||
|
||||
rate_card_data_walk_ups = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
|
|
@ -95,14 +106,59 @@ rate_card_data_walk_ups = {
|
|||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_example)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_walk_ups)
|
||||
rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)})
|
||||
|
||||
rate_card_data_empty = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty,empty , empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_data_cardo = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 260, 125, 125,
|
||||
415, 415, 225, 225, 175,
|
||||
135, 120 , "60 - check", 85, 125,
|
||||
60, 45, 45, 45, empty, empty,
|
||||
]
|
||||
}
|
||||
|
||||
rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)})
|
||||
|
||||
rate_card_data_seddon = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 225, 175,
|
||||
135, 120 , "60 - check", 85, 125,
|
||||
60, 45, 45, 45, empty, empty,
|
||||
]
|
||||
}
|
||||
|
||||
rate_cards.update({"9671463094": pd.DataFrame(rate_card_data_seddon)})
|
||||
|
||||
|
||||
board_to_record = {}
|
||||
for board in tqdm(board_ids):
|
||||
print(f"working on board {board}")
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
|
|
@ -133,150 +189,169 @@ for board in tqdm(board_ids):
|
|||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
board_to_record.update({board: all_records})
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
for board, all_records in board_to_record.items():
|
||||
df = pd.DataFrame(all_records)
|
||||
filtered_dfs = []
|
||||
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = pd.DataFrame()
|
||||
if column_name in col_id_map:
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
|
||||
|
||||
return _
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = pd.DataFrame()
|
||||
if column_name in col_id_map:
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
|
||||
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra invoicing status", ["to invoice"], "RA")
|
||||
if not ra.empty:
|
||||
filtered_dfs.append(ra)
|
||||
# RA
|
||||
ra = get_df(df, "ra invoicing status", ["to invoice"], "RA")
|
||||
if not ra.empty:
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
att = get_df(df, "post att invoicing status", ["to invoice"], "ATT")
|
||||
if not att.empty:
|
||||
filtered_dfs.append(att)
|
||||
att = get_df(df, "post att invoicing status", ["to invoice"], "ATT")
|
||||
if not att.empty:
|
||||
filtered_dfs.append(att)
|
||||
|
||||
modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling")
|
||||
if not modeling.empty:
|
||||
filtered_dfs.append(modeling)
|
||||
modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling")
|
||||
if not modeling.empty:
|
||||
filtered_dfs.append(modeling)
|
||||
|
||||
try:
|
||||
# Only needed for one board in wave 3
|
||||
full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp")
|
||||
if not full_cost.empty:
|
||||
filtered_dfs(full_cost)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
try:
|
||||
# Only needed for one board in wave 3
|
||||
full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp")
|
||||
if not full_cost.empty:
|
||||
filtered_dfs(full_cost)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1")
|
||||
if not v1.empty:
|
||||
filtered_dfs.append(v1)
|
||||
v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1")
|
||||
if not v1.empty:
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel")
|
||||
if not v2.empty:
|
||||
filtered_dfs.append(v2)
|
||||
v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel")
|
||||
if not v2.empty:
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel")
|
||||
if not v3.empty:
|
||||
filtered_dfs.append(v3)
|
||||
v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel")
|
||||
if not v3.empty:
|
||||
filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2")
|
||||
if not cors2.empty:
|
||||
filtered_dfs.append(cors2)
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2")
|
||||
if not cors2.empty:
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design archetype complex
|
||||
design = get_df(df, "design invoicing status", ["to invoice"])
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
|
||||
if not design1.empty :
|
||||
filtered_dfs.append(design1)
|
||||
# Design archetype complex
|
||||
design = get_df(df, "design invoicing status", ["to invoice"])
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
|
||||
if not design1.empty :
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design archetype simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
# Design archetype simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
# Design repetitive simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive complex
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
# Design repetitive complex
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Revision
|
||||
revision_letter = ['a', 'b', 'c', 'd']
|
||||
for letter in revision_letter:
|
||||
design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision")
|
||||
if not design.empty:
|
||||
filtered_dfs.append(design)
|
||||
# Design Revision
|
||||
revision_letter = ['a', 'b', 'c', 'd']
|
||||
for letter in revision_letter:
|
||||
design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision")
|
||||
if not design.empty:
|
||||
filtered_dfs.append(design)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
|
||||
if not lodg1.empty:
|
||||
filtered_dfs(lodg1)
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
|
||||
if not lodg1.empty:
|
||||
filtered_dfs(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2")
|
||||
if not lodg2.empty:
|
||||
filtered_dfs.append(lodg2)
|
||||
# Full Lodgement Phase
|
||||
lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2")
|
||||
if not lodg2.empty:
|
||||
filtered_dfs.append(lodg2)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC")
|
||||
if not post_epc.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC")
|
||||
if not post_epc.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR")
|
||||
if not post_epr.empty:
|
||||
filtered_dfs.append(post_epr)
|
||||
# POST EPR
|
||||
post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR")
|
||||
if not post_epr.empty:
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# post att
|
||||
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
|
||||
if not post_att.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
# post att
|
||||
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
|
||||
if not post_att.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
# Retrofit Evaluation
|
||||
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation")
|
||||
if not rc.empty:
|
||||
filtered_dfs.append(rc)
|
||||
# Retrofit Evaluation
|
||||
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation")
|
||||
if not rc.empty:
|
||||
filtered_dfs.append(rc)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW")
|
||||
if not ra_ns.empty:
|
||||
filtered_dfs.append(ra_ns)
|
||||
# RA NO Show
|
||||
ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW")
|
||||
if not ra_ns.empty:
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW")
|
||||
if not att_ns.empty:
|
||||
filtered_dfs.append(att_ns)
|
||||
# ATT NO Show
|
||||
att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW")
|
||||
if not att_ns.empty:
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW")
|
||||
if not epc_ns.empty:
|
||||
filtered_dfs.append(epc_ns)
|
||||
# Post visit no show
|
||||
epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW")
|
||||
if not epc_ns.empty:
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
if len(filtered_dfs) != 0:
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df = rate_cards[board]
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False)
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
master_folder_name = BoardID[f"_{board}"].value
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
# Wave 3's month end automation
|
||||
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
from MonthEndUploader import upload_to_month_end_folder
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
|
|
@ -88,7 +89,7 @@ if not second_payment.empty:
|
|||
|
||||
full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp")
|
||||
if not full_cost.empty:
|
||||
filtered_dfs(full_cost)
|
||||
filtered_dfs.append(full_cost)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
|
|
@ -100,5 +101,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
|||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
# Upload to sharepoint
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False)
|
||||
master_folder_name = "2501 Accent housing"
|
||||
file_name = f"{master_folder_name}_{timestamp}.xlsx"
|
||||
combined_with_rates[attribute].to_excel(file_name, index=False)
|
||||
|
||||
file_path = os.path.abspath(file_name)
|
||||
|
||||
upload_to_month_end_folder(file_name, file_path, master_folder_name)
|
||||
|
||||
invoice_name = "rate_card.xlsx"
|
||||
file_path = os.path.abspath(invoice_name)
|
||||
rate_card_df.to_excel(invoice_name, index=False)
|
||||
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ class VentilationInfo(BaseModel):
|
|||
are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room: bool
|
||||
are_there_sufficient_undercuts_on_the_closed_door: str
|
||||
is_there_any_open_flue_heating_appliances_within_the_room: bool
|
||||
location_of_any_damp_or_mould: Optional[str] = ""
|
||||
what_severity_of_damp_mould_would_you_consider_this_to_be: Optional[str] = ""
|
||||
|
||||
class WindowsInfo(BaseModel):
|
||||
does_the_room_have_any_windows: bool
|
||||
|
|
@ -104,6 +106,7 @@ class WindowsInfo(BaseModel):
|
|||
class RoomInfo(BaseModel):
|
||||
overall_condition_of_the_room: str
|
||||
does_the_room_have_any_defects: str
|
||||
description_of_defect: Optional[str] = ""
|
||||
are_there_any_sloped_ceiling_areas: Optional[bool] = None
|
||||
windows_info: Optional[WindowsInfo] = None
|
||||
ventilation_info: Optional[VentilationInfo] = None
|
||||
|
|
@ -147,16 +150,15 @@ class Bathroom(BaseModel):
|
|||
class LoftSpace(BaseModel):
|
||||
is_the_main_loft_space_accessible: str
|
||||
is_there_more_than_one_loft_space: bool
|
||||
overall_condition_of_the_loft_space: Optional[str]
|
||||
are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str]
|
||||
does_the_loft_space_have_any_defects: Optional[str]
|
||||
existing_depth_of_loft_insulation: Optional[str]
|
||||
is_the_insulation_layer_even_across_the_loft: Optional[str]
|
||||
is_the_loft_boarded_in_any_area: Optional[str]
|
||||
condition_of_existing_roof_lining: Optional[str]
|
||||
is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str]
|
||||
is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str]
|
||||
is_it_accessible: Optional[str]
|
||||
overall_condition_of_the_loft_space: Optional[str] = ""
|
||||
are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] = ""
|
||||
does_the_loft_space_have_any_defects: Optional[str] = ""
|
||||
existing_depth_of_loft_insulation: Optional[str] = ""
|
||||
is_the_insulation_layer_even_across_the_loft: Optional[str] = ""
|
||||
is_the_loft_boarded_in_any_area: Optional[str] = ""
|
||||
condition_of_existing_roof_lining: Optional[str] = ""
|
||||
is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] = ""
|
||||
is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] = ""
|
||||
|
||||
|
||||
|
||||
|
|
@ -213,12 +215,12 @@ class Renewables(BaseModel):
|
|||
is_there_any_renewable_energy_system_in_place: bool
|
||||
suitable_roof_orientation_for_solar_pv_water: str
|
||||
is_there_a_water_tank: bool
|
||||
type: str
|
||||
size: str
|
||||
tank_location: str
|
||||
is_the_tank_insulated: bool
|
||||
type_of_insulation: str
|
||||
thickness_of_insulation_in_mm: int
|
||||
# type: str
|
||||
# size: str
|
||||
# tank_location: str
|
||||
# is_the_tank_insulated: bool
|
||||
# type_of_insulation: str
|
||||
# thickness_of_insulation_in_mm: int
|
||||
|
||||
class HeatingSystem(BaseModel):
|
||||
general_condition: GeneralConditionHeatingSystem
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue