From 9419175ca539d04279e73bb1d6bea504af75aecc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 14:34:54 +0100 Subject: [PATCH 01/12] make it work for more example --- .../lambda/extractor_and_loader/docker/app.py | 3 +- etl/fileReader/sitenotes.py | 66 +++++++++++-------- etl/transform/conditionReportTypes.py | 31 +++++---- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 8a08c48..7d9dde8 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -203,12 +203,13 @@ def handler(event, context): local_path = download_private_s3_file(file_uri) # Local development of file, please comment out for prod - # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf")) print("Extracting file...") reader = pdfReaderToText(local_path) + # obj2 = WarmHomesConditionReport(reader.text_list, debug=True) obj = WarmHomesConditionReport(reader.text_list) json_ = make_final_json( obj.master_obj[0], diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index f2de52d..d583f56 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -102,12 +102,12 @@ class ECOConditionReport(SiteNotesExtractor): class WarmHomesConditionReport(SiteNotesExtractor): - def __init__(self, data_list): + def __init__(self, data_list, debug = False): super().__init__(data_list) self.type = ReportType.WARM_HOMES_CONDITION_REPORT - room, heating_system, occupant, access_and_elevation = self.setup_condition_report() - - self.master_obj = room, heating_system, occupant, access_and_elevation + if debug is False: + room, heating_system, occupant, access_and_elevation = self.setup_condition_report() + self.master_obj = room, heating_system, occupant, access_and_elevation def setup_condition_report(self): # general_information = self.get_section_1() @@ -292,7 +292,7 @@ class WarmHomesConditionReport(SiteNotesExtractor): # Gable 2 data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding") - state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False + state = False # TODO quick demo for waltham forest, set to always false if state is False: gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state) else: @@ -378,10 +378,16 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_hallway(self): data = self.get_data_between("Hallway", "Living Room") - room = self.get_room_info(data) + + hallway_state = True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False + + if hallway_state: + room = self.get_room_info(data) + else: + room = None return Hallway( - is_there_a_hallway=True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False, + is_there_a_hallway=hallway_state, room_info=room, ) @@ -494,20 +500,26 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_loft_space(self): data = self.get_data_between("Loft Space", "Room in Roof") - return LoftSpace( - is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), - is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, - overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), - are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), - does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), - existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), - is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), - is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), - condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), - is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), - is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), - is_it_accessible=self.get_next_value(data, "Is it accessible?"), - ) + state = True if self.get_next_value(data, "Is the main loft space accessible?").lower() == "yes" else False + if state: + return LoftSpace( + is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), + is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), + does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), + is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), + is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), + condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), + is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), + ) + else: + return LoftSpace( + is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), + is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + ) def get_room_in_roof(self): data = self.get_data_between("Room in Roof", "4. Heating System") @@ -596,12 +608,12 @@ class WarmHomesConditionReport(SiteNotesExtractor): is_there_any_renewable_energy_system_in_place=True if self.get_next_value(data, "Is there any renewable energy system in place?").lower() == "yes" else False, suitable_roof_orientation_for_solar_pv_water=self.get_next_value(data, "Suitable roof orientation for Solar PV/water:"), is_there_a_water_tank=True if self.get_next_value(data, "Is there a Water Tank?").lower() == "yes" else False, - type=self.get_next_value(data, "Type"), - size=self.get_next_value(data, "Size"), - tank_location=self.get_next_value(data, "Tank Location"), - is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False, - type_of_insulation=self.get_next_value(data, "Type of Insulation"), - thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")), + # type=self.get_next_value(data, "Type"), + # size=self.get_next_value(data, "Size"), + # tank_location=self.get_next_value(data, "Tank Location"), + # is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False, + # type_of_insulation=self.get_next_value(data, "Type of Insulation"), + # thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")), ) def get_general_condition_of_heating_system(self): diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index ee3c091..dea3912 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -147,16 +147,15 @@ class Bathroom(BaseModel): class LoftSpace(BaseModel): is_the_main_loft_space_accessible: str is_there_more_than_one_loft_space: bool - overall_condition_of_the_loft_space: Optional[str] - are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] - does_the_loft_space_have_any_defects: Optional[str] - existing_depth_of_loft_insulation: Optional[str] - is_the_insulation_layer_even_across_the_loft: Optional[str] - is_the_loft_boarded_in_any_area: Optional[str] - condition_of_existing_roof_lining: Optional[str] - is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] - is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] - is_it_accessible: Optional[str] + overall_condition_of_the_loft_space: Optional[str] = "" + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] = "" + does_the_loft_space_have_any_defects: Optional[str] = "" + existing_depth_of_loft_insulation: Optional[str] = "" + is_the_insulation_layer_even_across_the_loft: Optional[str] = "" + is_the_loft_boarded_in_any_area: Optional[str] = "" + condition_of_existing_roof_lining: Optional[str] = "" + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] = "" + is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] = "" @@ -213,12 +212,12 @@ class Renewables(BaseModel): is_there_any_renewable_energy_system_in_place: bool suitable_roof_orientation_for_solar_pv_water: str is_there_a_water_tank: bool - type: str - size: str - tank_location: str - is_the_tank_insulated: bool - type_of_insulation: str - thickness_of_insulation_in_mm: int + # type: str + # size: str + # tank_location: str + # is_the_tank_insulated: bool + # type_of_insulation: str + # thickness_of_insulation_in_mm: int class HeatingSystem(BaseModel): general_condition: GeneralConditionHeatingSystem From 33f03644d59ac23aca641e9f34ce067975d7fa6d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 15:25:33 +0100 Subject: [PATCH 02/12] new lambda dep[loy --- .../lambda/extractor_and_loader/docker/app.py | 8 ++-- etl/fileReader/sitenotes.py | 46 ++++++++++++++----- etl/transform/conditionReportTypes.py | 2 + 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 7d9dde8..b5577e5 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -57,7 +57,7 @@ def serialize_model(model: Any): else: return model -def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations): +def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations, bepoke_info): # Convert to dict recursively rooms_data = serialize_model(rooms_obj) heating_data = serialize_model(heating_system_obj) @@ -70,6 +70,7 @@ def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevatio "heating_system": heating_data, "occupant_info": occupant_data, "access_and_elevations": access_and_elevations_data, + "bespoke_data": bepoke_info } # Convert to pretty JSON string @@ -203,19 +204,20 @@ def handler(event, context): local_path = download_private_s3_file(file_uri) # Local development of file, please comment out for prod - # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf")) + local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) print("Extracting file...") reader = pdfReaderToText(local_path) - # obj2 = WarmHomesConditionReport(reader.text_list, debug=True) + obj2 = WarmHomesConditionReport(reader.text_list, debug=True) obj = WarmHomesConditionReport(reader.text_list) json_ = make_final_json( obj.master_obj[0], obj.master_obj[1], obj.master_obj[2], obj.master_obj[3], + {} ) print("Extracted completed, made json") diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index d583f56..fbb267e 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -347,12 +347,23 @@ class WarmHomesConditionReport(SiteNotesExtractor): ) def get_room_info(self, data): - ventilation = VentilationInfo( - is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, - are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, - are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), - is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, - ) + state = True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False + if state is False: + ventilation = VentilationInfo( + is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, + are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, + are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), + is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, + ) + else: + ventilation = VentilationInfo( + is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, + are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, + are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), + is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, + location_of_any_damp_or_mould=self.get_next_value(data, "Location of any damp/mould") + ) + windows_state = True if self.get_next_value(data, "Does the room have any windows?").lower() == "yes" else False if windows_state: @@ -368,12 +379,23 @@ class WarmHomesConditionReport(SiteNotesExtractor): does_the_room_have_any_windows=windows_state ) - return RoomInfo( - overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), - does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), - windows_info=windows, - ventilation_info=ventilation, - ) + state = True if self.get_next_value(data, "cracking in walls, etc.)").lower()=="yes" else False + if state: + return RoomInfo( + overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), + does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + description_of_defect=self.get_next_value(data, "Description of the defects", avoid=["Any additional recommendations?"]), + windows_info=windows, + ventilation_info=ventilation, + ) + else: + return RoomInfo( + overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), + does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + windows_info=windows, + ventilation_info=ventilation, + ) + def get_hallway(self): diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index dea3912..ee51b7c 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -93,6 +93,7 @@ class VentilationInfo(BaseModel): are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room: bool are_there_sufficient_undercuts_on_the_closed_door: str is_there_any_open_flue_heating_appliances_within_the_room: bool + location_of_any_damp_or_mould: Optional[str] = "" class WindowsInfo(BaseModel): does_the_room_have_any_windows: bool @@ -104,6 +105,7 @@ class WindowsInfo(BaseModel): class RoomInfo(BaseModel): overall_condition_of_the_room: str does_the_room_have_any_defects: str + description_of_defect: Optional[str] = "" are_there_any_sloped_ceiling_areas: Optional[bool] = None windows_info: Optional[WindowsInfo] = None ventilation_info: Optional[VentilationInfo] = None From b12e393b3b8b0edd2ee2515d3f50b143605c2ae1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 15:27:41 +0100 Subject: [PATCH 03/12] added proper lambda job --- deployment/lambda/extractor_and_loader/docker/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index b5577e5..4bf5d93 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -204,13 +204,17 @@ def handler(event, context): local_path = download_private_s3_file(file_uri) # Local development of file, please comment out for prod - local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) print("Extracting file...") reader = pdfReaderToText(local_path) - obj2 = WarmHomesConditionReport(reader.text_list, debug=True) + # obj2 = WarmHomesConditionReport(reader.text_list, debug=True) obj = WarmHomesConditionReport(reader.text_list) json_ = make_final_json( obj.master_obj[0], From 2331105f965b7d739bb74fe98266ad442d7cedf5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 26 Aug 2025 19:50:42 +0000 Subject: [PATCH 04/12] added severity --- deployment/lambda/extractor_and_loader/docker/app.py | 4 ++-- etl/fileReader/sitenotes.py | 3 ++- etl/transform/conditionReportTypes.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 4bf5d93..1502ef5 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -204,11 +204,11 @@ def handler(event, context): local_path = download_private_s3_file(file_uri) # Local development of file, please comment out for prod - # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1 1.pdf")) # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf")) # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf")) # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf")) - # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/6E-plantagenet-street.pdf")) diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index fbb267e..8f087e2 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -361,7 +361,8 @@ class WarmHomesConditionReport(SiteNotesExtractor): are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, - location_of_any_damp_or_mould=self.get_next_value(data, "Location of any damp/mould") + location_of_any_damp_or_mould=self.get_next_value(data, "Location of any damp/mould"), + what_severity_of_damp_mould_would_you_consider_this_to_be=self.get_next_value(data, "What severity of damp/mould would you consider this to be?"), ) diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index ee51b7c..7bfdebf 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -94,6 +94,7 @@ class VentilationInfo(BaseModel): are_there_sufficient_undercuts_on_the_closed_door: str is_there_any_open_flue_heating_appliances_within_the_room: bool location_of_any_damp_or_mould: Optional[str] = "" + what_severity_of_damp_mould_would_you_consider_this_to_be: Optional[str] = "" class WindowsInfo(BaseModel): does_the_room_have_any_windows: bool From f55e8f7a6486765db6049cb3e2decd4f1d27598c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 Aug 2025 09:34:54 +0000 Subject: [PATCH 05/12] added files for modifgication --- etl/MonthEndUploader.py | 21 +++++++++++++++++++++ etl/month_end_automation_wave_2_layout.py | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 etl/MonthEndUploader.py diff --git a/etl/MonthEndUploader.py b/etl/MonthEndUploader.py new file mode 100644 index 0000000..4a7231e --- /dev/null +++ b/etl/MonthEndUploader.py @@ -0,0 +1,21 @@ +import os +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" +from etl.scraper.scraper import SharePointScraper, SharePointInstaller +import datetime + + +def upload_to_month_end_folder(file_name_on_sp, local_file_path): + sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) + + parent_folder = "General/Junte Kim/month end" + today = datetime.today() + # Format as "Month YYYY" + formatted_date = today.strftime("%B %Y") + + sharepoint.create_dir(formatted_date, parent_folder) + sharepoint_path = parent_folder + "/" + formatted_date + print("Uploading to sharepoint...") + sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp) + print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}") diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 7dca011..22fe724 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -1,11 +1,13 @@ -# Wave 2's month end automation +# Wave 2's month end automation from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder + monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) From d0fa92b0d1cab925c5f050aee83d1ec6abd73e2b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 Aug 2025 11:17:42 +0000 Subject: [PATCH 06/12] added month end automation --- .github/workflows/months_end.yml | 31 +++++++++++++++++++++++ etl/MonthEndUploader.py | 7 +++-- etl/month_end_automation_wave_2_layout.py | 15 ++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/months_end.yml diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml new file mode 100644 index 0000000..9185ecc --- /dev/null +++ b/.github/workflows/months_end.yml @@ -0,0 +1,31 @@ +name: Months End +on: + schedule: + # - cron: '0 17 * * 1-5' + push: + branches: [main, feature/month_end_automation_of_all] + workflow_dispatch: + +jobs: + surveyed-needs-sign-off: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: run script + run: | + pwd + ls -la + poetry run python etl/month_end_automation_wave_2_layout.py + env: + PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/MonthEndUploader.py b/etl/MonthEndUploader.py index 4a7231e..927b5c0 100644 --- a/etl/MonthEndUploader.py +++ b/etl/MonthEndUploader.py @@ -3,10 +3,10 @@ os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" from etl.scraper.scraper import SharePointScraper, SharePointInstaller -import datetime +from datetime import datetime -def upload_to_month_end_folder(file_name_on_sp, local_file_path): +def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path): sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) parent_folder = "General/Junte Kim/month end" @@ -16,6 +16,9 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path): sharepoint.create_dir(formatted_date, parent_folder) sharepoint_path = parent_folder + "/" + formatted_date + sharepoint.create_dir(add_to_path, sharepoint_path) + sharepoint_path += "/" + add_to_path + print("Uploading to sharepoint...") sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp) print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}") diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 22fe724..e6b4ff2 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -7,6 +7,7 @@ from pprint import pprint import pandas as pd import json from MonthEndUploader import upload_to_month_end_folder +import os monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" @@ -234,5 +235,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False) \ No newline at end of file +master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2" +file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) \ No newline at end of file From 4185a610b0205d64a8f51bcb133f27dab65e0d28 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 Aug 2025 11:23:55 +0000 Subject: [PATCH 07/12] added month end automation --- .github/workflows/months_end.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 9185ecc..e316b3d 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -1,7 +1,7 @@ name: Months End on: - schedule: - # - cron: '0 17 * * 1-5' + # schedule: + # # - cron: '0 17 * * 1-5' push: branches: [main, feature/month_end_automation_of_all] workflow_dispatch: From 7bfabf9e5c16ea20f09ee5678f5ff364df673c5a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 Aug 2025 15:37:01 +0000 Subject: [PATCH 08/12] save workflow invoicing so far --- .github/workflows/months_end.yml | 12 +++++++ etl/month_end_automation_wave_2_no_10.py | 16 +++++++++- etl/month_end_automation_wave_2_no_11.py | 16 +++++++++- etl/month_end_automation_wave_2_no_12.py | 20 ++++++++++-- etl/month_end_automation_wave_2_no_13.py | 16 +++++++++- etl/month_end_automation_wave_2_no_3.py | 17 ++++++++-- etl/month_end_automation_wave_2_no_4.py | 18 +++++++++-- etl/month_end_automation_wave_2_no_5.py | 17 ++++++++-- etl/month_end_automation_wave_2_no_6.py | 19 ++++++++++- etl/month_end_automation_wave_2_no_7.py | 40 +++++++++++++++++++++--- etl/month_end_automation_wave_2_no_8.py | 17 +++++++++- etl/month_end_automation_wave_2_no_9.py | 16 +++++++++- 12 files changed, 205 insertions(+), 19 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index e316b3d..758045b 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -2,6 +2,7 @@ name: Months End on: # schedule: # # - cron: '0 17 * * 1-5' + #adslfhjk asdlfjldsakjf push: branches: [main, feature/month_end_automation_of_all] workflow_dispatch: @@ -27,5 +28,16 @@ jobs: pwd ls -la poetry run python etl/month_end_automation_wave_2_layout.py + poetry run python etl/month_end_automation_wave_2_no_3.py + poetry run python etl/month_end_automation_wave_2_no_4.py + poetry run python etl/month_end_automation_wave_2_no_5.py + poetry run python etl/month_end_automation_wave_2_no_6.py + poetry run python etl/month_end_automation_wave_2_no_7.py + poetry run python etl/month_end_automation_wave_2_no_8.py + poetry run python etl/month_end_automation_wave_2_no_9.py + poetry run python etl/month_end_automation_wave_2_no_10.py + poetry run python etl/month_end_automation_wave_2_no_11.py + poetry run python etl/month_end_automation_wave_2_no_12.py + poetry run python etl/month_end_automation_wave_2_no_13.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index d854fdc..9b7b3e7 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA Derbyshire Dales (DDDCC) SHDF" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 41f4438..874eaa3 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -198,5 +200,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False) +master_folder_name = "Northumberland LAD2 & HUG2" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 7fa4c1a..464d211 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -149,13 +151,13 @@ full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Fu filtered_dfs.append(full_lodgement) # POST EPC -post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +post_epc = get_df(df, "post-epc epr status", ["uploaded", "completed"], "POST EPC") filtered_dfs.append(post_epc) # POST EPR post_epr = df[ - df["post-epc status"].str.lower().isin(["post epr completed"]) + df["post-epc epr status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) @@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False) +master_folder_name = "Shropshire Council HUG 2" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index d4a6a1d..7637222 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -196,5 +198,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False) +master_folder_name = "Stonewater SHDF 2.0 - Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index b349a7a..2500d25 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -1,7 +1,8 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient +from MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd @@ -230,5 +231,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False) +master_folder_name = "Platform Housing W2 (in use)" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 30b949f..1b7f6f4 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -237,5 +238,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False) +master_folder_name = "Stonewater - (in use)" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index f26d3b0..c26834b 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -229,5 +230,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False) +master_folder_name = "ECO 4 NCHA Almshouse Operation" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 0ad996f..8091aa1 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -1,7 +1,8 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient +from MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd @@ -205,3 +206,19 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False) + + +# Upload to sharepoint +attribute = ['address', 'client', 'job_type', 'rate'] +master_folder_name = "ECO 4 Wates Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 214c233..e1fbf9c 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -195,7 +196,7 @@ post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT -post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT") +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") filtered_dfs.append(post_att) @@ -236,6 +237,7 @@ rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower( rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower() rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower() + # Now perform the merge net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)] sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)] @@ -247,11 +249,41 @@ combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_me import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) +master_folder_name = "Home Group" -# TO DO check everything in excel -# make logic for seperation \ No newline at end of file +# Wave 2SP+ +file_name = f"{master_folder_name}_Wave 2SP+_{timestamp}.xlsx" +combined_with_rates_sp_plus[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "wave_2sp_plus_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_sp_plus.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) + + +# Net Zero +file_name = f"{master_folder_name}_NetZero_{timestamp}.xlsx" +combined_with_rates_net_zero_df[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "net_zero_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_net_zero.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) + +# Unsure +file_name = f"{master_folder_name}_unsure_who_to_bill_{timestamp}.xlsx" +combined_with_rates_other_from_home_group[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "unsure_who_to_bill_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_error_message.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 0aff6fc..d465dd6 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -214,5 +216,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA SHDF 2.1 SBS" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index 15bf2fb..7d92c72 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA Almshouses" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) From f3abeeb1eb6352166d1b7936e03ae104f325691e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 29 Aug 2025 12:03:15 +0000 Subject: [PATCH 09/12] month end --- .github/workflows/months_end.yml | 4 ++++ etl/month_end_automation_wave_2_no_14.py | 16 +++++++++++++++- etl/month_end_automation_wave_2_no_15.py | 16 +++++++++++++++- etl/month_end_automation_wave_2_no_16.py | 16 +++++++++++++++- ...onth_end_automation_wave_accent_housing.py | 19 ++++++++++++++++--- 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 758045b..80955ad 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -39,5 +39,9 @@ jobs: poetry run python etl/month_end_automation_wave_2_no_11.py poetry run python etl/month_end_automation_wave_2_no_12.py poetry run python etl/month_end_automation_wave_2_no_13.py + poetry run python etl/month_end_automation_wave_2_no_14.py + poetry run python etl/month_end_automation_wave_2_no_15.py + poetry run python etl/month_end_automation_wave_2_no_16.py + poetry run python etl/month_end_automation_wave_accent_housing.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 27cadd0..1099185 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -202,5 +204,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False) +master_folder_name = "Decent Homes Stonewater - Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 8644db9..1b10f25 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -210,5 +212,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) +master_folder_name = "WCHG SHDF 2.1 Mansard" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 0af8bfe..664ebff 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -5,6 +5,8 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False) \ No newline at end of file +master_folder_name = "NCHA SHDF Wave 3 On Hold" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index e02cdd9..b13d155 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -1,11 +1,12 @@ # Wave 3's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -88,7 +89,7 @@ if not second_payment.empty: full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp") if not full_cost.empty: - filtered_dfs(full_cost) + filtered_dfs.append(full_cost) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -100,5 +101,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False) +master_folder_name = "2501 Accent housing" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) From 7d92cc9f3493873248f4071615fd95e68fd1f296 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 29 Aug 2025 15:40:30 +0000 Subject: [PATCH 10/12] files autoamted --- .github/workflows/months_end.yml | 8 +- etl/month_end_automation_wave_3_layout.py | 359 +++++++++++++--------- 2 files changed, 221 insertions(+), 146 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 80955ad..f014aa0 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -1,10 +1,9 @@ name: Months End on: - # schedule: - # # - cron: '0 17 * * 1-5' - #adslfhjk asdlfjldsakjf + schedule: + - cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month push: - branches: [main, feature/month_end_automation_of_all] + branches: [main] workflow_dispatch: jobs: @@ -43,5 +42,6 @@ jobs: poetry run python etl/month_end_automation_wave_2_no_15.py poetry run python etl/month_end_automation_wave_2_no_16.py poetry run python etl/month_end_automation_wave_accent_housing.py + poetry run python etl/month_end_automation_wave_3_layout.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 704668c..32125a4 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -6,18 +6,37 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder +from enum import Enum + monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) + +class BoardID(Enum): + _9349630181 = "WCHG_WALKUPS_OPERATIONS" + _8830772914 = "LQ_LONDON" + _9601691730 = "CARDO_WALES_WEST_WAVE3" + _9660895490 = "NORTHUMBERLAND_COUNTY_SHDF_WAVE3" + _9641491000 = "WATFORD_WARM_HOMES" + _9671463094 = "SEDDON" + + board_ids = [ - # "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - "9641491000", #Watford Warm Homes + "9349630181", # WCHG Walkups-Operations x + "8830772914", # "L&Q London" x + "9601691730", # Cardo Wales & West - Wave 3 x + "9660895490", # Northumberland County SHDF Wave 3 x + "9641491000", # Watford Warm Homes x + "9671463094", # Seddon ] empty = "Rate card info missing" + +rate_cards = {} + + rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -32,22 +51,9 @@ rate_card_data_watford_warm_homes = { empty, empty, empty, empty, empty, ] } +rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes)}) -rate_card_data_2502_accent_housing = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} rate_card_data_l_and_q_london = { "job_type": [ @@ -64,22 +70,27 @@ rate_card_data_l_and_q_london = { ] } +rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)}) + rate_card_data_northhumberland_country_shdf_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" ], "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 130, + 259, 125, "280 or 310 depending on SH or LA", 125, 125, + 650, 415, 195, 225, 175, + 135, 120, "60 - check", "85 or 90 depending on SH or LA", "110 or 125 depending on SH or LA", + 60, 45, 45, 45, empty, empty ] } +pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)}) + rate_card_data_walk_ups = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -95,14 +106,59 @@ rate_card_data_walk_ups = { ] } -rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes) -# rate_card_df = pd.DataFrame(rate_card_data_example) -# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) -# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -# rate_card_df = pd.DataFrame(rate_card_data_walk_ups) +rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)}) + +rate_card_data_empty = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty,empty , empty, empty, empty, + empty, empty, empty, empty, empty, + ] +} + +rate_card_data_cardo = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 260, 125, 125, + 415, 415, 225, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)}) + +rate_card_data_seddon = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9671463094": pd.DataFrame(rate_card_data_seddon)}) +board_to_record = {} for board in tqdm(board_ids): print(f"working on board {board}") board_data = monday.boards.fetch_boards_by_id(board) @@ -133,150 +189,169 @@ for board in tqdm(board_ids): reversed_col_id_map[col.get("id")]: col.get("text") }) all_records.append(data) + board_to_record.update({board: all_records}) # Convert to DataFrame -df = pd.DataFrame(all_records) - -filtered_dfs = [] +for board, all_records in board_to_record.items(): + df = pd.DataFrame(all_records) + filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name=None): - _ = pd.DataFrame() - if column_name in col_id_map: - _ = df[ - df[column_name].str.lower().isin(success_critera) - ].copy() - if job_name: - _["job_type"] = job_name - - - return _ + def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ -# RA -ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if not ra.empty: - filtered_dfs.append(ra) + # RA + ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") + if not ra.empty: + filtered_dfs.append(ra) -att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if not att.empty: - filtered_dfs.append(att) + att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") + if not att.empty: + filtered_dfs.append(att) -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") -if not modeling.empty: - filtered_dfs.append(modeling) + modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") + if not modeling.empty: + filtered_dfs.append(modeling) -try: - # Only needed for one board in wave 3 - full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") - if not full_cost.empty: - filtered_dfs(full_cost) -except Exception as e: - print(e) + try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) + except Exception as e: + print(e) -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") -if not v1.empty: - filtered_dfs.append(v1) + v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") + if not v1.empty: + filtered_dfs.append(v1) -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") -if not v2.empty: - filtered_dfs.append(v2) + v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") + if not v2.empty: + filtered_dfs.append(v2) -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") -if not v3.empty: - filtered_dfs.append(v3) + v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") + if not v3.empty: + filtered_dfs.append(v3) -# Coordination stage 2 Please complete -cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") -if not cors2.empty: - filtered_dfs.append(cors2) + # Coordination stage 2 Please complete + cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") + if not cors2.empty: + filtered_dfs.append(cors2) -# Design archetype complex -design = get_df(df, "design invoicing status", ["to invoice"]) -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") -if not design1.empty : - filtered_dfs.append(design1) + # Design archetype complex + design = get_df(df, "design invoicing status", ["to invoice"]) + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") + if not design1.empty : + filtered_dfs.append(design1) -# Design archetype simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") -if not design1.empty: - filtered_dfs.append(design1) + # Design archetype simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive complex -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive complex + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") + if not design1.empty: + filtered_dfs.append(design1) -# Design Revision -revision_letter = ['a', 'b', 'c', 'd'] -for letter in revision_letter: - design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if not design.empty: - filtered_dfs.append(design) + # Design Revision + revision_letter = ['a', 'b', 'c', 'd'] + for letter in revision_letter: + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if not design.empty: + filtered_dfs.append(design) -# Lodgement Phase 1 -lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if not lodg1.empty: - filtered_dfs(lodg1) + # Lodgement Phase 1 + lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") + if not lodg1.empty: + filtered_dfs(lodg1) -# Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") -if not lodg2.empty: - filtered_dfs.append(lodg2) + # Full Lodgement Phase + lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") + if not lodg2.empty: + filtered_dfs.append(lodg2) -# POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") -if not post_epc.empty: - filtered_dfs.append(post_epc) + # POST EPC + post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") + if not post_epc.empty: + filtered_dfs.append(post_epc) -# POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") -if not post_epr.empty: - filtered_dfs.append(post_epr) + # POST EPR + post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") + if not post_epr.empty: + filtered_dfs.append(post_epr) -# post att -post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if not post_att.empty: - filtered_dfs.append(post_epc) + # post att + post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") + if not post_att.empty: + filtered_dfs.append(post_epc) -# Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") -if not rc.empty: - filtered_dfs.append(rc) + # Retrofit Evaluation + rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") + if not rc.empty: + filtered_dfs.append(rc) -# RA NO Show -ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if not ra_ns.empty: - filtered_dfs.append(ra_ns) + # RA NO Show + ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") + if not ra_ns.empty: + filtered_dfs.append(ra_ns) -# ATT NO Show -att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if not att_ns.empty: - filtered_dfs.append(att_ns) + # ATT NO Show + att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") + if not att_ns.empty: + filtered_dfs.append(att_ns) -# Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") -if not epc_ns.empty: - filtered_dfs.append(epc_ns) + # Post visit no show + epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") + if not epc_ns.empty: + filtered_dfs.append(epc_ns) -final_df = pd.concat(filtered_dfs).reset_index(drop=True) + if len(filtered_dfs) != 0: + final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + final_df["job_type"] = final_df["job_type"].str.lower() + rate_card_df = rate_cards[board] + rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() -# Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") -import datetime -timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + # Now perform the merge + combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") + import datetime + timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + + attribute = ['address', 'client', 'job_type', 'rate'] + combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) + + # Upload to sharepoint + attribute = ['address', 'client', 'job_type', 'rate'] + master_folder_name = BoardID[f"_{board}"].value + file_name = f"{master_folder_name}_{timestamp}.xlsx" + combined_with_rates[attribute].to_excel(file_name, index=False) + + file_path = os.path.abspath(file_name) + + upload_to_month_end_folder(file_name, file_path, master_folder_name) + + invoice_name = "rate_card.xlsx" + file_path = os.path.abspath(invoice_name) + rate_card_df.to_excel(invoice_name, index=False) + upload_to_month_end_folder(invoice_name, file_path, master_folder_name) -attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) From 2bdef4b1ccce18944789f2bf7f7031e50b363c86 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 29 Aug 2025 16:42:15 +0100 Subject: [PATCH 11/12] automatic script --- .github/workflows/months_end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index f014aa0..eb3c4ff 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -3,7 +3,6 @@ on: schedule: - cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month push: - branches: [main] workflow_dispatch: jobs: From 55fec1a2742da56b1078a5786f1a31be3b569b17 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 29 Aug 2025 16:42:36 +0100 Subject: [PATCH 12/12] save --- .github/workflows/months_end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index eb3c4ff..69a1f91 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -2,7 +2,6 @@ name: Months End on: schedule: - cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month - push: workflow_dispatch: jobs: