diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml new file mode 100644 index 0000000..69a1f91 --- /dev/null +++ b/.github/workflows/months_end.yml @@ -0,0 +1,45 @@ +name: Months End +on: + schedule: + - cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month + workflow_dispatch: + +jobs: + surveyed-needs-sign-off: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: run script + run: | + pwd + ls -la + poetry run python etl/month_end_automation_wave_2_layout.py + poetry run python etl/month_end_automation_wave_2_no_3.py + poetry run python etl/month_end_automation_wave_2_no_4.py + poetry run python etl/month_end_automation_wave_2_no_5.py + poetry run python etl/month_end_automation_wave_2_no_6.py + poetry run python etl/month_end_automation_wave_2_no_7.py + poetry run python etl/month_end_automation_wave_2_no_8.py + poetry run python etl/month_end_automation_wave_2_no_9.py + poetry run python etl/month_end_automation_wave_2_no_10.py + poetry run python etl/month_end_automation_wave_2_no_11.py + poetry run python etl/month_end_automation_wave_2_no_12.py + poetry run python etl/month_end_automation_wave_2_no_13.py + poetry run python etl/month_end_automation_wave_2_no_14.py + poetry run python etl/month_end_automation_wave_2_no_15.py + poetry run python etl/month_end_automation_wave_2_no_16.py + poetry run python etl/month_end_automation_wave_accent_housing.py + poetry run python etl/month_end_automation_wave_3_layout.py + env: + PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py index 8a08c48..1502ef5 100644 --- a/deployment/lambda/extractor_and_loader/docker/app.py +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -57,7 +57,7 @@ def serialize_model(model: Any): else: return model -def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations): +def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations, bepoke_info): # Convert to dict recursively rooms_data = serialize_model(rooms_obj) heating_data = serialize_model(heating_system_obj) @@ -70,6 +70,7 @@ def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevatio "heating_system": heating_data, "occupant_info": occupant_data, "access_and_elevations": access_and_elevations_data, + "bespoke_data": bepoke_info } # Convert to pretty JSON string @@ -203,18 +204,24 @@ def handler(event, context): local_path = download_private_s3_file(file_uri) # Local development of file, please comment out for prod - # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/67-Aylestone-Road-1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1 1.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf")) + # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/6E-plantagenet-street.pdf")) print("Extracting file...") reader = pdfReaderToText(local_path) + # obj2 = WarmHomesConditionReport(reader.text_list, debug=True) obj = WarmHomesConditionReport(reader.text_list) json_ = make_final_json( obj.master_obj[0], obj.master_obj[1], obj.master_obj[2], obj.master_obj[3], + {} ) print("Extracted completed, made json") diff --git a/etl/MonthEndUploader.py b/etl/MonthEndUploader.py new file mode 100644 index 0000000..927b5c0 --- /dev/null +++ b/etl/MonthEndUploader.py @@ -0,0 +1,24 @@ +import os +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" +from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from datetime import datetime + + +def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path): + sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) + + parent_folder = "General/Junte Kim/month end" + today = datetime.today() + # Format as "Month YYYY" + formatted_date = today.strftime("%B %Y") + + sharepoint.create_dir(formatted_date, parent_folder) + sharepoint_path = parent_folder + "/" + formatted_date + sharepoint.create_dir(add_to_path, sharepoint_path) + sharepoint_path += "/" + add_to_path + + print("Uploading to sharepoint...") + sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp) + print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}") diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index f2de52d..8f087e2 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -102,12 +102,12 @@ class ECOConditionReport(SiteNotesExtractor): class WarmHomesConditionReport(SiteNotesExtractor): - def __init__(self, data_list): + def __init__(self, data_list, debug = False): super().__init__(data_list) self.type = ReportType.WARM_HOMES_CONDITION_REPORT - room, heating_system, occupant, access_and_elevation = self.setup_condition_report() - - self.master_obj = room, heating_system, occupant, access_and_elevation + if debug is False: + room, heating_system, occupant, access_and_elevation = self.setup_condition_report() + self.master_obj = room, heating_system, occupant, access_and_elevation def setup_condition_report(self): # general_information = self.get_section_1() @@ -292,7 +292,7 @@ class WarmHomesConditionReport(SiteNotesExtractor): # Gable 2 data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding") - state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False + state = False # TODO quick demo for waltham forest, set to always false if state is False: gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state) else: @@ -347,12 +347,24 @@ class WarmHomesConditionReport(SiteNotesExtractor): ) def get_room_info(self, data): - ventilation = VentilationInfo( - is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, - are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, - are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), - is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, - ) + state = True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False + if state is False: + ventilation = VentilationInfo( + is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, + are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, + are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), + is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, + ) + else: + ventilation = VentilationInfo( + is_there_a_ventilation_system_present_in_the_room=True if self.get_next_value(data, "Is there a ventilation system present in the room?").lower() == "yes" else False, + are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room=True if self.get_next_value(data, "excessive condensation within the room?").lower() == "yes" else False, + are_there_sufficient_undercuts_on_the_closed_door=self.get_next_value(data, "- min 10mm)?"), + is_there_any_open_flue_heating_appliances_within_the_room=True if self.get_next_value(data, "Is there any open flue heating appliances within the room?").lower() == "yes" else False, + location_of_any_damp_or_mould=self.get_next_value(data, "Location of any damp/mould"), + what_severity_of_damp_mould_would_you_consider_this_to_be=self.get_next_value(data, "What severity of damp/mould would you consider this to be?"), + ) + windows_state = True if self.get_next_value(data, "Does the room have any windows?").lower() == "yes" else False if windows_state: @@ -368,20 +380,37 @@ class WarmHomesConditionReport(SiteNotesExtractor): does_the_room_have_any_windows=windows_state ) - return RoomInfo( - overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), - does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), - windows_info=windows, - ventilation_info=ventilation, - ) + state = True if self.get_next_value(data, "cracking in walls, etc.)").lower()=="yes" else False + if state: + return RoomInfo( + overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), + does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + description_of_defect=self.get_next_value(data, "Description of the defects", avoid=["Any additional recommendations?"]), + windows_info=windows, + ventilation_info=ventilation, + ) + else: + return RoomInfo( + overall_condition_of_the_room=self.get_next_value(data, "Overall condition of the room"), + does_the_room_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + windows_info=windows, + ventilation_info=ventilation, + ) + def get_hallway(self): data = self.get_data_between("Hallway", "Living Room") - room = self.get_room_info(data) + + hallway_state = True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False + + if hallway_state: + room = self.get_room_info(data) + else: + room = None return Hallway( - is_there_a_hallway=True if self.get_next_value(data, "Is there a hallway?").lower() == "yes" else False, + is_there_a_hallway=hallway_state, room_info=room, ) @@ -494,20 +523,26 @@ class WarmHomesConditionReport(SiteNotesExtractor): def get_loft_space(self): data = self.get_data_between("Loft Space", "Room in Roof") - return LoftSpace( - is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), - is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, - overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), - are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), - does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), - existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), - is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), - is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), - condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), - is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), - is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), - is_it_accessible=self.get_next_value(data, "Is it accessible?"), - ) + state = True if self.get_next_value(data, "Is the main loft space accessible?").lower() == "yes" else False + if state: + return LoftSpace( + is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), + is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + overall_condition_of_the_loft_space=self.get_next_value(data, "Overall condition of the loft space"), + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer=self.get_next_value(data, "insulation layer?"), + does_the_loft_space_have_any_defects=self.get_next_value(data, "cracking in walls, etc.)"), + existing_depth_of_loft_insulation=self.get_next_value(data, "Existing Depth of Loft Insulation"), + is_the_insulation_layer_even_across_the_loft=self.get_next_value(data, "Is the insulation layer even across the loft?"), + is_the_loft_boarded_in_any_area=self.get_next_value(data, "Is the loft boarded in any area?"), + condition_of_existing_roof_lining=self.get_next_value(data, "Condition of existing roof lining"), + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft=self.get_next_value(data, "the loft?"), + is_there_any_open_flue_heating_applicanes_within_the_room=self.get_next_value(data, "Is there any open flue heating appliances within the room?"), + ) + else: + return LoftSpace( + is_the_main_loft_space_accessible=self.get_next_value(data, "Is the main loft space accessible?"), + is_there_more_than_one_loft_space=True if self.get_next_value(data, "extension)?").lower() == "yes" else False, + ) def get_room_in_roof(self): data = self.get_data_between("Room in Roof", "4. Heating System") @@ -596,12 +631,12 @@ class WarmHomesConditionReport(SiteNotesExtractor): is_there_any_renewable_energy_system_in_place=True if self.get_next_value(data, "Is there any renewable energy system in place?").lower() == "yes" else False, suitable_roof_orientation_for_solar_pv_water=self.get_next_value(data, "Suitable roof orientation for Solar PV/water:"), is_there_a_water_tank=True if self.get_next_value(data, "Is there a Water Tank?").lower() == "yes" else False, - type=self.get_next_value(data, "Type"), - size=self.get_next_value(data, "Size"), - tank_location=self.get_next_value(data, "Tank Location"), - is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False, - type_of_insulation=self.get_next_value(data, "Type of Insulation"), - thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")), + # type=self.get_next_value(data, "Type"), + # size=self.get_next_value(data, "Size"), + # tank_location=self.get_next_value(data, "Tank Location"), + # is_the_tank_insulated=True if self.get_next_value(data, "Is the tank Insulated") else False, + # type_of_insulation=self.get_next_value(data, "Type of Insulation"), + # thickness_of_insulation_in_mm=int(self.get_next_value(data, "Thickness of Insulation (mm)")), ) def get_general_condition_of_heating_system(self): diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 7dca011..e6b4ff2 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -1,11 +1,14 @@ -# Wave 2's month end automation +# Wave 2's month end automation from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder +import os + monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -232,5 +235,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False) \ No newline at end of file +master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2" +file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index d854fdc..9b7b3e7 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA Derbyshire Dales (DDDCC) SHDF" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 41f4438..874eaa3 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -198,5 +200,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False) +master_folder_name = "Northumberland LAD2 & HUG2" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 7fa4c1a..464d211 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -149,13 +151,13 @@ full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Fu filtered_dfs.append(full_lodgement) # POST EPC -post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +post_epc = get_df(df, "post-epc epr status", ["uploaded", "completed"], "POST EPC") filtered_dfs.append(post_epc) # POST EPR post_epr = df[ - df["post-epc status"].str.lower().isin(["post epr completed"]) + df["post-epc epr status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) @@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False) +master_folder_name = "Shropshire Council HUG 2" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index d4a6a1d..7637222 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -196,5 +198,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False) +master_folder_name = "Stonewater SHDF 2.0 - Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 27cadd0..1099185 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -202,5 +204,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False) +master_folder_name = "Decent Homes Stonewater - Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 8644db9..1b10f25 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -210,5 +212,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) +master_folder_name = "WCHG SHDF 2.1 Mansard" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 0af8bfe..664ebff 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -5,6 +5,8 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False) \ No newline at end of file +master_folder_name = "NCHA SHDF Wave 3 On Hold" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index b349a7a..2500d25 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -1,7 +1,8 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient +from MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd @@ -230,5 +231,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False) +master_folder_name = "Platform Housing W2 (in use)" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 30b949f..1b7f6f4 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -237,5 +238,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False) +master_folder_name = "Stonewater - (in use)" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index f26d3b0..c26834b 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -229,5 +230,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False) +master_folder_name = "ECO 4 NCHA Almshouse Operation" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 0ad996f..8091aa1 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -1,7 +1,8 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient +from MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd @@ -205,3 +206,19 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False) + + +# Upload to sharepoint +attribute = ['address', 'client', 'job_type', 'rate'] +master_folder_name = "ECO 4 Wates Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 214c233..e1fbf9c 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -1,11 +1,12 @@ # Wave 2's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -195,7 +196,7 @@ post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT -post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT") +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") filtered_dfs.append(post_att) @@ -236,6 +237,7 @@ rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower( rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower() rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower() + # Now perform the merge net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)] sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)] @@ -247,11 +249,41 @@ combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_me import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) +master_folder_name = "Home Group" -# TO DO check everything in excel -# make logic for seperation \ No newline at end of file +# Wave 2SP+ +file_name = f"{master_folder_name}_Wave 2SP+_{timestamp}.xlsx" +combined_with_rates_sp_plus[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "wave_2sp_plus_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_sp_plus.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) + + +# Net Zero +file_name = f"{master_folder_name}_NetZero_{timestamp}.xlsx" +combined_with_rates_net_zero_df[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "net_zero_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_net_zero.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) + +# Unsure +file_name = f"{master_folder_name}_unsure_who_to_bill_{timestamp}.xlsx" +combined_with_rates_other_from_home_group[attribute].to_excel(file_name, index=False) +file_path = os.path.abspath(file_name) +upload_to_month_end_folder(file_name, file_path, master_folder_name) +invoice_name = "unsure_who_to_bill_rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df_error_message.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 0aff6fc..d465dd6 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -214,5 +216,18 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA SHDF 2.1 SBS" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index 15bf2fb..7d92c72 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -203,5 +205,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False) +master_folder_name = "NCHA Almshouses" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 704668c..32125a4 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -6,18 +6,37 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder +from enum import Enum + monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) + +class BoardID(Enum): + _9349630181 = "WCHG_WALKUPS_OPERATIONS" + _8830772914 = "LQ_LONDON" + _9601691730 = "CARDO_WALES_WEST_WAVE3" + _9660895490 = "NORTHUMBERLAND_COUNTY_SHDF_WAVE3" + _9641491000 = "WATFORD_WARM_HOMES" + _9671463094 = "SEDDON" + + board_ids = [ - # "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - "9641491000", #Watford Warm Homes + "9349630181", # WCHG Walkups-Operations x + "8830772914", # "L&Q London" x + "9601691730", # Cardo Wales & West - Wave 3 x + "9660895490", # Northumberland County SHDF Wave 3 x + "9641491000", # Watford Warm Homes x + "9671463094", # Seddon ] empty = "Rate card info missing" + +rate_cards = {} + + rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -32,22 +51,9 @@ rate_card_data_watford_warm_homes = { empty, empty, empty, empty, empty, ] } +rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes)}) -rate_card_data_2502_accent_housing = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} rate_card_data_l_and_q_london = { "job_type": [ @@ -64,22 +70,27 @@ rate_card_data_l_and_q_london = { ] } +rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)}) + rate_card_data_northhumberland_country_shdf_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" ], "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 130, + 259, 125, "280 or 310 depending on SH or LA", 125, 125, + 650, 415, 195, 225, 175, + 135, 120, "60 - check", "85 or 90 depending on SH or LA", "110 or 125 depending on SH or LA", + 60, 45, 45, 45, empty, empty ] } +pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)}) + rate_card_data_walk_ups = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -95,14 +106,59 @@ rate_card_data_walk_ups = { ] } -rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes) -# rate_card_df = pd.DataFrame(rate_card_data_example) -# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) -# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -# rate_card_df = pd.DataFrame(rate_card_data_walk_ups) +rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)}) + +rate_card_data_empty = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty,empty , empty, empty, empty, + empty, empty, empty, empty, empty, + ] +} + +rate_card_data_cardo = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 260, 125, 125, + 415, 415, 225, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)}) + +rate_card_data_seddon = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9671463094": pd.DataFrame(rate_card_data_seddon)}) +board_to_record = {} for board in tqdm(board_ids): print(f"working on board {board}") board_data = monday.boards.fetch_boards_by_id(board) @@ -133,150 +189,169 @@ for board in tqdm(board_ids): reversed_col_id_map[col.get("id")]: col.get("text") }) all_records.append(data) + board_to_record.update({board: all_records}) # Convert to DataFrame -df = pd.DataFrame(all_records) - -filtered_dfs = [] +for board, all_records in board_to_record.items(): + df = pd.DataFrame(all_records) + filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name=None): - _ = pd.DataFrame() - if column_name in col_id_map: - _ = df[ - df[column_name].str.lower().isin(success_critera) - ].copy() - if job_name: - _["job_type"] = job_name - - - return _ + def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ -# RA -ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if not ra.empty: - filtered_dfs.append(ra) + # RA + ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") + if not ra.empty: + filtered_dfs.append(ra) -att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if not att.empty: - filtered_dfs.append(att) + att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") + if not att.empty: + filtered_dfs.append(att) -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") -if not modeling.empty: - filtered_dfs.append(modeling) + modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") + if not modeling.empty: + filtered_dfs.append(modeling) -try: - # Only needed for one board in wave 3 - full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") - if not full_cost.empty: - filtered_dfs(full_cost) -except Exception as e: - print(e) + try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) + except Exception as e: + print(e) -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") -if not v1.empty: - filtered_dfs.append(v1) + v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") + if not v1.empty: + filtered_dfs.append(v1) -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") -if not v2.empty: - filtered_dfs.append(v2) + v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") + if not v2.empty: + filtered_dfs.append(v2) -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") -if not v3.empty: - filtered_dfs.append(v3) + v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") + if not v3.empty: + filtered_dfs.append(v3) -# Coordination stage 2 Please complete -cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") -if not cors2.empty: - filtered_dfs.append(cors2) + # Coordination stage 2 Please complete + cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") + if not cors2.empty: + filtered_dfs.append(cors2) -# Design archetype complex -design = get_df(df, "design invoicing status", ["to invoice"]) -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") -if not design1.empty : - filtered_dfs.append(design1) + # Design archetype complex + design = get_df(df, "design invoicing status", ["to invoice"]) + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") + if not design1.empty : + filtered_dfs.append(design1) -# Design archetype simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") -if not design1.empty: - filtered_dfs.append(design1) + # Design archetype simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive complex -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive complex + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") + if not design1.empty: + filtered_dfs.append(design1) -# Design Revision -revision_letter = ['a', 'b', 'c', 'd'] -for letter in revision_letter: - design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if not design.empty: - filtered_dfs.append(design) + # Design Revision + revision_letter = ['a', 'b', 'c', 'd'] + for letter in revision_letter: + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if not design.empty: + filtered_dfs.append(design) -# Lodgement Phase 1 -lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if not lodg1.empty: - filtered_dfs(lodg1) + # Lodgement Phase 1 + lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") + if not lodg1.empty: + filtered_dfs(lodg1) -# Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") -if not lodg2.empty: - filtered_dfs.append(lodg2) + # Full Lodgement Phase + lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") + if not lodg2.empty: + filtered_dfs.append(lodg2) -# POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") -if not post_epc.empty: - filtered_dfs.append(post_epc) + # POST EPC + post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") + if not post_epc.empty: + filtered_dfs.append(post_epc) -# POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") -if not post_epr.empty: - filtered_dfs.append(post_epr) + # POST EPR + post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") + if not post_epr.empty: + filtered_dfs.append(post_epr) -# post att -post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if not post_att.empty: - filtered_dfs.append(post_epc) + # post att + post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") + if not post_att.empty: + filtered_dfs.append(post_epc) -# Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") -if not rc.empty: - filtered_dfs.append(rc) + # Retrofit Evaluation + rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") + if not rc.empty: + filtered_dfs.append(rc) -# RA NO Show -ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if not ra_ns.empty: - filtered_dfs.append(ra_ns) + # RA NO Show + ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") + if not ra_ns.empty: + filtered_dfs.append(ra_ns) -# ATT NO Show -att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if not att_ns.empty: - filtered_dfs.append(att_ns) + # ATT NO Show + att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") + if not att_ns.empty: + filtered_dfs.append(att_ns) -# Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") -if not epc_ns.empty: - filtered_dfs.append(epc_ns) + # Post visit no show + epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") + if not epc_ns.empty: + filtered_dfs.append(epc_ns) -final_df = pd.concat(filtered_dfs).reset_index(drop=True) + if len(filtered_dfs) != 0: + final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + final_df["job_type"] = final_df["job_type"].str.lower() + rate_card_df = rate_cards[board] + rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() -# Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") -import datetime -timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + # Now perform the merge + combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") + import datetime + timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + + attribute = ['address', 'client', 'job_type', 'rate'] + combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) + + # Upload to sharepoint + attribute = ['address', 'client', 'job_type', 'rate'] + master_folder_name = BoardID[f"_{board}"].value + file_name = f"{master_folder_name}_{timestamp}.xlsx" + combined_with_rates[attribute].to_excel(file_name, index=False) + + file_path = os.path.abspath(file_name) + + upload_to_month_end_folder(file_name, file_path, master_folder_name) + + invoice_name = "rate_card.xlsx" + file_path = os.path.abspath(invoice_name) + rate_card_df.to_excel(invoice_name, index=False) + upload_to_month_end_folder(invoice_name, file_path, master_folder_name) -attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index e02cdd9..b13d155 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -1,11 +1,12 @@ # Wave 3's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -88,7 +89,7 @@ if not second_payment.empty: full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp") if not full_cost.empty: - filtered_dfs(full_cost) + filtered_dfs.append(full_cost) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -100,5 +101,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False) +master_folder_name = "2501 Accent housing" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/transform/conditionReportTypes.py b/etl/transform/conditionReportTypes.py index ee3c091..7bfdebf 100644 --- a/etl/transform/conditionReportTypes.py +++ b/etl/transform/conditionReportTypes.py @@ -93,6 +93,8 @@ class VentilationInfo(BaseModel): are_there_any_visible_or_reported_signs_of_damp_mould_or_excessive_condensation_within_the_room: bool are_there_sufficient_undercuts_on_the_closed_door: str is_there_any_open_flue_heating_appliances_within_the_room: bool + location_of_any_damp_or_mould: Optional[str] = "" + what_severity_of_damp_mould_would_you_consider_this_to_be: Optional[str] = "" class WindowsInfo(BaseModel): does_the_room_have_any_windows: bool @@ -104,6 +106,7 @@ class WindowsInfo(BaseModel): class RoomInfo(BaseModel): overall_condition_of_the_room: str does_the_room_have_any_defects: str + description_of_defect: Optional[str] = "" are_there_any_sloped_ceiling_areas: Optional[bool] = None windows_info: Optional[WindowsInfo] = None ventilation_info: Optional[VentilationInfo] = None @@ -147,16 +150,15 @@ class Bathroom(BaseModel): class LoftSpace(BaseModel): is_the_main_loft_space_accessible: str is_there_more_than_one_loft_space: bool - overall_condition_of_the_loft_space: Optional[str] - are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] - does_the_loft_space_have_any_defects: Optional[str] - existing_depth_of_loft_insulation: Optional[str] - is_the_insulation_layer_even_across_the_loft: Optional[str] - is_the_loft_boarded_in_any_area: Optional[str] - condition_of_existing_roof_lining: Optional[str] - is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] - is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] - is_it_accessible: Optional[str] + overall_condition_of_the_loft_space: Optional[str] = "" + are_there_visible_signs_of_condesnation_on_the_roof_lining_or_insualtion_layer: Optional[str] = "" + does_the_loft_space_have_any_defects: Optional[str] = "" + existing_depth_of_loft_insulation: Optional[str] = "" + is_the_insulation_layer_even_across_the_loft: Optional[str] = "" + is_the_loft_boarded_in_any_area: Optional[str] = "" + condition_of_existing_roof_lining: Optional[str] = "" + is_there_an_existing_heating_system_or_plumbing_located_in_the_loft: Optional[str] = "" + is_there_any_open_flue_heating_applicanes_within_the_room: Optional[str] = "" @@ -213,12 +215,12 @@ class Renewables(BaseModel): is_there_any_renewable_energy_system_in_place: bool suitable_roof_orientation_for_solar_pv_water: str is_there_a_water_tank: bool - type: str - size: str - tank_location: str - is_the_tank_insulated: bool - type_of_insulation: str - thickness_of_insulation_in_mm: int + # type: str + # size: str + # tank_location: str + # is_the_tank_insulated: bool + # type_of_insulation: str + # thickness_of_insulation_in_mm: int class HeatingSystem(BaseModel): general_condition: GeneralConditionHeatingSystem