diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py index 866c75a..e0a7032 100644 --- a/etl/epr_etl_example.py +++ b/etl/epr_etl_example.py @@ -1,7 +1,7 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor files = [ - "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", + # "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf" ] diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index 9668eed..c1d5834 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -33,7 +33,6 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') - pprint(self.text_list) def get_list_of_text(self): return self.text_list diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 1df8747..7799252 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -298,7 +298,6 @@ class WarmHomesConditionReport(SiteNotesExtractor): # Gable 2 data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding") - pprint(self.raw_data) state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False if state is False: gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state) @@ -1601,9 +1600,9 @@ class EnergyPerformanceReportWithData(SiteNotesExtractor): class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor): def __init__(self, data_list): - super().__init__(data_list) + self.raw_data = data_list self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION - self.master_obj = self.setup() + self.setup() def setup(self): pass \ No newline at end of file diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index e004243..c042d6d 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -81,7 +81,7 @@ class SubmissionInfoFromDeal(BaseModel): missing_items.append("Condition Report") if sdp.epr_summary_information is None: - missing_items.append("EPR Energy report with data is missing") + missing_items.append("EPR Energy report with data") if sdp.rd_sap_xml is None: missing_items.append("RDSAP XML") @@ -90,7 +90,7 @@ class SubmissionInfoFromDeal(BaseModel): missing_items.append("LIG SAP XML") if sdp.epr_summary_information is None: - missing_items.append("EPR Summary information is missing") + missing_items.append("EPR Summary information") if missing_items: raise ValueError(f"Missing required items: {', '.join(missing_items)}") diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx index ceada9c..822b436 100644 Binary files a/etl/osmosis_data/asset_list.xlsx and b/etl/osmosis_data/asset_list.xlsx differ diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 4e83612..40118f4 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -8,57 +8,81 @@ from etl.scraper.scraper import SharePointInstaller from etl.scraper.scraper import SharePointScraper import pandas as pd from tqdm import tqdm +import time -osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) +osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3) -parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders" +parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders" -asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1") +asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1") new_asset_list = [] # Create asset list and location for index, address in tqdm(asset_list.iterrows()): - folder_name = address['Name'] + " " + address['Postcode'] - webUrl = osmosis.create_dir(folder_name, parent_folder) - - first_folder = "1. Retrofit Assessment" - osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") - osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") + if index > 39: + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + time.sleep(1) + print(f"building folders insidea {folder_name}") - second_folder = "2. RC Mid-Term Plan" - osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") + print("building retrofit assessment") + first_folder = "1. Retrofit Assessment" + osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") - third_folder = "3. Retrofit Design" - osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") + print("building RC MID Term plan") + second_folder = "2. RC Mid-Term Plan" + osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") - fourth_folder = "4. Post EPC" - osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") + print("building Retrofit Design") + third_folder = "3. Retrofit Design" + osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") - fifth_folder = "5. Trustmark Lodgement" - osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") + print("building post epc") + fourth_folder = "4. Post EPC" + osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") - osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - - asset_data = { - "Name": address['Name'], - "Postcode": address['Postcode'], - "Sharepoint": webUrl, - } + print("Building Trust mark Lodgement") + fifth_folder = "5. Trustmark Lodgement" + osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") - new_asset_list.append(asset_data) + osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + + new_asset_list.append(asset_data) -# Osmosist File strucutre +# Run this is you just want to get url +def just_url(asset_list): + new_asset_list = [] + for index, address in tqdm(asset_list.iterrows()): + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + new_asset_list.append(asset_data) + return new_asset_list +new_asset_list = just_url(asset_list=asset_list) df = pd.DataFrame(new_asset_list) df.to_csv("output.csv", index=False)