junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

osmosis monday

Browse source
This commit is contained in:
Jun-te Kim 2025-07-16 14:26:40 +00:00
parent b3f9929554
commit bedfc1dab4
6 changed files with 60 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
etl/epr_etl_example.py
View file

@ -1,7 +1,7 @@
from etl.surveyedData.surveryedData import surveyedDataProcessor
files = [
"/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
# "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
"/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf"
]

1
etl/fileReader/pdfReaderToText.py
View file

@ -33,7 +33,6 @@ class pdfReaderToText():
self.all_text += text
self.text_list = self.all_text.split('\n')
pprint(self.text_list)
def get_list_of_text(self):
return self.text_list

5
etl/fileReader/sitenotes.py
View file

@ -298,7 +298,6 @@ class WarmHomesConditionReport(SiteNotesExtractor):
# Gable 2
data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding")
pprint(self.raw_data)
state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False
if state is False:
gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state)
@ -1601,9 +1600,9 @@ class EnergyPerformanceReportWithData(SiteNotesExtractor):
class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor):
def __init__(self, data_list):
super().__init__(data_list)
self.raw_data = data_list
self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION
self.master_obj = self.setup()
self.setup()
def setup(self):
pass

4
etl/hubSpotClient/types.py
View file

@ -81,7 +81,7 @@ class SubmissionInfoFromDeal(BaseModel):
missing_items.append("Condition Report")
if sdp.epr_summary_information is None:
missing_items.append("EPR Energy report with data is missing")
missing_items.append("EPR Energy report with data")
if sdp.rd_sap_xml is None:
missing_items.append("RDSAP XML")
@ -90,7 +90,7 @@ class SubmissionInfoFromDeal(BaseModel):
missing_items.append("LIG SAP XML")
if sdp.epr_summary_information is None:
missing_items.append("EPR Summary information is missing")
missing_items.append("EPR Summary information")
if missing_items:
raise ValueError(f"Missing required items: {', '.join(missing_items)}")

BIN
etl/osmosis_data/asset_list.xlsx
View file

Binary file not shown.

86
etl/osmosis_monday_to_sharepoint_automation.py
View file

@ -8,57 +8,81 @@ from etl.scraper.scraper import SharePointInstaller
from etl.scraper.scraper import SharePointScraper
import pandas as pd
from tqdm import tqdm
import time
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3)
parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders"
parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders"
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1")
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1")
new_asset_list = []
# Create asset list and location
for index, address in tqdm(asset_list.iterrows()):
folder_name = address['Name'] + " " + address['Postcode']
webUrl = osmosis.create_dir(folder_name, parent_folder)
first_folder = "1. Retrofit Assessment"
osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}")
osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}")
if index > 39:
folder_name = address['Name'] + " " + address['Postcode']
webUrl = osmosis.create_dir(folder_name, parent_folder)
time.sleep(1)
print(f"building folders insidea {folder_name}")
second_folder = "2. RC Mid-Term Plan"
osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}")
print("building retrofit assessment")
first_folder = "1. Retrofit Assessment"
osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}")
osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}")
third_folder = "3. Retrofit Design"
osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}")
print("building RC MID Term plan")
second_folder = "2. RC Mid-Term Plan"
osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}")
fourth_folder = "4. Post EPC"
osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}")
print("building Retrofit Design")
third_folder = "3. Retrofit Design"
osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}")
fifth_folder = "5. Trustmark Lodgement"
osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}")
print("building post epc")
fourth_folder = "4. Post EPC"
osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}")
osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
asset_data = {
"Name": address['Name'],
"Postcode": address['Postcode'],
"Sharepoint": webUrl,
}
print("Building Trust mark Lodgement")
fifth_folder = "5. Trustmark Lodgement"
osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}")
osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}")
new_asset_list.append(asset_data)
osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
asset_data = {
"Name": address['Name'],
"Postcode": address['Postcode'],
"Sharepoint": webUrl,
}
print(asset_data)
new_asset_list.append(asset_data)
# Osmosist File strucutre
# Run this is you just want to get url
def just_url(asset_list):
new_asset_list = []
for index, address in tqdm(asset_list.iterrows()):
folder_name = address['Name'] + " " + address['Postcode']
webUrl = osmosis.create_dir(folder_name, parent_folder)
asset_data = {
"Name": address['Name'],
"Postcode": address['Postcode'],
"Sharepoint": webUrl,
}
print(asset_data)
new_asset_list.append(asset_data)
return new_asset_list
new_asset_list = just_url(asset_list=asset_list)
df = pd.DataFrame(new_asset_list)
df.to_csv("output.csv", index=False)