diff --git a/etl/osmosis_complaince_address_to_files.py b/etl/osmosis_complaince_address_to_files.py index d3d14d9..f2d3c9f 100644 --- a/etl/osmosis_complaince_address_to_files.py +++ b/etl/osmosis_complaince_address_to_files.py @@ -23,31 +23,47 @@ parent_folder = "/Osmosis ACD/Osmosis ACD Projects/Installer Documentation/" parent_folder += "Platform Housing Group/Broadoak" -def download_file(url): +def extract_asset_ids(item, file_column_id): + for col in item["column_values"]: + if col["id"] == file_column_id and col["value"]: + try: + value = json.loads(col["value"]) + return [f["assetId"] for f in value.get("files", [])] + except Exception as e: + print(f"Error parsing file column: {e}") + return [] + +def get_public_url(asset_id): + url = "https://api.monday.com/v2" headers = { "Authorization": monday_key, - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/125.0.0.0 Safari/537.36", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/apng,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.9", - "Accept-Encoding": "gzip, deflate, br", - "Connection": "keep-alive", - "Referer": "https://osmosis-acd-team.monday.com/", # Optional but helpful - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "same-origin", - "Sec-Fetch-User": "?1", + "Content-Type": "application/json" } - local_filename = os.path.join("/tmp", url.split("/")[-1]) - with requests.get(url.strip(), headers=headers, stream=True) as r: + query = { + "query": f""" + query {{ + assets(ids: {asset_id}) {{ + public_url + name + }} + }} + """ + } + response = requests.post(url, json=query, headers=headers) + response.raise_for_status() + asset = response.json()["data"]["assets"][0] + return asset["public_url"], asset["name"] + +def download_file_from_public_url(public_url, filename): + local_path = os.path.join("/tmp", filename) + with requests.get(public_url, stream=True) as r: r.raise_for_status() - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - return local_filename + with open(local_path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + return local_path + def get_all_items(board_id, monday): # Parameters @@ -102,18 +118,17 @@ if not name_id or not files_id: items = get_all_items(board_id, monday) for item in tqdm(items): item_name = item["name"] - item_id = item["id"] + asset_ids = extract_asset_ids(item, files_id) - print(f"Downloading '{item_name}'...") - for val in item["column_values"]: - # files - if val["id"] == files_id: - all_files_csv = val["text"] - files = all_files_csv.split(",") + to_upload = [] + for asset_id in asset_ids: + try: + public_url, file_name = get_public_url(asset_id) + print(f"Downloading {file_name} from {public_url}") + file_path = download_file_from_public_url(public_url, file_name) + to_upload.append(file_path) + except Exception as e: + print(f"Failed to download/upload asset {asset_id}: {e}") - to_upload = [] - for file in tqdm(files): - print(f"Downloading {file}") - to_upload.append(download_file(file)) - upload_to_sharepoint(to_upload, item_name) - \ No newline at end of file + if to_upload: + upload_to_sharepoint(to_upload, item_name) \ No newline at end of file