fix by downloading the actual contents and not just the html

etl/osmosis_complaince_address_to_files.py

@@ -23,31 +23,47 @@ parent_folder = "/Osmosis ACD/Osmosis ACD Projects/Installer Documentation/"
 parent_folder += "Platform Housing Group/Broadoak"
 
 
-def download_file(url):
+def extract_asset_ids(item, file_column_id):
+    for col in item["column_values"]:
+        if col["id"] == file_column_id and col["value"]:
+            try:
+                value = json.loads(col["value"])
+                return [f["assetId"] for f in value.get("files", [])]
+            except Exception as e:
+                print(f"Error parsing file column: {e}")
+    return []
+
+def get_public_url(asset_id):
+    url = "https://api.monday.com/v2"
     headers = {
         "Authorization": monday_key,
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                    "AppleWebKit/537.36 (KHTML, like Gecko) "
-                    "Chrome/125.0.0.0 Safari/537.36",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,"
-                "image/avif,image/webp,image/apng,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Accept-Encoding": "gzip, deflate, br",
-        "Connection": "keep-alive",
-        "Referer": "https://osmosis-acd-team.monday.com/",  # Optional but helpful
-        "Upgrade-Insecure-Requests": "1",
-        "Sec-Fetch-Dest": "document",
-        "Sec-Fetch-Mode": "navigate",
-        "Sec-Fetch-Site": "same-origin",
-        "Sec-Fetch-User": "?1",
+        "Content-Type": "application/json"
     }
-    local_filename = os.path.join("/tmp", url.split("/")[-1])
-    with requests.get(url.strip(), headers=headers, stream=True) as r:
+    query = {
+        "query": f"""
+        query {{
+            assets(ids: {asset_id}) {{
+                public_url
+                name
+            }}
+        }}
+        """
+    }
+    response = requests.post(url, json=query, headers=headers)
+    response.raise_for_status()
+    asset = response.json()["data"]["assets"][0]
+    return asset["public_url"], asset["name"]
+
+def download_file_from_public_url(public_url, filename):
+    local_path = os.path.join("/tmp", filename)
+    with requests.get(public_url, stream=True) as r:
         r.raise_for_status()
-        with open(local_filename, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192): 
-                f.write(chunk)
-    return local_filename
+        with open(local_path, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+    return local_path
+
 
 def get_all_items(board_id, monday):
     # Parameters
@@ -102,18 +118,17 @@ if not name_id or not files_id:
 items = get_all_items(board_id, monday)
 for item in tqdm(items):
     item_name = item["name"]
-    item_id = item["id"]
+    asset_ids = extract_asset_ids(item, files_id)
 
-    print(f"Downloading '{item_name}'...")
-    for val in item["column_values"]:
-        # files
-        if val["id"] == files_id:
-            all_files_csv = val["text"]
-            files = all_files_csv.split(",")
+    to_upload = []
+    for asset_id in asset_ids:
+        try:
+            public_url, file_name = get_public_url(asset_id)
+            print(f"Downloading {file_name} from {public_url}")
+            file_path = download_file_from_public_url(public_url, file_name)
+            to_upload.append(file_path)
+        except Exception as e:
+            print(f"Failed to download/upload asset {asset_id}: {e}")
 
-            to_upload = []
-            for file in tqdm(files):
-                print(f"Downloading {file}")
-                to_upload.append(download_file(file))
-            upload_to_sharepoint(to_upload, item_name) 
-            
+    if to_upload:
+        upload_to_sharepoint(to_upload, item_name)