added osmosis script to do location automaticallyu

This commit is contained in:
Jun-te Kim 2025-04-11 16:18:57 +00:00
parent fd4cbddd5e
commit 2fac7d6112
4 changed files with 129 additions and 1 deletions

105
etl/osmosis_google_maps_.py Normal file
View file

@ -0,0 +1,105 @@
from monday import MondayClient
import json
import requests
board_id = "8829428746"
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
import time
def get_all_items(board_id, monday):
# Parameters
limit = 25 # Adjust the limit based on how many items you want per request
all_items = [] # List to store all fetched items
cursor = None # Start without a cursor for the first page
# Loop through pages
while True:
# Fetch items for the current page
response = monday.boards.fetch_items_by_board_id(
board_ids=board_id,
limit=limit,
cursor=cursor
)
items = response['data']['boards'][0]['items_page']['items']
# If no items are returned, stop the loop
if not items:
break
# Append items from this page to the all_items list
all_items.extend(items)
# Get the cursor for the next page (if there is one)
cursor = response['data']['boards'][0]['items_page'].get('cursor') # Get the current cursor
# If there's no cursor, we've reached the last page
if not cursor:
break
print(f"cursor {cursor}")
print(f"len all_itemms {len(all_items)}")
return all_items
def get_coords_from_address(address):
url = "https://nominatim.openstreetmap.org/search"
params = {
"q": address,
"format": "json",
"limit": 1
}
headers = {
"User-Agent": "YourAppNameHere"
}
response = requests.get(url, params=params, headers=headers)
data = response.json()
if data:
return {
"lat": float(data[0]["lat"]),
"lng": float(data[0]["lon"]),
"address": address
}
else:
return None
# Step 1: Fetch column IDs
board_data = monday.boards.fetch_boards_by_id(board_id)
columns = board_data["data"]["boards"][0]["columns"]
col_id_map = {col["title"].lower(): col["id"] for col in columns}
postcode_col_id = col_id_map.get("postcode") # Replace with actual title if different
location_col_id = col_id_map.get("location") # Replace with actual title if different
if not postcode_col_id or not location_col_id:
raise Exception("Could not find 'postcode' or 'location' columns")
items = get_all_items(board_id, monday)
for item in items:
item_name = item["name"]
item_id = item["id"]
# Get postcode value
postcode = None
for val in item["column_values"]:
if val["id"] == postcode_col_id:
postcode = val["text"]
break
if postcode:
try:
location = get_coords_from_address(f"{item_name}, {postcode}, UK")
if location:
print(f"Updating '{item_name}' with coords {location}")
# Step 3: Update location field
monday.items.change_multiple_column_values(
board_id,
item_id,
{location_col_id: location},
)
except Exception as e:
print(f"Error updating item '{item_name}': {e}")
else:
print(f"No postcode found for item '{item_name}'")

View file

@ -1,3 +1,5 @@
class surveyPrice():
"""
A class to work out all prices and uploads to sharepoint
@ -5,6 +7,14 @@ class surveyPrice():
def __init__(self):
pass
def download_price_card(self):
pass
def get_price_card():
pass
# Step one

14
poetry.lock generated
View file

@ -860,6 +860,18 @@ files = [
[package.dependencies]
traitlets = "*"
[[package]]
name = "monday"
version = "2.0.1"
description = "A Python client library for Monday.com"
optional = false
python-versions = ">=3.6"
groups = ["main"]
files = [
{file = "monday-2.0.1-py3-none-any.whl", hash = "sha256:17f347dea47439c2181e2896a4c49f43289ea9c78a4156f240c652e54ceb193d"},
{file = "monday-2.0.1.tar.gz", hash = "sha256:fd080590fd9b23a88e838a092d243d20ba6e2aea963405c4f21c02dc617daab1"},
]
[[package]]
name = "msal"
version = "1.32.0"
@ -1913,4 +1925,4 @@ files = [
[metadata]
lock-version = "2.1"
python-versions = ">=3.12"
content-hash = "5778c3bdce3109fe21e0ca7861ffc9c738d6af0545e0c27517bbe88c89209082"
content-hash = "55a974b3a81d57c429f61ee6a12a84d38f5c703fdfdfdf2553bec6ba21c29bf5"

View file

@ -20,6 +20,7 @@ dependencies = [
"alembic (>=1.15.1,<2.0.0)",
"pytest (>=8.3.5,<9.0.0)",
"hubspot-api-client (>=11.1.0,<12.0.0)",
"monday (>=2.0.1,<3.0.0)",
]
[tool.poetry]