mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
added osmosis script to do location automaticallyu
This commit is contained in:
parent
fd4cbddd5e
commit
2fac7d6112
4 changed files with 129 additions and 1 deletions
105
etl/osmosis_google_maps_.py
Normal file
105
etl/osmosis_google_maps_.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
from monday import MondayClient
|
||||
import json
|
||||
import requests
|
||||
board_id = "8829428746"
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
import time
|
||||
|
||||
def get_all_items(board_id, monday):
|
||||
# Parameters
|
||||
limit = 25 # Adjust the limit based on how many items you want per request
|
||||
all_items = [] # List to store all fetched items
|
||||
cursor = None # Start without a cursor for the first page
|
||||
|
||||
# Loop through pages
|
||||
while True:
|
||||
# Fetch items for the current page
|
||||
response = monday.boards.fetch_items_by_board_id(
|
||||
board_ids=board_id,
|
||||
limit=limit,
|
||||
cursor=cursor
|
||||
)
|
||||
|
||||
items = response['data']['boards'][0]['items_page']['items']
|
||||
|
||||
# If no items are returned, stop the loop
|
||||
if not items:
|
||||
break
|
||||
|
||||
# Append items from this page to the all_items list
|
||||
all_items.extend(items)
|
||||
|
||||
# Get the cursor for the next page (if there is one)
|
||||
cursor = response['data']['boards'][0]['items_page'].get('cursor') # Get the current cursor
|
||||
|
||||
# If there's no cursor, we've reached the last page
|
||||
if not cursor:
|
||||
break
|
||||
print(f"cursor {cursor}")
|
||||
print(f"len all_itemms {len(all_items)}")
|
||||
return all_items
|
||||
|
||||
def get_coords_from_address(address):
|
||||
url = "https://nominatim.openstreetmap.org/search"
|
||||
params = {
|
||||
"q": address,
|
||||
"format": "json",
|
||||
"limit": 1
|
||||
}
|
||||
headers = {
|
||||
"User-Agent": "YourAppNameHere"
|
||||
}
|
||||
response = requests.get(url, params=params, headers=headers)
|
||||
data = response.json()
|
||||
if data:
|
||||
return {
|
||||
"lat": float(data[0]["lat"]),
|
||||
"lng": float(data[0]["lon"]),
|
||||
"address": address
|
||||
}
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
|
||||
# Step 1: Fetch column IDs
|
||||
board_data = monday.boards.fetch_boards_by_id(board_id)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
|
||||
postcode_col_id = col_id_map.get("postcode") # Replace with actual title if different
|
||||
location_col_id = col_id_map.get("location") # Replace with actual title if different
|
||||
|
||||
if not postcode_col_id or not location_col_id:
|
||||
raise Exception("Could not find 'postcode' or 'location' columns")
|
||||
|
||||
items = get_all_items(board_id, monday)
|
||||
for item in items:
|
||||
item_name = item["name"]
|
||||
item_id = item["id"]
|
||||
|
||||
# Get postcode value
|
||||
postcode = None
|
||||
for val in item["column_values"]:
|
||||
if val["id"] == postcode_col_id:
|
||||
postcode = val["text"]
|
||||
break
|
||||
|
||||
if postcode:
|
||||
try:
|
||||
location = get_coords_from_address(f"{item_name}, {postcode}, UK")
|
||||
if location:
|
||||
print(f"Updating '{item_name}' with coords {location}")
|
||||
|
||||
# Step 3: Update location field
|
||||
monday.items.change_multiple_column_values(
|
||||
board_id,
|
||||
item_id,
|
||||
{location_col_id: location},
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error updating item '{item_name}': {e}")
|
||||
|
||||
else:
|
||||
print(f"No postcode found for item '{item_name}'")
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
|
||||
|
||||
class surveyPrice():
|
||||
"""
|
||||
A class to work out all prices and uploads to sharepoint
|
||||
|
|
@ -5,6 +7,14 @@ class surveyPrice():
|
|||
def __init__(self):
|
||||
pass
|
||||
|
||||
def download_price_card(self):
|
||||
pass
|
||||
|
||||
def get_price_card():
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Step one
|
||||
|
|
|
|||
14
poetry.lock
generated
14
poetry.lock
generated
|
|
@ -860,6 +860,18 @@ files = [
|
|||
[package.dependencies]
|
||||
traitlets = "*"
|
||||
|
||||
[[package]]
|
||||
name = "monday"
|
||||
version = "2.0.1"
|
||||
description = "A Python client library for Monday.com"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "monday-2.0.1-py3-none-any.whl", hash = "sha256:17f347dea47439c2181e2896a4c49f43289ea9c78a4156f240c652e54ceb193d"},
|
||||
{file = "monday-2.0.1.tar.gz", hash = "sha256:fd080590fd9b23a88e838a092d243d20ba6e2aea963405c4f21c02dc617daab1"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "msal"
|
||||
version = "1.32.0"
|
||||
|
|
@ -1913,4 +1925,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.12"
|
||||
content-hash = "5778c3bdce3109fe21e0ca7861ffc9c738d6af0545e0c27517bbe88c89209082"
|
||||
content-hash = "55a974b3a81d57c429f61ee6a12a84d38f5c703fdfdfdf2553bec6ba21c29bf5"
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ dependencies = [
|
|||
"alembic (>=1.15.1,<2.0.0)",
|
||||
"pytest (>=8.3.5,<9.0.0)",
|
||||
"hubspot-api-client (>=11.1.0,<12.0.0)",
|
||||
"monday (>=2.0.1,<3.0.0)",
|
||||
]
|
||||
|
||||
[tool.poetry]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue