overpass as class
Some checks failed
Build and deploy the backend to staging / Build and push image (pull_request) Successful in 2m4s
Run linting on the backend code / Build (pull_request) Successful in 29s
Run testing on the backend code / Build (pull_request) Failing after 4m39s
Build and deploy the backend to staging / Deploy to staging (pull_request) Successful in 24s

This commit is contained in:
Helldragon67 2025-01-23 16:02:33 +01:00
parent 1cc935fb34
commit 577ee232fc
5 changed files with 134 additions and 126 deletions

File diff suppressed because one or more lines are too long

View File

@ -5,116 +5,122 @@ import logging
import xml.etree.ElementTree as ET
from .caching_strategy import get_cache_key, CachingStrategy
from ..constants import OSM_CACHE_DIR
logger = logging.getLogger('Overpass')
osm_types = List[Literal['way', 'node', 'relation']]
logger = logging.getLogger('overpass')
ElementTypes = List[Literal['way', 'node', 'relation']]
def build_query(area: tuple, element_types: ElementTypes,
selector: str, conditions=[], out='center'):
class Overpass :
"""
Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data.
Args:
area (tuple): A tuple representing the geographical search area, typically in the format
(radius, latitude, longitude). The first element is a string like "around:2000"
specifying the search radius, and the second and third elements represent
the latitude and longitude as floats or strings.
element_types (list[str]): A list of OSM element types to search for. Must be one or more of
'Way', 'Node', or 'Relation'.
selector (str): The key or tag to filter the OSM elements (e.g., 'amenity', 'highway', etc.).
conditions (list, optional): A list of conditions to apply as additional filters for the
selected OSM elements. The conditions should be written in
the Overpass QL format, and they are combined with '&&' if
multiple are provided. Defaults to an empty list.
out (str, optional): Specifies the output type, such as 'center', 'body', or 'tags'.
Defaults to 'center'.
Returns:
str: The constructed Overpass QL query string.
Notes:
- If no conditions are provided, the query will just use the `selector` to filter the OSM
elements without additional constraints.
- The search area must always formatted as "(radius, lat, lon)".
Overpass class to manage the query building and sending to overpass api.
The caching strategy is a part of this class and initialized upon creation of the Overpass object.
"""
if not isinstance(conditions, list) :
conditions = [conditions]
if not isinstance(element_types, list) :
element_types = [element_types]
query = '('
# Round the radius to nearest 50 and coordinates to generate less queries
if area[0] > 500 :
search_radius = round(area[0] / 50) * 50
loc = tuple((round(area[1], 2), round(area[2], 2)))
else :
search_radius = round(area[0] / 25) * 25
loc = tuple((round(area[1], 3), round(area[2], 3)))
search_area = f"(around:{search_radius}, {str(loc[0])}, {str(loc[1])})"
if conditions :
conditions = '(if: ' + ' && '.join(conditions) + ')'
else :
conditions = ''
for elem in element_types :
query += elem + '[' + selector + ']' + conditions + search_area + ';'
query += ');' + f'out {out};'
return query
def __init__(self, caching_strategy: str = 'XML', cache_dir: str = OSM_CACHE_DIR) :
"""
Initialize the Overpass instance with the url, headers and caching strategy.
"""
self.overpass_url = "https://overpass-api.de/api/interpreter"
self.headers = {'User-Agent': 'Mozilla/5.0 (compatible; OverpassQuery/1.0; +http://example.com)',}
self.caching_strategy = CachingStrategy.use(caching_strategy, cache_dir=cache_dir)
def send_query(query: str) -> dict:
"""
Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
def build_query(self, area: tuple, osm_types: osm_types,
selector: str, conditions=[], out='center'):
"""
Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data.
Args:
query (str): The Overpass QL query to be sent to the Overpass API.
Args:
area (tuple): A tuple representing the geographical search area, typically in the format
(radius, latitude, longitude). The first element is a string like "around:2000"
specifying the search radius, and the second and third elements represent
the latitude and longitude as floats or strings.
osm_types (list[str]): A list of OSM element types to search for. Must be one or more of
'Way', 'Node', or 'Relation'.
selector (str): The key or tag to filter the OSM elements (e.g., 'amenity', 'highway', etc.).
conditions (list, optional): A list of conditions to apply as additional filters for the
selected OSM elements. The conditions should be written in
the Overpass QL format, and they are combined with '&&' if
multiple are provided. Defaults to an empty list.
out (str, optional): Specifies the output type, such as 'center', 'body', or 'tags'.
Defaults to 'center'.
Returns:
dict: The parsed JSON response from the Overpass API, or None if the request fails.
"""
Returns:
str: The constructed Overpass QL query string.
# Generate a cache key for the current query
cache_key = get_cache_key(query)
Notes:
- If no conditions are provided, the query will just use the `selector` to filter the OSM
elements without additional constraints.
- The search area must always formatted as "(radius, lat, lon)".
"""
if not isinstance(conditions, list) :
conditions = [conditions]
if not isinstance(osm_types, list) :
osm_types = [osm_types]
# Try to fetch the result from the cache
cached_response = CachingStrategy.get(cache_key)
if cached_response is not None :
logger.debug("Cache hit.")
return cached_response
query = '('
# Define the Overpass API endpoint
overpass_url = "https://overpass-api.de/api/interpreter"
# Round the radius to nearest 50 and coordinates to generate less queries
if area[0] > 500 :
search_radius = round(area[0] / 50) * 50
loc = tuple((round(area[1], 2), round(area[2], 2)))
else :
search_radius = round(area[0] / 25) * 25
loc = tuple((round(area[1], 3), round(area[2], 3)))
# Prepare the data to be sent as POST request, encoded as bytes
data = urllib.parse.urlencode({'data': query}).encode('utf-8')
search_area = f"(around:{search_radius}, {str(loc[0])}, {str(loc[1])})"
# Create a custom header with a User-Agent
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; OverpassQuery/1.0; +http://example.com)',
}
if conditions :
conditions = '(if: ' + ' && '.join(conditions) + ')'
else :
conditions = ''
try:
# Create a Request object with the specified URL, data, and headers
request = urllib.request.Request(overpass_url, data=data, headers=headers)
for elem in osm_types :
query += elem + '[' + selector + ']' + conditions + search_area + ';'
# Send the request and read the response
with urllib.request.urlopen(request) as response:
# Read and decode the response
response_data = response.read().decode('utf-8')
root = ET.fromstring(response_data)
query += ');' + f'out {out};'
# Cache the response data as an ElementTree root
CachingStrategy.set(cache_key, root)
logger.debug("Response data added to cache.")
return query
return root
except urllib.error.URLError as e:
raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
def send_query(self, query: str) -> dict:
"""
Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
Args:
query (str): The Overpass QL query to be sent to the Overpass API.
Returns:
dict: The parsed JSON response from the Overpass API, or None if the request fails.
"""
# Generate a cache key for the current query
cache_key = get_cache_key(query)
# Try to fetch the result from the cache
cached_response = self.caching_strategy.get(cache_key)
if cached_response is not None :
logger.debug("Cache hit.")
return cached_response
# Prepare the data to be sent as POST request, encoded as bytes
data = urllib.parse.urlencode({'data': query}).encode('utf-8')
try:
# Create a Request object with the specified URL, data, and headers
request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
# Send the request and read the response
with urllib.request.urlopen(request) as response:
# Read and decode the response
response_data = response.read().decode('utf-8')
root = ET.fromstring(response_data)
# Cache the response data as an ElementTree root
self.caching_strategy.set(cache_key, root)
logger.debug("Response data added to cache.")
return root
except urllib.error.URLError as e:
raise ConnectionError(f"Error connecting to Overpass API: {e}") from e

View File

@ -6,15 +6,14 @@ import numpy as np
from sklearn.cluster import DBSCAN
from pydantic import BaseModel
from ..overpass.overpass import build_query, send_query
from ..overpass.caching_strategy import CachingStrategy
from ..overpass.overpass import Overpass
from ..structs.landmark import Landmark
from .get_time_distance import get_distance
from ..constants import OSM_CACHE_DIR
# silence the overpass logger
logging.getLogger('overpass').setLevel(level=logging.CRITICAL)
logging.getLogger('Overpass').setLevel(level=logging.CRITICAL)
class Cluster(BaseModel):
@ -79,7 +78,9 @@ class ClusterManager:
Args:
bbox: The bounding box coordinates (around:radius, center_lat, center_lon).
"""
CachingStrategy.use('XML', cache_dir=OSM_CACHE_DIR)
# Setup the caching in the Overpass class.
self.overpass = Overpass(caching_strategy='XML', cache_dir=OSM_CACHE_DIR)
self.cluster_type = cluster_type
if cluster_type == 'shopping' :
@ -94,16 +95,16 @@ class ClusterManager:
raise NotImplementedError("Please choose only an available option for cluster detection")
# Initialize the points for cluster detection
query = build_query(
query = self.overpass.build_query(
area = bbox,
element_types = osm_types,
osm_types = osm_types,
selector = sel,
out = out
)
self.logger.debug(f"Cluster query: {query}")
try:
result = send_query(query)
result = self.overpass.send_query(query)
except Exception as e:
self.logger.error(f"Error fetching landmarks: {e}")
@ -243,15 +244,15 @@ class ClusterManager:
osm_types = ['node', 'way', 'relation']
for sel in selectors :
query = build_query(
query = self.overpass.build_query(
area = bbox,
element_types = osm_types,
osm_types = osm_types,
selector = sel,
out = 'ids center'
)
try:
result = send_query(query)
result = self.overpass.send_query(query)
except Exception as e:
self.logger.error(f"Error fetching landmarks: {e}")
continue

View File

@ -8,13 +8,12 @@ from ..structs.preferences import Preferences
from ..structs.landmark import Landmark
from .take_most_important import take_most_important
from .cluster_manager import ClusterManager
from ..overpass.overpass import build_query, send_query
from ..overpass.caching_strategy import CachingStrategy
from ..overpass.overpass import Overpass
from ..constants import AMENITY_SELECTORS_PATH, LANDMARK_PARAMETERS_PATH, OPTIMIZER_PARAMETERS_PATH, OSM_CACHE_DIR
# silence the overpass logger
logging.getLogger('overpass').setLevel(level=logging.CRITICAL)
logging.getLogger('Overpass').setLevel(level=logging.CRITICAL)
class LandmarkManager:
@ -56,7 +55,8 @@ class LandmarkManager:
self.walking_speed = parameters['average_walking_speed']
self.detour_factor = parameters['detour_factor']
CachingStrategy.use('XML', cache_dir=OSM_CACHE_DIR)
# Setup the caching in the Overpass class.
self.overpass = Overpass(caching_strategy='XML', cache_dir=OSM_CACHE_DIR)
self.logger.info('LandmakManager successfully initialized.')
@ -189,15 +189,15 @@ class LandmarkManager:
for sel in dict_to_selector_list(amenity_selector):
# self.logger.debug(f"Current selector: {sel}")
element_types = ['way', 'relation']
osm_types = ['way', 'relation']
if 'viewpoint' in sel :
query_conditions = []
element_types.append('node')
osm_types.append('node')
query = build_query(
query = self.overpass.build_query(
area = bbox,
element_types = element_types,
osm_types = osm_types,
selector = sel,
conditions = query_conditions, # except for nature....
out = 'center'
@ -205,7 +205,7 @@ class LandmarkManager:
self.logger.debug(f"Query: {query}")
try:
result = send_query(query)
result = self.overpass.send_query(query)
except Exception as e:
self.logger.error(f"Error fetching landmarks: {e}")
continue

View File

@ -2,14 +2,13 @@
import logging
import xml.etree.ElementTree as ET
from ..overpass.overpass import build_query, send_query
from ..overpass.caching_strategy import CachingStrategy
from ..overpass.overpass import Overpass
from ..structs.landmark import Toilets
from ..constants import OSM_CACHE_DIR
# silence the overpass logger
logging.getLogger('overpass').setLevel(level=logging.CRITICAL)
logging.getLogger('Overpass').setLevel(level=logging.CRITICAL)
class ToiletsManager:
"""
@ -40,7 +39,9 @@ class ToiletsManager:
self.radius = radius
self.location = location
CachingStrategy.use('XML', cache_dir=OSM_CACHE_DIR)
# Setup the caching in the Overpass class.
self.overpass = Overpass(caching_strategy='XML', cache_dir=OSM_CACHE_DIR)
def generate_toilet_list(self) -> list[Toilets] :
@ -56,16 +57,16 @@ class ToiletsManager:
osm_types = ['node', 'way', 'relation']
toilets_list = []
query = build_query(
query = self.overpass.build_query(
area = bbox,
element_types = osm_types,
osm_types = osm_types,
selector = '"amenity"="toilets"',
out = 'ids center tags'
)
self.logger.debug(f"Query: {query}")
try:
result = send_query(query)
result = self.overpass.send_query(query)
except Exception as e:
self.logger.error(f"Error fetching landmarks: {e}")
return None