first homemade OSM
Some checks failed
Build and deploy the backend to staging / Build and push image (pull_request) Successful in 2m50s
Run linting on the backend code / Build (pull_request) Successful in 26s
Run testing on the backend code / Build (pull_request) Failing after 1m44s
Build and deploy the backend to staging / Deploy to staging (pull_request) Successful in 24s

This commit is contained in:
Helldragon67 2025-01-22 20:21:00 +01:00
parent 98576cff0a
commit c668158341
13 changed files with 602 additions and 39 deletions

File diff suppressed because one or more lines are too long

View File

@ -60,15 +60,18 @@ sightseeing:
- cantilever
- abandoned
building:
- church
- chapel
- mosque
- synagogue
- ruins
- temple
# - government
- cathedral
- castle
# unused sightseeing/buildings:
# - church
# - chapel
# - mosque
# - synagogue
# - ruins
# - temple
# - government
# - cathedral
# - castle
# - museum
museums:

View File

@ -4,9 +4,9 @@ church_coeff: 0.65
nature_coeff: 1.35
overall_coeff: 10
tag_exponent: 1.15
image_bonus: 10
image_bonus: 1.1
viewpoint_bonus: 5
wikipedia_bonus: 4
wikipedia_bonus: 1.1
name_bonus: 3
N_important: 40
pay_bonus: -1

View File

@ -0,0 +1,288 @@
from typing import Literal, List, Optional
from pydantic import BaseModel
import urllib.request
import urllib.parse
import json
import yaml
from pathlib import Path
import xml.etree.ElementTree as ET
OSM_ENDPOINT = 'http://overpass-api.de/api/'
LOCATION_PREFIX = Path('src')
PARAMETERS_DIR = LOCATION_PREFIX / 'parameters'
AMENITY_SELECTORS_PATH = PARAMETERS_DIR / 'amenity_selectors.yaml'
ElementTypes = List[Literal['way', 'node', 'relation']]
# Output to frontend
class Landmark(BaseModel) :
"""
A class representing a landmark or point of interest (POI) in the context of a trip.
The Landmark class is used to model visitable locations, such as tourist attractions,
natural sites, shopping locations, and start/end points in travel itineraries. It
holds information about the landmark's attributes and supports comparisons and
calculations, such as distance between landmarks.
Attributes:
name (str): The name of the landmark.
type (Literal): The type of the landmark, which can be one of ['sightseeing', 'nature',
'shopping', 'start', 'finish'].
location (tuple): A tuple representing the (latitude, longitude) of the landmark.
osm_type (str): The OpenStreetMap (OSM) type of the landmark.
osm_id (int): The OpenStreetMap (OSM) ID of the landmark.
attractiveness (int): A score representing the attractiveness of the landmark.
n_tags (int): The number of tags associated with the landmark.
image_url (Optional[str]): A URL to an image of the landmark.
website_url (Optional[str]): A URL to the landmark's official website.
description (Optional[str]): A text description of the landmark.
duration (Optional[int]): The estimated time to visit the landmark (in minutes).
name_en (Optional[str]): The English name of the landmark.
uuid (UUID): A unique identifier for the landmark, generated by default using uuid4.
must_do (Optional[bool]): Whether the landmark is a "must-do" attraction.
must_avoid (Optional[bool]): Whether the landmark should be avoided.
is_secondary (Optional[bool]): Whether the landmark is secondary or less important.
time_to_reach_next (Optional[int]): Estimated time (in minutes) to reach the next landmark.
next_uuid (Optional[UUID]): UUID of the next landmark in sequence (if applicable).
"""
# Properties of the landmark
name : str
type: Literal['sightseeing', 'nature', 'shopping', 'start', 'finish']
location : tuple
osm_type : str
osm_id : int
attractiveness : int
n_tags : int
image_url : Optional[str] = None
website_url : Optional[str] = None
wiki_url : Optional[str] = None
description : Optional[str] = None # TODO future
duration : Optional[int] = 0
name_en : Optional[str] = None
# Additional properties depending on specific tour
must_do : Optional[bool] = False
must_avoid : Optional[bool] = False
is_secondary : Optional[bool] = False
time_to_reach_next : Optional[int] = 0
# More properties to define the score
is_viewpoint : Optional[bool] = False
is_cathedral : Optional[bool] = False
is_place_of_worship : Optional[bool] = False
def OverpassQueryBuilder(area: tuple, element_types: ElementTypes, selector: str,
conditions=[], out='center'):
if not isinstance(conditions, list) :
conditions = [conditions]
query = '('
search_area = f"({', '.join(map(str, area))})"
if conditions :
conditions = '(if: ' + ' && '.join(conditions) + ')'
else :
conditions = ''
for elem in element_types :
query += elem + '[' + selector + ']' + conditions + search_area + ';'
query += ');' + f'out {out};'
return query
def dict_to_selector_list(d: dict) -> list:
"""
Convert a dictionary of key-value pairs to a list of Overpass query strings.
Args:
d (dict): A dictionary of key-value pairs representing the selector.
Returns:
list: A list of strings representing the Overpass query selectors.
"""
return_list = []
for key, value in d.items():
if isinstance(value, list):
val = '|'.join(value)
return_list.append(f'{key}~"^({val})$"')
elif isinstance(value, str) and len(value) == 0:
return_list.append(f'{key}')
else:
return_list.append(f'{key}={value}')
return return_list
def send_overpass_query(query: str) -> dict:
"""
Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
Args:
query (str): The Overpass QL query to be sent to the Overpass API.
Returns:
dict: The parsed JSON response from the Overpass API, or None if the request fails.
"""
# Define the Overpass API endpoint
overpass_url = "https://overpass-api.de/api/interpreter"
# Prepare the data to be sent as POST request, encoded as bytes
data = urllib.parse.urlencode({'data': query}).encode('utf-8')
# Create a custom header with a User-Agent
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; OverpassQuery/1.0; +http://example.com)',
}
try:
# Create a Request object with the specified URL, data, and headers
request = urllib.request.Request(overpass_url, data=data, headers=headers)
# Send the request and read the response
with urllib.request.urlopen(request) as response:
# Read and decode the response
response_data = response.read().decode('utf-8')
return ET.fromstring(response_data)
except urllib.error.URLError as e:
print(f"Error connecting to Overpass API: {e}")
return None
except json.JSONDecodeError:
print("Error decoding the JSON response from Overpass API.")
return None
def parse_result(root: ET.Element, elem_type) -> List[Landmark]:
landmarks = []
if root is None :
return landmarks
for osm_type in ['node', 'way', 'relation'] :
for elem in root.findall(osm_type):
# Extract basic info from the landmark.
name = elem.find("tag[@k='name']").get('v') if elem.find("tag[@k='name']") is not None else None
center = elem.find('center')
tags = elem.findall('tag')
# Extract the center latitude and longitude if available.
if name is not None and center is not None:
lat = center.get('lat')
lon = center.get('lon')
coords = tuple((lat, lon))
else :
continue
# Convert this to Landmark object
landmark = Landmark(name=name,
type=elem_type,
location=coords,
osm_id=elem.get('id'),
osm_type=osm_type,
attractiveness=0,
n_tags=len(tags))
# Browse through tags to add information to landmark.
for tag in tags:
key = tag.get('k')
value = tag.get('v')
# Skip this landmark if not suitable.
if key == 'building:part' and value == 'yes' :
break
if 'disused:' in key :
break
if 'boundary:' in key :
break
if 'shop' in key and elem_type != 'shopping' :
break
# if value == 'apartments' :
# break
# Fill in the other attributes.
if key == 'image' :
landmark.image_url = value
if key == 'website' :
landmark.website_url = value
if key == 'place_of_worship' :
landmark.is_place_of_worship = True
if key == 'wikipedia' :
landmark.wiki_url = value
if key == 'name:en' :
landmark.name_en = value
if 'building:' in key or 'pay' in key :
landmark.n_tags -= 1
# Set the duration.
if value in ['museum', 'aquarium', 'planetarium'] :
landmark.duration = 60
elif value == 'viewpoint' :
landmark.is_viewpoint = True
landmark.duration = 10
elif value == 'cathedral' :
landmark.is_place_of_worship = False
landmark.duration = 10
else :
landmark.duration = 5
else:
set_score(landmark, elem_type)
landmarks.append(landmark)
continue
return landmarks
def set_score(landmark: Landmark, landmarktype: str) :
score = landmark.n_tags**1.15
if landmark.wiki_url :
score *= 1.1
if landmark.image_url :
score *= 1.1
if landmark.website_url :
score *= 1.1
if landmark.is_place_of_worship :
score *= 0.65
if landmark.is_viewpoint :
# print(f"{landmark.name}: n_tags={landmark.n_tags} and score={score*3*1.35*10}")
score *= 3
if landmarktype == 'nature' :
score *= 1.35
landmark.attractiveness = int(score * 10)
with AMENITY_SELECTORS_PATH.open('r') as f:
amenity_selectors = yaml.safe_load(f)
amenity_selector = amenity_selectors['nature']
bbox = tuple(('around:1714', 45.7576485, 4.8330241))
landmarks = []
for sel in dict_to_selector_list(amenity_selector):
query = OverpassQueryBuilder(area=bbox,
element_types=['way', 'relation'],
selector=sel,
# conditions='count_tags()>5',
out='center')
print(query + '\n')
root = send_overpass_query(query)
landmarks += parse_result(root, 'nature')
print(len(landmarks))

View File

@ -45,8 +45,11 @@ class Landmark(BaseModel) :
osm_id : int
attractiveness : int
n_tags : int
# Optional properties to gather more information.
image_url : Optional[str] = None
website_url : Optional[str] = None
wiki_url : Optional[str] = None
description : Optional[str] = None # TODO future
duration : Optional[int] = 0
name_en : Optional[str] = None
@ -62,6 +65,10 @@ class Landmark(BaseModel) :
time_to_reach_next : Optional[int] = 0
next_uuid : Optional[UUID] = None
# More properties to define the score
is_viewpoint : Optional[bool] = False
is_place_of_worship : Optional[bool] = False
def __str__(self) -> str:
"""
String representation of the Landmark object.

View File

@ -1,7 +1,7 @@
"""Linked and ordered list of Landmarks that represents the visiting order."""
from .landmark import Landmark
from ..utils.get_time_separation import get_time
from ..utils.get_time_distance import get_time
class LinkedLandmarks:
"""

View File

@ -11,7 +11,7 @@ def client():
"""Client used to call the app."""
return TestClient(app)
'''
def test_turckheim(client, request): # pylint: disable=redefined-outer-name
"""
Test n°1 : Custom test in Turckheim to ensure small villages are also supported.
@ -54,7 +54,7 @@ def test_turckheim(client, request): # pylint: disable=redefined-outer-name
assert len(landmarks) > 2 # check that there is something to visit
assert comp_time < 30, f"Computation time exceeded 30 seconds: {comp_time:.2f} seconds"
# assert 2==3
'''
def test_bellecour(client, request) : # pylint: disable=redefined-outer-name
"""
@ -67,6 +67,7 @@ def test_bellecour(client, request) : # pylint: disable=redefined-outer-name
start_time = time.time() # Start timer
duration_minutes = 120
response = client.post(
"/trip/new",
json={
@ -96,7 +97,7 @@ def test_bellecour(client, request) : # pylint: disable=redefined-outer-name
assert duration_minutes*0.8 < int(result['total_time']) < duration_minutes*1.2
# assert 2 == 3
'''
def test_cologne(client, request) : # pylint: disable=redefined-outer-name
"""
Test n°2 : Custom test in Lyon centre to ensure proper decision making in crowded area.
@ -335,7 +336,7 @@ def test_shopping(client, request) : # pylint: disable=redefined-outer-name
assert response.status_code == 200 # check for successful planning
assert comp_time < 30, f"Computation time exceeded 30 seconds: {comp_time:.2f} seconds"
assert duration_minutes*0.8 < int(result['total_time']) < duration_minutes*1.2
'''
# def test_new_trip_single_prefs(client):
# response = client.post(

View File

@ -9,7 +9,7 @@ from OSMPythonTools.overpass import Overpass, overpassQueryBuilder
from OSMPythonTools.cachingStrategy import CachingStrategy, JSON
from ..structs.landmark import Landmark
from ..utils.get_time_separation import get_distance
from .get_time_distance import get_distance
from ..constants import OSM_CACHE_DIR

View File

@ -1,4 +1,4 @@
"""Computes the distance (in meters) or the walking time (in minutes) between two coordinates."""
"""Contains various helper functions to help with distance or score computations."""
from math import sin, cos, sqrt, atan2, radians
import yaml

View File

@ -1,13 +1,12 @@
"""Module used to import data from OSM and arrange them in categories."""
import logging
import yaml
from OSMPythonTools.overpass import Overpass, overpassQueryBuilder
from OSMPythonTools.cachingStrategy import CachingStrategy, JSON
from ..structs.preferences import Preferences
from ..structs.landmark import Landmark
from .take_most_important import take_most_important
from .cluster_manager import ClusterManager
from .overpass import OverpassQueryBuilder, send_overpass_query, parse_result
from ..constants import AMENITY_SELECTORS_PATH, LANDMARK_PARAMETERS_PATH, OPTIMIZER_PARAMETERS_PATH, OSM_CACHE_DIR
@ -54,8 +53,8 @@ class LandmarkManager:
self.walking_speed = parameters['average_walking_speed']
self.detour_factor = parameters['detour_factor']
self.overpass = Overpass()
CachingStrategy.use(JSON, cacheDir=OSM_CACHE_DIR)
# self.overpass = Overpass()
# CachingStrategy.use(JSON, cacheDir=OSM_CACHE_DIR)
self.logger.info('LandmakManager successfully initialized.')
@ -78,13 +77,13 @@ class LandmarkManager:
- A list of the most important landmarks based on the user's preferences.
"""
self.logger.debug('Starting to fetch landmarks...')
max_walk_dist = (preferences.max_time_minute/2)/60*self.walking_speed*1000/self.detour_factor
max_walk_dist = int((preferences.max_time_minute/2)/60*self.walking_speed*1000/self.detour_factor)
reachable_bbox_side = min(max_walk_dist, self.max_bbox_side)
# use set to avoid duplicates, this requires some __methods__ to be set in Landmark
all_landmarks = set()
# Create a bbox using the around technique
# Create a bbox using the around technique, tuple of strings
bbox = tuple((f"around:{min(2000, reachable_bbox_side/2)}", str(center_coordinates[0]), str(center_coordinates[1])))
# list for sightseeing
@ -134,7 +133,21 @@ class LandmarkManager:
return all_landmarks, landmarks_constrained
def set_score(self, landmark: Landmark, landmarktype: str, preference_level: int) :
score = landmark.n_tags**self.tag_exponent
if landmark.wiki_url :
score *= self.wikipedia_bonus
if landmark.image_url :
score *= self.image_bonus
if landmark.website_url :
score *= self.wikipedia_bonus
if landmark.is_place_of_worship :
score *= self.church_coeff
landmark.attractiveness = int(score * preference_level)
'''
def fetch_landmarks(self, bbox: tuple, amenity_selector: dict, landmarktype: str, score_function: callable) -> list[Landmark]:
"""
Fetches landmarks of a specified type from OpenStreetMap (OSM) within a bounding box centered on given coordinates.
@ -170,15 +183,11 @@ class LandmarkManager:
query_conditions = []
element_types.append('node')
query = overpassQueryBuilder(
bbox = bbox,
elementType = element_types,
# selector can in principle be a list already,
# but it generates the intersection of the queries
# we want the union
query = OverpassQueryBuilder(
area = bbox,
element_types = element_types,
selector = sel,
conditions = query_conditions, # except for nature....
includeCenter = True,
out = 'center'
)
self.logger.debug(f"Query: {query}")
@ -295,7 +304,63 @@ class LandmarkManager:
self.logger.debug(f"Fetched {len(return_list)} landmarks of type {landmarktype} in {bbox}")
return return_list
'''
def fetch_landmarks(self, bbox: tuple, amenity_selector: dict, landmarktype: str, score_function: callable) -> list[Landmark]:
"""
Fetches landmarks of a specified type from OpenStreetMap (OSM) within a bounding box centered on given coordinates.
Args:
bbox (tuple[float, float, float, float]): The bounding box coordinates (around:radius, center_lat, center_lon).
amenity_selector (dict): The Overpass API query selector for the desired landmark type.
landmarktype (str): The type of the landmark (e.g., 'sightseeing', 'nature', 'shopping').
score_function (callable): The function to compute the score of the landmark based on its attributes.
Returns:
list[Landmark]: A list of Landmark objects that were fetched and filtered based on the provided criteria.
Notes:
- Landmarks are fetched using Overpass API queries.
- Selectors are translated from the dictionary to the Overpass query format. (e.g., 'amenity'='place_of_worship')
- Landmarks are filtered based on various conditions including tags and type.
- Scores are assigned to landmarks based on their attributes and surrounding elements.
"""
return_list = []
if landmarktype == 'nature' : query_conditions = []
else : query_conditions = ['count_tags()>5']
# caution, when applying a list of selectors, overpass will search for elements that match ALL selectors simultaneously
# we need to split the selectors into separate queries and merge the results
for sel in dict_to_selector_list(amenity_selector):
# self.logger.debug(f"Current selector: {sel}")
element_types = ['way', 'relation']
if 'viewpoint' in sel :
query_conditions = []
element_types.append('node')
query = OverpassQueryBuilder(
area = bbox,
element_types = element_types,
selector = sel,
conditions = query_conditions, # except for nature....
out = 'center'
)
self.logger.debug(f"Query: {query}")
try:
result = send_overpass_query(query)
except Exception as e:
self.logger.error(f"Error fetching landmarks: {e}")
continue
return_list = parse_result(result, landmarktype)
self.logger.debug(f"Fetched {len(return_list)} landmarks of type {landmarktype} in {bbox}")
return return_list
def dict_to_selector_list(d: dict) -> list:
"""

View File

@ -6,7 +6,7 @@ import numpy as np
import pulp as pl
from ..structs.landmark import Landmark
from .get_time_separation import get_time
from .get_time_distance import get_time
from ..constants import OPTIMIZER_PARAMETERS_PATH

View File

@ -0,0 +1,199 @@
from typing import Literal, List
import urllib
import json
import xml.etree.ElementTree as ET
from ..structs.landmark import Landmark
ElementTypes = List[Literal['way', 'node', 'relation']]
def OverpassQueryBuilder(area: tuple, element_types: ElementTypes, selector: str,
conditions=[], out='center'):
"""
Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data.
Args:
area (tuple): A tuple representing the geographical search area, typically in the format
(radius, latitude, longitude). The first element is a string like "around:2000"
specifying the search radius, and the second and third elements represent
the latitude and longitude as floats or strings.
element_types (list[str]): A list of OSM element types to search for. Must be one or more of
'Way', 'Node', or 'Relation'.
selector (str): The key or tag to filter the OSM elements (e.g., 'amenity', 'highway', etc.).
conditions (list, optional): A list of conditions to apply as additional filters for the
selected OSM elements. The conditions should be written in
the Overpass QL format, and they are combined with '&&' if
multiple are provided. Defaults to an empty list.
out (str, optional): Specifies the output type, such as 'center', 'body', or 'tags'.
Defaults to 'center'.
Returns:
str: The constructed Overpass QL query string.
Notes:
- If no conditions are provided, the query will just use the `selector` to filter the OSM
elements without additional constraints.
- The search area must always formatted as "(radius, lat, lon)".
"""
if not isinstance(conditions, list) :
conditions = [conditions]
query = '('
search_area = f"({', '.join(map(str, area))})"
if conditions :
conditions = '(if: ' + ' && '.join(conditions) + ')'
else :
conditions = ''
for elem in element_types :
query += elem + '[' + selector + ']' + conditions + search_area + ';'
query += ');' + f'out {out};'
return query
def send_overpass_query(query: str) -> dict:
"""
Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
Args:
query (str): The Overpass QL query to be sent to the Overpass API.
Returns:
dict: The parsed JSON response from the Overpass API, or None if the request fails.
"""
# Define the Overpass API endpoint
overpass_url = "https://overpass-api.de/api/interpreter"
# Prepare the data to be sent as POST request, encoded as bytes
data = urllib.parse.urlencode({'data': query}).encode('utf-8')
# Create a custom header with a User-Agent
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; OverpassQuery/1.0; +http://example.com)',
}
try:
# Create a Request object with the specified URL, data, and headers
request = urllib.request.Request(overpass_url, data=data, headers=headers)
# Send the request and read the response
with urllib.request.urlopen(request) as response:
# Read and decode the response
response_data = response.read().decode('utf-8')
return ET.fromstring(response_data)
except urllib.error.URLError as e:
print(f"Error connecting to Overpass API: {e}")
return None
except json.JSONDecodeError:
print("Error decoding the JSON response from Overpass API.")
return None
def parse_result(root: ET.Element, elem_type) -> List[Landmark]:
landmarks = []
if root is None :
return landmarks
for osm_type in ['node', 'way', 'relation'] :
for elem in root.findall(osm_type):
# Extract basic info from the landmark.
name = elem.find("tag[@k='name']").get('v') if elem.find("tag[@k='name']") is not None else None
center = elem.find('center')
tags = elem.findall('tag')
# Extract the center latitude and longitude if available.
if name is not None and center is not None:
lat = float(center.get('lat'))
lon = float(center.get('lon'))
coords = tuple((lat, lon))
else :
continue
# Convert this to Landmark object
landmark = Landmark(name=name,
type=elem_type,
location=coords,
osm_id=elem.get('id'),
osm_type=osm_type,
attractiveness=0,
n_tags=len(tags))
# Browse through tags to add information to landmark.
for tag in tags:
key = tag.get('k')
value = tag.get('v')
# Skip this landmark if not suitable.
if key == 'building:part' and value == 'yes' :
break
if 'disused:' in key :
break
if 'boundary:' in key :
break
if 'shop' in key and elem_type != 'shopping' :
break
# if value == 'apartments' :
# break
# Fill in the other attributes.
if key == 'image' :
landmark.image_url = value
if key == 'website' :
landmark.website_url = value
if key == 'place_of_worship' :
landmark.is_place_of_worship = True
if key == 'wikipedia' :
landmark.wiki_url = value
if key == 'name:en' :
landmark.name_en = value
if 'building:' in key or 'pay' in key :
landmark.n_tags -= 1
# Set the duration.
if value in ['museum', 'aquarium', 'planetarium'] :
landmark.duration = 60
elif value == 'viewpoint' :
landmark.is_viewpoint = True
landmark.duration = 10
elif value == 'cathedral' :
landmark.is_place_of_worship = False
landmark.duration = 10
else :
landmark.duration = 5
else:
set_score(landmark, elem_type)
landmarks.append(landmark)
continue
return landmarks
def set_score(landmark: Landmark, landmarktype: str) :
score = landmark.n_tags**1.15
if landmark.wiki_url :
score *= 1.1
if landmark.image_url :
score *= 1.1
if landmark.website_url :
score *= 1.1
if landmark.is_place_of_worship :
score *= 0.65
if landmark.is_viewpoint :
# print(f"{landmark.name}: n_tags={landmark.n_tags} and score={score*3*1.35*10}")
score *= 3
if landmarktype == 'nature' :
score *= 1.35
landmark.attractiveness = int(score * 10)

View File

@ -6,7 +6,7 @@ from shapely import buffer, LineString, Point, Polygon, MultiPoint, concave_hull
from ..structs.landmark import Landmark
from . import take_most_important, get_time_separation
from . import get_time_distance, take_most_important
from .optimizer import Optimizer
from ..constants import OPTIMIZER_PARAMETERS_PATH
@ -195,7 +195,7 @@ class Refiner :
# Step 4: Use nearest neighbor heuristic to visit all landmarks
while unvisited_landmarks:
nearest_landmark = min(unvisited_landmarks, key=lambda lm: get_time_separation.get_time(current_landmark.location, lm.location))
nearest_landmark = min(unvisited_landmarks, key=lambda lm: get_time_distance.get_time(current_landmark.location, lm.location))
path.append(nearest_landmark)
coordinates.append(nearest_landmark.location)
current_landmark = nearest_landmark