Merge pull request 'feature/backend/better_time_management' (#34) from feature/backend/better_time_management into main

Reviewed-on: #34

.gitea/workflows/frontend_build-android.yaml

@@ -6,7 +6,7 @@ on:
       - frontend/**
 
 
-name: Build and release APK
+name: Build and release debug APK
 
 jobs:
   build:
@@ -55,7 +55,7 @@ jobs:
         ls -lah android
       working-directory: ./frontend
 
-    - run: flutter build apk --release --split-per-abi --build-number=${{ gitea.run_number }}
+    - run: flutter build apk --debug --split-per-abi --build-number=${{ gitea.run_number }}
       working-directory: ./frontend
 
     - name: Upload APKs to artifacts

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "cmake.ignoreCMakeListsMissing": true
+}

backend/Pipfile

@@ -9,9 +9,9 @@ name = "pypi"
 numpy = "*"
 fastapi = "*"
 pydantic = "*"
-geopy = "*"
 shapely = "*"
 scipy = "*"
 osmpythontools = "*"
 pywikibot = "*"
 pymemcache = "*"
+fastapi-cli = "*"

backend/README.md

@@ -1,12 +1,37 @@
 # Backend
 
-This repository contains the backend code for the application. It utilizes FastAPI that allows to quickly create a RESTful API that exposes the endpoints of the route optimizer.
-
+This repository contains the backend code for the application. It utilizes **FastAPI** to quickly create a RESTful API that exposes the endpoints of the route optimizer.
 
 ## Getting Started
-- The code of the python application is located in the `src` directory.
-- Package management is handled with `pipenv` and the dependencies are listed in the `Pipfile`.
-- Since the application is aimed to be deployed in a container, the `Dockerfile` is provided to build the image.
+
+### Directory Structure
+- The code for the Python application is located in the `src` directory.
+- Package management is handled with **pipenv**, and the dependencies are listed in the `Pipfile`.
+- Since the application is designed to be deployed in a container, the `Dockerfile` is provided to build the image.
+
+### Setting Up the Development Environment
+
+To set up your development environment using **pipenv**, follow these steps:
+
+1. Install `pipenv` by running:
+    ```bash
+    sudo apt install pipenv
+    ```
+
+2. Create and activate a virtual environment:
+    ```bash
+    pipenv shell
+    ```
+
+3. Install the dependencies listed in the `Pipfile`:
+    ```bash
+    pipenv install
+    ```
+
+4. The virtual environment will be created under:
+    ```bash
+    ~/.local/share/virtualenvs/...
+    ```
 
 ### Deployment
 To deploy the backend docker container, we use kubernetes. Modifications to the backend are automatically pushed to a two-stage environment through the CI pipeline. See [deployment/README](deployment/README.md] for further information.

backend/src/constants.py

@@ -16,7 +16,7 @@ OSM_CACHE_DIR = Path(cache_dir_string)
 
 import logging
 # if we are in a debug session, set verbose and rich logging
-if os.getenv('DEBUG', False):
+if os.getenv('DEBUG', "false") == "true":
     from rich.logging import RichHandler
     logging.basicConfig(
         level=logging.DEBUG,

backend/src/parameters/amenity_selectors.yaml

@@ -1,3 +1,6 @@
+# Tags were picked mostly arbitrarily, based on the OSM wiki and the OSM tags page.
+# See https://taginfo.openstreetmap.org for more inspiration.
+
 nature:
   leisure: park
   geological: ''
@@ -11,7 +14,24 @@ nature:
     - alpine_hut
     - viewpoint
     - zoo
-  waterway: waterfall
+    - resort
+    - picnic_site
+  water:
+    - pond
+    - lake
+    - river
+    - basin
+    - stream
+    - lagoon
+    - rapids
+  waterway:
+    - waterfall
+    - river
+    - canal
+    - dam
+    - dock
+    - boatyard
+
 
 shopping:
   shop:
@@ -23,10 +43,47 @@ sightseeing:
     - museum
     - attraction
     - gallery
+    - artwork
+    - aquarium
   historic: ''
   amenity:
     - planetarium
     - place_of_worship
     - fountain
+    - townhall
   water:
     - reflecting_pool
+  bridge:
+    - aqueduct
+    - viaduct
+    - boardwalk
+    - cantilever
+    - abandoned
+  building:
+    - church
+    - chapel
+    - mosque
+    - synagogue
+    - ruins
+    - temple
+    - government
+    - cathedral
+    - castle
+    - museum
+
+
+
+
+# to be used later on
+restauration:
+  shop:
+    - coffee
+    - bakery
+    - restaurant
+    - pastry
+  amenity:
+    - restaurant
+    - cafe
+    - ice_cream
+    - food_court
+    - biergarten

backend/src/persistence.py

@@ -21,8 +21,8 @@ if constants.MEMCACHED_HOST_PATH is None:
 else:
     client = Client(
         constants.MEMCACHED_HOST_PATH,
-        timeout=1,
-        allow_unicode_keys=True,
-        encoding='utf-8',
-        serde=serde.pickle_serde
+        timeout = 1,
+        allow_unicode_keys = True,
+        encoding = 'utf-8',
+        serde = serde.pickle_serde
     )

backend/src/structs/landmark.py

@@ -5,7 +5,7 @@ from uuid import uuid4
 
 # Output to frontend
 class Landmark(BaseModel) :
-    
+
     # Properties of the landmark
     name : str
     type: Literal['sightseeing', 'nature', 'shopping', 'start', 'finish']
@@ -22,22 +22,22 @@ class Landmark(BaseModel) :
 
     # Unique ID of a given landmark
     uuid: str = Field(default_factory=uuid4)
-    
+
     # Additional properties depending on specific tour
     must_do : Optional[bool] = False
     must_avoid : Optional[bool] = False
     is_secondary : Optional[bool] = False                       # TODO future   
-    
+
     time_to_reach_next : Optional[int] = 0
     next_uuid : Optional[str] = None
-    
+
     def __str__(self) -> str:
         time_to_next_str = f", time_to_next={self.time_to_reach_next}" if self.time_to_reach_next else ""
         is_secondary_str = f", secondary" if self.is_secondary else ""
         type_str = '(' + self.type + ')'
         if self.type in ["start", "finish", "nature", "shopping"] : type_str += '\t '
         return f'Landmark{type_str}: [{self.name} @{self.location}, score={self.attractiveness}{time_to_next_str}{is_secondary_str}]'
-    
+
     def distance(self, value: 'Landmark') -> float:
         return (self.location[0] - value.location[0])**2 + (self.location[1] - value.location[1])**2
 

backend/src/structs/trip.py

@@ -22,7 +22,8 @@ class Trip(BaseModel):
 
         # Store the trip in the cache
         cache_client.set(f"trip_{trip.uuid}", trip)
-        cache_client.set_many({f"landmark_{landmark.uuid}": landmark for landmark in landmarks}, expire=3600)
+        # make sure to await the result (noreply=False). Otherwise the cache might not be inplace when the trip is actually requested
+        cache_client.set_many({f"landmark_{landmark.uuid}": landmark for landmark in landmarks}, expire=3600, noreply=False)
         # is equivalent to:
         # for landmark in landmarks:
         #     cache_client.set(f"landmark_{landmark.uuid}", landmark, expire=3600)

backend/src/tester.py

@@ -23,7 +23,7 @@ def test(start_coords: tuple[float, float], finish_coords: tuple[float, float] =
         sightseeing=Preference(type='sightseeing', score = 5),
         nature=Preference(type='nature', score = 5),
         shopping=Preference(type='shopping', score = 5),
-        max_time_minute=100,
+        max_time_minute=15,
         detour_tolerance_minute=0
     )
 
@@ -74,6 +74,7 @@ def test(start_coords: tuple[float, float], finish_coords: tuple[float, float] =
 # test(tuple((48.8344400, 2.3220540)))       # Café Chez César 
 # test(tuple((48.8375946, 2.2949904)))       # Point random
 # test(tuple((47.377859, 8.540585)))         # Zurich HB
-# test(tuple((45.758217, 4.831814)))      # Lyon Bellecour
-test(tuple((48.5848435, 7.7332974)))      # Strasbourg Gare
+# test(tuple((45.758217, 4.831814)))         # Lyon Bellecour
+# test(tuple((48.5848435, 7.7332974)))       # Strasbourg Gare
 # test(tuple((48.2067858, 16.3692340)))      # Vienne
+test(tuple((48.084588, 7.280405)))         # Turckheim 

backend/src/utils/get_time_separation.py

@@ -1,5 +1,5 @@
 import yaml
-from geopy.distance import geodesic
+from math import sin, cos, sqrt, atan2, radians
 
 import constants
 
@@ -8,6 +8,7 @@ with constants.OPTIMIZER_PARAMETERS_PATH.open('r') as f:
     DETOUR_FACTOR = parameters['detour_factor']
     AVERAGE_WALKING_SPEED = parameters['average_walking_speed']
 
+EARTH_RADIUS_KM = 6373
 
 def get_time(p1: tuple[float, float], p2: tuple[float, float]) -> int:
     """
@@ -22,16 +23,28 @@ def get_time(p1: tuple[float, float], p2: tuple[float, float]) -> int:
     """
 
 
-    # Compute the straight-line distance in km
-    if p1 == p2 :
+    if p1 == p2:
         return 0
-    else: 
-        dist = geodesic(p1, p2).kilometers
+    else:
+        # Compute the distance in km along the surface of the Earth
+        # (assume spherical Earth)
+        # this is the haversine formula, stolen from stackoverflow
+        # in order to not use any external libraries
+        lat1, lon1 = radians(p1[0]), radians(p1[1])
+        lat2, lon2 = radians(p2[0]), radians(p2[1])
 
-    # Consider the detour factor for average cityto deterline walking distance (in km)
-    walk_dist = dist*DETOUR_FACTOR
+        dlon = lon2 - lon1
+        dlat = lat2 - lat1
+
+        a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
+        c = 2 * atan2(sqrt(a), sqrt(1 - a))
+
+        distance = EARTH_RADIUS_KM * c
+
+    # Consider the detour factor for average an average city
+    walk_distance = distance * DETOUR_FACTOR
 
     # Time to walk this distance (in minutes)
-    walk_time = walk_dist/AVERAGE_WALKING_SPEED*60
+    walk_time = walk_distance / AVERAGE_WALKING_SPEED * 60
 
     return round(walk_time)

backend/src/utils/landmarks_manager.py

@@ -10,7 +10,8 @@ from structs.landmark import Landmark
 from .take_most_important import take_most_important
 import constants
 
-
+# silence the overpass logger
+logging.getLogger('OSMPythonTools').setLevel(level=logging.CRITICAL)
 
 
 class LandmarkManager:
@@ -93,6 +94,8 @@ class LandmarkManager:
         if preferences.shopping.score != 0:
             score_function = lambda score: score * 10 * preferences.shopping.score / 5
             current_landmarks = self.fetch_landmarks(bbox, self.amenity_selectors['shopping'], preferences.shopping.type, score_function)
+            # set time for all shopping activites :
+            for landmark in current_landmarks : landmark.duration = 45
             all_landmarks.update(current_landmarks)
 
 
@@ -206,11 +209,15 @@ class LandmarkManager:
             query = overpassQueryBuilder(
                 bbox = bbox,
                 elementType = ['way', 'relation'],
+                # selector can in principle be a list already,
+                # but it generates the intersection of the queries
+                # we want the union
                 selector = sel,
-                # conditions = [],
+                conditions = ['count_tags()>5'],
                 includeCenter = True,
                 out = 'body'
                 )
+            self.logger.debug(f"Query: {query}")
 
             try:
                 result = self.overpass.query(query)
@@ -241,11 +248,11 @@ class LandmarkManager:
                 image_url = None
                 name_en = None
 
-                # remove specific tags
+                # Adjust scoring
                 skip = False
                 for tag in elem.tags().keys():
                     if "pay" in tag:
-                        # payment options are a good sign
+                        # payment options are misleading and should not count for the scoring.
                         score += self.pay_bonus
 
                     if "disused" in tag:
@@ -258,10 +265,12 @@ class LandmarkManager:
                         score += self.wikipedia_bonus
 
                     if "viewpoint" in tag:
+                        # viewpoints must count more
                         score += self.viewpoint_bonus
                         duration = 10
 
                     if "image" in tag:
+                        # images must count more
                         score += self.image_bonus
 
                     if elem_type != "nature":
@@ -277,6 +286,7 @@ class LandmarkManager:
                             skip = True
                             break
 
+                    # Extract image, website and english name
                     if tag in ['website', 'contact:website']:
                         website_url = elem.tag(tag)
                     if tag == 'image':
@@ -290,10 +300,9 @@ class LandmarkManager:
                 score = score_function(score)
                 if "place_of_worship" in elem.tags().values():
                     score = score * self.church_coeff
-                    duration = 15
+                    duration = 10
                 
-                elif "museum" in elem.tags().values():
-                    score = score * self.church_coeff
+                elif "museum" in elem.tags().values() or "aquarium" in elem.tags().values() or "planetarium" in elem.tags().values():
                     duration = 60
                 
                 else:
@@ -336,7 +345,7 @@ def dict_to_selector_list(d: dict) -> list:
     for key, value in d.items():
         if type(value) == list:
             val = '|'.join(value)
-            return_list.append(f'{key}~"{val}"')
+            return_list.append(f'{key}~"^({val})$"')
         elif type(value) == str and len(value) == 0:
             return_list.append(f'{key}')
         else:

backend/src/utils/optimizer.py

@@ -3,7 +3,6 @@ import numpy as np
 
 from scipy.optimize import linprog
 from collections import defaultdict, deque
-from geopy.distance import geodesic
 
 from structs.landmark import Landmark
 from .get_time_separation import get_time
@@ -488,7 +487,7 @@ class Optimizer:
 
         # Raise error if no solution is found
         if not res.success :
-            raise ArithmeticError("No solution could be found, the problem is overconstrained. Please adapt your must_dos")
+            raise ArithmeticError("No solution could be found, the problem is overconstrained. Try with a longer trip (>30 minutes).")
 
         # If there is a solution, we're good to go, just check for connectiveness
         order, circles = self.is_connected(res.x)

backend/src/utils/refiner.py

@@ -2,6 +2,7 @@ import yaml, logging
 
 from shapely import buffer, LineString, Point, Polygon, MultiPoint, concave_hull
 from math import pi
+from typing import List
 
 from structs.landmark import Landmark
 from . import take_most_important, get_time_separation
@@ -133,6 +134,21 @@ class Refiner :
             i += 1
     
         return tour
+    
+    def integrate_landmarks(self, sub_list: List[Landmark], main_list: List[Landmark]) :
+        """
+        Inserts 'sub_list' of Landmarks inside the 'main_list' by leaving the ends untouched.
+        
+        Args: 
+            sub_list    : the list of Landmarks to be inserted inside of the 'main_list'.
+            main_list   : the original list with start and finish.
+
+        Returns:
+            the full list.
+        """
+        sub_list.append(main_list[-1])          # add finish back
+        return main_list[:-1] + sub_list        # create full set of possible landmarks
+
 
 
     def find_shortest_path_through_all_landmarks(self, landmarks: list[Landmark]) -> tuple[list[Landmark], Polygon]:
@@ -253,6 +269,11 @@ class Refiner :
         except :
             better_tour_poly = concave_hull(MultiPoint(coords))  # Create concave hull with "core" of tour leaving out start and finish
             xs, ys = better_tour_poly.exterior.xy
+            """ 
+            ERROR HERE : 
+                Exception has occurred: AttributeError
+                'LineString' object has no attribute 'exterior'
+            """
 
 
         # reverse the xs and ys
@@ -315,26 +336,30 @@ class Refiner :
 
         self.logger.info(f"Using {len(minor_landmarks)} minor landmarks around the predicted path")
 
-        # full set of visitable landmarks
-        full_set = base_tour[:-1] + minor_landmarks   # create full set of possible landmarks (without finish)
-        full_set.append(base_tour[-1])                # add finish back
+        # Full set of visitable landmarks.
+        full_set = self.integrate_landmarks(minor_landmarks, base_tour)     # could probably be optimized with less overhead
 
-        # get a new tour
+        # Generate a new tour with the optimizer.
         new_tour = self.optimizer.solve_optimization(
             max_time = max_time + detour,
             landmarks = full_set, 
             max_landmarks = self.max_landmarks_refiner
         )
 
+        # If unsuccessful optimization, use the base_tour.
         if new_tour is None:
             self.logger.warning("No solution found for the refined tour. Returning the initial tour.")
             new_tour = base_tour
 
+        # If only one landmark, return it.
+        if len(new_tour) < 4 :
+            return new_tour
 
-        # Find shortest path using the nearest neighbor heuristic
+        # Find shortest path using the nearest neighbor heuristic.
         better_tour, better_poly = self.find_shortest_path_through_all_landmarks(new_tour)
 
-        # Fix the tour using Polygons if the path looks weird
+        # Fix the tour using Polygons if the path looks weird. 
+        # Conditions : circular trip and invalid polygon.
         if base_tour[0].location == base_tour[-1].location and not better_poly.is_valid :
             better_tour = self.fix_using_polygon(better_tour)
 

frontend/.github/workflows/build_app_android.yaml

@@ -37,12 +37,12 @@ jobs:
           REF_NAME: ${{ github.ref_name }}
         run:
           # remove the 'v' prefix from the tag name
-          echo "VERSION_NAME=${REF_NAME//v}" >> $GITHUB_ENV
+          echo "BUILD_NAME=${REF_NAME//v}" >> $GITHUB_ENV
 
       - name: Load secrets from github
         run: |
           echo "${{ secrets.ANDROID_SECRET_PROPERTIES_BASE64 }}" | base64 -d > secrets.properties
-          echo "${{ secrets.ANDROID_GOOGLE_PLAY_JSON_BASE64 }}" | base64 -d > fastlane/google-key.json
+          echo "${{ secrets.ANDROID_GOOGLE_PLAY_JSON_BASE64 }}" | base64 -d > google-key.json
           echo "${{ secrets.ANDROID_KEYSTORE_BASE64 }}" | base64 -d > release.keystore
         working-directory: android
 
@@ -53,4 +53,6 @@ jobs:
       - name: Run fastlane lane
         run: bundle exec fastlane deploy_testing
         working-directory: android
-        # the environment variable VERSION_NAME is implicitly available
+        env:
+          BUILD_NUMBER: ${{ github.run_number }}
+          # BUILD_NAME is implicitly available

frontend/android/app/build.gradle

@@ -30,14 +30,19 @@ if (flutterVersionName == null) {
 
 
 def secretPropertiesFile = rootProject.file('secrets.properties')
+def fallbackPropertiesFile = rootProject.file('fallback.properties')
 def secretProperties = new Properties()
 
 if (secretPropertiesFile.exists()) {
     secretPropertiesFile.withReader('UTF-8') { reader ->
         secretProperties.load(reader)
     }
+} else if (fallbackPropertiesFile.exists()) {
+    fallbackPropertiesFile.withReader('UTF-8') { reader ->
+        secretProperties.load(reader)
+    }
 } else {
-    throw new GradleException("Secrets file secrets.properties not found")
+    throw new GradleException("Secrets file (secrets.properties, fallback.properties) not found")
 }
 
 

frontend/android/fallback.properties

@@ -1 +1,3 @@
+# This file mirrors the state of secrets.properties as a reference for the developer.
+# And as a fallback for build.gradle
 MAPS_API_KEY=Key

frontend/android/fastlane/Fastfile

@@ -5,22 +5,28 @@ default_platform(:android)
 
 platform :android do
 
-  desc "Deploy a new version as a preview version"
+  desc "Deploy a new version to closed testing"
   lane :deploy_testing do
-    version_name = ENV["VERSION_NAME"]
+    build_name = ENV["BUILD_NAME"]
+    build_number = ENV["BUILD_NUMBER"]
 
     sh(
       "flutter",
       "build",
       "appbundle",
       "--release",
-      "--build-name=#{version_name}",
+      "--build-name=#{build_name}",
+      "--build-number=#{build_number}",
       )
     
     upload_to_play_store(
       track: 'alpha',
       skip_upload_apk: true,
       skip_upload_changelogs: true,
+      aab: "../build/app/outputs/bundle/release/app-release.aab",
+      # this is the default output of flutter build ... --release
+      # in particular this the build folder lies in the flutter root folder
+      # this is the parent folder for the android folder
       )
   end
 
@@ -28,6 +34,7 @@ platform :android do
   lane :deploy_release do
     gradle(
       task: "clean assembleRelease",
+      # todo update to a flutter call
       properties: {
         # loaded from environment
         "android.injected.version.name" => ENV["VERSION_NAME"],
@@ -37,6 +44,10 @@ platform :android do
       track: "production",
       skip_upload_apk: true,
       skip_upload_changelogs: true,
+      aab: "../build/app/outputs/bundle/release/app-release.aab",
+      # this is the default output of flutter build ... --release
+      # in particular this the build folder lies in the flutter root folder
+      # this is the parent folder for the android folder
     )
   end
 end