Merge pull request 'use additional loki logger' (#48) from feature/backend/centralized-logging into main

Reviewed-on: #48

.gitea/workflows/backend_run_lint.yaml

@@ -25,8 +25,6 @@ jobs:
         ls -la
         # only install dev-packages
         pipenv install --categories=dev-packages
-        pipenv run pip freeze        
-
       working-directory: backend
 
     - name: Run linter

.gitea/workflows/backend_run_test.yaml

@@ -25,7 +25,6 @@ jobs:
         ls -la
         # install all packages, including dev-packages
         pipenv install --dev
-        pipenv run pip freeze        
       working-directory: backend
 
     - name: Run Tests

.vscode/launch.json

@@ -9,18 +9,16 @@
             "name": "Backend - debug",
             "type": "debugpy",
             "request": "launch",
-            "module": "uvicorn",
             "env": {
                 "DEBUG": "true"
             },
-            "args": [
-                // "--app-dir",
-                // "src",
-                "src.main:app",
-                "--reload",
-            ],
             "jinja": true,
-            "cwd": "${workspaceFolder}/backend"
+            "cwd": "${workspaceFolder}/backend",
+            "module": "fastapi",
+            "args": [
+                "dev",
+                "src/main.py"
+            ]
         },
         {
             "name": "Backend - tester",

.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "cmake.ignoreCMakeListsMissing": true
-}

backend/Dockerfile

@@ -14,5 +14,7 @@ EXPOSE 8000
 ENV NUM_WORKERS=1
 ENV OSM_CACHE_DIR=/cache
 ENV MEMCACHED_HOST_PATH=none
+ENV LOKI_URL=none
 
+# explicitly use a string instead of an argument list to force a shell and variable expansion
 CMD fastapi run src/main.py --port 8000 --workers $NUM_WORKERS

backend/Pipfile

@@ -25,3 +25,4 @@ pymemcache = "*"
 fastapi-cli = "*"
 scikit-learn = "*"
 pyqt6 = "*"
+loki-logger-handler = "*"

backend/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "bb22b4e28c7aa199c94b688ad93d3ab0ccf1089a172131f4aec03b78e7bd7f1c"
+            "sha256": "6edd6644586e8814a0b4526adb3352dfc17828ca129de7a68c1d5929efe94daa"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -507,6 +507,15 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.4.7"
         },
+        "loki-logger-handler": {
+            "hashes": [
+                "sha256:aa1a9c933282c134a1e4271aba3cbaa2a3660eab6ea415bad7a072444ab98aa8",
+                "sha256:f6114727a9e5e6f3f2058b9b5324d1cab6d1a04e802079f7b57a8aeb7bd0a112"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '2.7'",
+            "version": "==1.0.2"
+        },
         "lxml": {
             "hashes": [
                 "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e",

backend/deployment

@@ -1 +1 @@
-Subproject commit 718df09e88b63c9524c882ccbb8247ca1448d3ff
+Subproject commit 904f16bfc0624b6ab8569e0a70050aaa3bd64b3f

backend/src/constants.py

@@ -1,6 +1,5 @@
-"""Module allowing to access the parameters of route generation"""
+"""Module setting global parameters for the application such as cache, route generation, etc."""
 
-import logging
 import os
 from pathlib import Path
 
@@ -16,21 +15,6 @@ cache_dir_string = os.getenv('OSM_CACHE_DIR', './cache')
 OSM_CACHE_DIR = Path(cache_dir_string)
 
 
-# if we are in a debug session, set verbose and rich logging
-if os.getenv('DEBUG', "false") == "true":
-    from rich.logging import RichHandler
-    logging.basicConfig(
-        level=logging.DEBUG,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        handlers=[RichHandler()]
-    )
-else:
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-    )
-
-
 MEMCACHED_HOST_PATH = os.getenv('MEMCACHED_HOST_PATH', None)
 if MEMCACHED_HOST_PATH == "none":
     MEMCACHED_HOST_PATH = None

backend/src/logging_config.py

@@ -0,0 +1,58 @@
+"""Sets up global logging configuration for the application."""
+
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def configure_logging():
+    """
+    Called at startup of a FastAPI application instance to setup logging. Depending on the environment, it will log to stdout or to Loki.
+    """
+
+    is_debug = os.getenv('DEBUG', "false") == "true"
+    is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST') is not None
+
+
+    if is_kubernetes:
+        # in that case we want to log to stdout and also to loki
+        from loki_logger_handler.loki_logger_handler import LokiLoggerHandler
+        loki_url = os.getenv('LOKI_URL')
+        loki_url = "http://localhost:3100/loki/api/v1/push"
+        if loki_url is None:
+            raise ValueError("LOKI_URL environment variable is not set")
+        
+        loki_handler = LokiLoggerHandler(
+            url = loki_url,
+            labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'}
+        )
+
+        logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}")
+        logging_handlers = [loki_handler, logging.StreamHandler()]
+        logging_level = logging.DEBUG if is_debug else logging.INFO
+        # silence the chatty logs loki generates itself
+        logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
+        # no need for time since it's added by loki or can be shown in kube logs
+        logging_format = '%(name)s - %(levelname)s - %(message)s'
+
+    else:
+        # if we are in a debug (local) session, set verbose and rich logging
+        from rich.logging import RichHandler
+        logging_handlers = [RichHandler()]
+        logging_level = logging.DEBUG if is_debug else logging.INFO
+        logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+
+
+    logging.basicConfig(
+        level = logging_level,
+        format = logging_format,
+        handlers = logging_handlers
+    )
+
+    # also overwrite the uvicorn loggers
+    logging.getLogger('uvicorn').handlers = logging_handlers
+    logging.getLogger('uvicorn.access').handlers = logging_handlers
+    logging.getLogger('uvicorn.error').handlers = logging_handlers
+

backend/src/main.py

@@ -2,7 +2,9 @@
 
 import logging
 from fastapi import FastAPI, HTTPException, Query
+from contextlib import asynccontextmanager
 
+from .logging_config import configure_logging
 from .structs.landmark import Landmark, Toilets
 from .structs.preferences import Preferences
 from .structs.linked_landmarks import LinkedLandmarks
@@ -11,17 +13,28 @@ from .utils.landmarks_manager import LandmarkManager
 from .utils.toilets_manager import ToiletsManager
 from .utils.optimizer import Optimizer
 from .utils.refiner import Refiner
-from .persistence import client as cache_client
-
+from .cache import client as cache_client
 
 logger = logging.getLogger(__name__)
 
-app = FastAPI()
 manager = LandmarkManager()
 optimizer = Optimizer()
 refiner = Refiner(optimizer=optimizer)
 
 
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Function to run at the start of the app"""
+    logger.info("Setting up logging")
+    configure_logging()
+    yield
+    logger.info("Shutting down logging")
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+
 @app.post("/trip/new")
 def new_trip(preferences: Preferences,
              start: tuple[float, float],

backend/src/structs/landmark.py

@@ -1,7 +1,7 @@
 """Definition of the Landmark class to handle visitable objects across the world."""
 
 from typing import Optional, Literal
-from uuid import uuid4
+from uuid import uuid4, UUID
 from pydantic import BaseModel, Field
 
 
@@ -29,12 +29,12 @@ class Landmark(BaseModel) :
         description (Optional[str]): A text description of the landmark.
         duration (Optional[int]): The estimated time to visit the landmark (in minutes).
         name_en (Optional[str]): The English name of the landmark.
-        uuid (str): A unique identifier for the landmark, generated by default using uuid4.
+        uuid (UUID): A unique identifier for the landmark, generated by default using uuid4.
         must_do (Optional[bool]): Whether the landmark is a "must-do" attraction.
         must_avoid (Optional[bool]): Whether the landmark should be avoided.
         is_secondary (Optional[bool]): Whether the landmark is secondary or less important.
         time_to_reach_next (Optional[int]): Estimated time (in minutes) to reach the next landmark.
-        next_uuid (Optional[str]): UUID of the next landmark in sequence (if applicable).
+        next_uuid (Optional[UUID]): UUID of the next landmark in sequence (if applicable).
     """
 
     # Properties of the landmark
@@ -52,7 +52,7 @@ class Landmark(BaseModel) :
     name_en : Optional[str] = None
 
     # Unique ID of a given landmark
-    uuid: str = Field(default_factory=uuid4)
+    uuid: UUID = Field(default_factory=uuid4)
 
     # Additional properties depending on specific tour
     must_do : Optional[bool] = False
@@ -60,7 +60,7 @@ class Landmark(BaseModel) :
     is_secondary : Optional[bool] = False
 
     time_to_reach_next : Optional[int] = 0
-    next_uuid : Optional[str] = None
+    next_uuid : Optional[UUID] = None
 
     def __str__(self) -> str:
         """
@@ -139,4 +139,4 @@ class Toilets(BaseModel) :
     
     class Config:
         # This allows us to easily convert the model to and from dictionaries
-        orm_mode = True
+        from_attributes = True

backend/src/structs/trip.py

@@ -1,6 +1,6 @@
 """Definition of the Trip class."""
 
-import uuid
+from uuid import uuid4, UUID
 from pydantic import BaseModel, Field
 from pymemcache.client.base import Client
 
@@ -19,9 +19,9 @@ class Trip(BaseModel):
     Methods:
         from_linked_landmarks: create a Trip from LinkedLandmarks object.
     """
-    uuid: str = Field(default_factory=uuid.uuid4)
+    uuid: UUID = Field(default_factory=uuid4)
     total_time: int
-    first_landmark_uuid: str
+    first_landmark_uuid: UUID
 
 
     @classmethod
@@ -31,7 +31,7 @@ class Trip(BaseModel):
         """
         trip = Trip(
             total_time = landmarks.total_time,
-            first_landmark_uuid = str(landmarks[0].uuid)
+            first_landmark_uuid = landmarks[0].uuid
         )
 
         # Store the trip in the cache

backend/src/tests/test_utils.py

@@ -4,7 +4,7 @@ from fastapi import HTTPException
 from pydantic import ValidationError
 
 from ..structs.landmark import Landmark
-from ..persistence import client as cache_client
+from ..cache import client as cache_client
 
 
 def landmarks_to_osmid(landmarks: list[Landmark]) -> list[int] :