logging cleanup

.gitea/workflows/backend_run_lint.yaml

@@ -25,8 +25,6 @@ jobs:
         ls -la
         # only install dev-packages
         pipenv install --categories=dev-packages
-        pipenv run pip freeze        
-
       working-directory: backend
 
     - name: Run linter

.gitea/workflows/backend_run_test.yaml

@@ -25,7 +25,6 @@ jobs:
         ls -la
         # install all packages, including dev-packages
         pipenv install --dev
-        pipenv run pip freeze        
       working-directory: backend
 
     - name: Run Tests

.vscode/launch.json

@@ -4,21 +4,15 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        // backend - python using fastapi
+        // backend - python app that launches a uvicorn server
         {
             "name": "Backend - debug",
             "type": "debugpy",
             "request": "launch",
-            "module": "uvicorn",
+            "program": "launcher.py",
             "env": {
                 "DEBUG": "true"
             },
-            "args": [
-                // "--app-dir",
-                // "src",
-                "src.main:app",
-                "--reload",
-            ],
             "jinja": true,
             "cwd": "${workspaceFolder}/backend"
         },

backend/Dockerfile

@@ -7,6 +7,7 @@ RUN pip install pipenv
 RUN pipenv install --deploy --system
 
 COPY src src
+COPY launcher.py .
 
 EXPOSE 8000
 
@@ -14,5 +15,6 @@ EXPOSE 8000
 ENV NUM_WORKERS=1
 ENV OSM_CACHE_DIR=/cache
 ENV MEMCACHED_HOST_PATH=none
+ENV LOKI_URL=none
 
-CMD fastapi run src/main.py --port 8000 --workers $NUM_WORKERS
+CMD ["python", "launcher.py"]

backend/launcher.py

@@ -0,0 +1,82 @@
+"""Launcher for the FastAPI application. Fundametally this replicates the functionality of the uvicorn and fastapi CLI interfaces, but we need this to setup the logging correctly (and most importantly globally)"""
+
+import os
+import uvicorn
+import logging
+
+logger = logging.getLogger(__name__)
+is_debug = os.getenv('DEBUG', "false") == "true"
+is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST', None) is not None
+
+def logger_setup():
+    """
+    Setup the global logging configuration
+    """
+    logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+    # Make uvicorn conform to the global logging configuration
+    uvicorn_logger = logging.getLogger('uvicorn')
+    uvicorn_logger.propagate = True
+    uvicorn_logger.handlers = []  # Remove default handlers to avoid duplicate logs
+
+    if is_kubernetes:
+        # in that case we want to log to stdout and also to loki
+        from loki_logger_handler.loki_logger_handler import LokiLoggerHandler
+        loki_url = os.getenv('LOKI_URL')
+        if loki_url is None:
+            raise ValueError("LOKI_URL environment variable is not set")
+        
+        loki_handler = LokiLoggerHandler(
+            url = loki_url,
+            labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'}
+        )
+
+        logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}")
+        if is_debug:
+            logging.basicConfig(
+                format = logging_format,
+                level = logging.DEBUG,
+                handlers = [loki_handler, logging.StreamHandler()]
+            )
+            # we need to silence the debug logs made by the loki handler
+            logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO)
+        else:
+            logging.basicConfig(
+                format = logging_format,
+                level = logging.INFO,
+                handlers = [loki_handler, logging.StreamHandler()]
+            )
+    else:
+        # if we are in a debug (local) session, set verbose and rich logging
+        from rich.logging import RichHandler
+        logging.basicConfig(
+            format = logging_format,
+            level = logging.DEBUG,
+            handlers = [RichHandler()]
+        )
+
+
+def uvicorn_run():
+    """
+    Run the FastAPI application using uvicorn
+    """
+    num_workers = os.getenv('NUM_WORKERS', 1)
+    logger.info(f"Starting FastAPI+uvicorn with {num_workers=}, {is_debug=}, {is_kubernetes=}")
+    uvicorn.run(
+        # we could in theory directly import the app and pass it as an object
+        # this 'import string' is required for hot reloading and scaling
+        'src.main:app',
+        host = '0.0.0.0',
+        port = 8000,
+        log_config = None,
+        access_log = True,
+        # Disable uvicorn's logging configuration so that it inherits the global one
+        workers = num_workers,
+        # Hot reload breaks logging, so we leave it disabled
+        # reload = is_debug
+    )
+    
+
+if __name__ == '__main__':
+    logger_setup()
+    uvicorn_run()

backend/logging_config.yaml

@@ -0,0 +1,29 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+    standard:
+        format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+handlers:
+    console:
+        class: logging.StreamHandler
+        formatter: standard
+        level: DEBUG
+    rich:
+        class: rich.logging.RichHandler
+        level: DEBUG
+    loki:
+        class: loki_logger_handler.loki_logger_handler.LokiLoggerHandler
+        level: DEBUG
+        formatter: standard
+        url: ${LOKI_URL}
+        labels:
+            app: anyway
+            environment: ${ENVIRONMENT}
+loggers:
+    uvicorn:
+        handlers: [console, rich, loki]
+        level: DEBUG
+        propagate: False
+root:
+    handlers: [console, rich, loki]
+    level: DEBUG

backend/src/constants.py

@@ -1,6 +1,5 @@
-"""Module setting global parameters for the application, such as logging, cache, route generation, etc."""
+"""Module setting global parameters for the application such as cache, route generation, etc."""
 
-import logging
 import os
 from pathlib import Path
 
@@ -16,43 +15,6 @@ cache_dir_string = os.getenv('OSM_CACHE_DIR', './cache')
 OSM_CACHE_DIR = Path(cache_dir_string)
 
 
-# if we are in a debug (local) session, set verbose and rich logging
-debug = os.getenv('DEBUG', "false") == "true"
-if os.getenv('KUBERNETES_SERVICE_HOST', None) is not None:
-    # in that case we want to log to stdout and also to loki
-    from loki_logger_handler.loki_logger_handler import LokiLoggerHandler
-    loki_url = os.getenv('LOKI_URL')
-    if loki_url is None:
-        raise ValueError("LOKI_URL environment variable is not set")
-    
-    loki_handler = LokiLoggerHandler(
-        url = loki_url,
-        labels = {'app': 'anyway', 'environment': 'production' if debug else 'staging'}
-    )
-    print(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {debug=}")
-    if debug:
-        # we need to silence the debug logs made by the loki handler
-        logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO)
-
-        logging.basicConfig(
-            level=logging.DEBUG,
-            handlers=[loki_handler, logging.StreamHandler()]
-        )
-    else:
-        logging.basicConfig(
-            level=logging.INFO,
-            handlers=[loki_handler, logging.StreamHandler()]
-        )
-else:
-    # in that case we are local and we want to log to stdout only, but make it pretty
-    from rich.logging import RichHandler
-    logging.basicConfig(
-        level=logging.DEBUG,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        handlers=[RichHandler()]
-    )
-
-
 MEMCACHED_HOST_PATH = os.getenv('MEMCACHED_HOST_PATH', None)
 if MEMCACHED_HOST_PATH == "none":
     MEMCACHED_HOST_PATH = None

backend/src/structs/landmark.py

@@ -139,4 +139,4 @@ class Toilets(BaseModel) :
     
     class Config:
         # This allows us to easily convert the model to and from dictionaries
-        orm_mode = True
+        from_attributes = True