From fa083a1080277cb68df711fd92cae6cad8eacf40 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 28 Dec 2024 22:25:42 +0100 Subject: [PATCH] logging cleanup --- .gitea/workflows/backend_run_lint.yaml | 2 - .gitea/workflows/backend_run_test.yaml | 1 - .vscode/launch.json | 10 +--- backend/Dockerfile | 4 +- backend/launcher.py | 82 ++++++++++++++++++++++++++ backend/logging_config.yaml | 29 +++++++++ backend/src/constants.py | 40 +------------ backend/src/structs/landmark.py | 2 +- 8 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 backend/launcher.py create mode 100644 backend/logging_config.yaml diff --git a/.gitea/workflows/backend_run_lint.yaml b/.gitea/workflows/backend_run_lint.yaml index e3fd418..5db2228 100644 --- a/.gitea/workflows/backend_run_lint.yaml +++ b/.gitea/workflows/backend_run_lint.yaml @@ -25,8 +25,6 @@ jobs: ls -la # only install dev-packages pipenv install --categories=dev-packages - pipenv run pip freeze - working-directory: backend - name: Run linter diff --git a/.gitea/workflows/backend_run_test.yaml b/.gitea/workflows/backend_run_test.yaml index 36b5ee8..82ad499 100644 --- a/.gitea/workflows/backend_run_test.yaml +++ b/.gitea/workflows/backend_run_test.yaml @@ -25,7 +25,6 @@ jobs: ls -la # install all packages, including dev-packages pipenv install --dev - pipenv run pip freeze working-directory: backend - name: Run Tests diff --git a/.vscode/launch.json b/.vscode/launch.json index 48d1c86..f5cc3b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,21 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - // backend - python using fastapi + // backend - python app that launches a uvicorn server { "name": "Backend - debug", "type": "debugpy", "request": "launch", - "module": "uvicorn", + "program": "launcher.py", "env": { "DEBUG": "true" }, - "args": [ - // "--app-dir", - // "src", - "src.main:app", - "--reload", - ], "jinja": true, "cwd": "${workspaceFolder}/backend" }, diff --git a/backend/Dockerfile b/backend/Dockerfile index 25a5e31..fae8d25 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,6 +7,7 @@ RUN pip install pipenv RUN pipenv install --deploy --system COPY src src +COPY launcher.py . EXPOSE 8000 @@ -14,5 +15,6 @@ EXPOSE 8000 ENV NUM_WORKERS=1 ENV OSM_CACHE_DIR=/cache ENV MEMCACHED_HOST_PATH=none +ENV LOKI_URL=none -CMD fastapi run src/main.py --port 8000 --workers $NUM_WORKERS +CMD ["python", "launcher.py"] diff --git a/backend/launcher.py b/backend/launcher.py new file mode 100644 index 0000000..269433b --- /dev/null +++ b/backend/launcher.py @@ -0,0 +1,82 @@ +"""Launcher for the FastAPI application. Fundametally this replicates the functionality of the uvicorn and fastapi CLI interfaces, but we need this to setup the logging correctly (and most importantly globally)""" + +import os +import uvicorn +import logging + +logger = logging.getLogger(__name__) +is_debug = os.getenv('DEBUG', "false") == "true" +is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST', None) is not None + +def logger_setup(): + """ + Setup the global logging configuration + """ + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + # Make uvicorn conform to the global logging configuration + uvicorn_logger = logging.getLogger('uvicorn') + uvicorn_logger.propagate = True + uvicorn_logger.handlers = [] # Remove default handlers to avoid duplicate logs + + if is_kubernetes: + # in that case we want to log to stdout and also to loki + from loki_logger_handler.loki_logger_handler import LokiLoggerHandler + loki_url = os.getenv('LOKI_URL') + if loki_url is None: + raise ValueError("LOKI_URL environment variable is not set") + + loki_handler = LokiLoggerHandler( + url = loki_url, + labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'} + ) + + logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}") + if is_debug: + logging.basicConfig( + format = logging_format, + level = logging.DEBUG, + handlers = [loki_handler, logging.StreamHandler()] + ) + # we need to silence the debug logs made by the loki handler + logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) + else: + logging.basicConfig( + format = logging_format, + level = logging.INFO, + handlers = [loki_handler, logging.StreamHandler()] + ) + else: + # if we are in a debug (local) session, set verbose and rich logging + from rich.logging import RichHandler + logging.basicConfig( + format = logging_format, + level = logging.DEBUG, + handlers = [RichHandler()] + ) + + +def uvicorn_run(): + """ + Run the FastAPI application using uvicorn + """ + num_workers = os.getenv('NUM_WORKERS', 1) + logger.info(f"Starting FastAPI+uvicorn with {num_workers=}, {is_debug=}, {is_kubernetes=}") + uvicorn.run( + # we could in theory directly import the app and pass it as an object + # this 'import string' is required for hot reloading and scaling + 'src.main:app', + host = '0.0.0.0', + port = 8000, + log_config = None, + access_log = True, + # Disable uvicorn's logging configuration so that it inherits the global one + workers = num_workers, + # Hot reload breaks logging, so we leave it disabled + # reload = is_debug + ) + + +if __name__ == '__main__': + logger_setup() + uvicorn_run() diff --git a/backend/logging_config.yaml b/backend/logging_config.yaml new file mode 100644 index 0000000..d6b7a1d --- /dev/null +++ b/backend/logging_config.yaml @@ -0,0 +1,29 @@ +version: 1 +disable_existing_loggers: False +formatters: + standard: + format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +handlers: + console: + class: logging.StreamHandler + formatter: standard + level: DEBUG + rich: + class: rich.logging.RichHandler + level: DEBUG + loki: + class: loki_logger_handler.loki_logger_handler.LokiLoggerHandler + level: DEBUG + formatter: standard + url: ${LOKI_URL} + labels: + app: anyway + environment: ${ENVIRONMENT} +loggers: + uvicorn: + handlers: [console, rich, loki] + level: DEBUG + propagate: False +root: + handlers: [console, rich, loki] + level: DEBUG \ No newline at end of file diff --git a/backend/src/constants.py b/backend/src/constants.py index 208bfa6..60a26c7 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -1,6 +1,5 @@ -"""Module setting global parameters for the application, such as logging, cache, route generation, etc.""" +"""Module setting global parameters for the application such as cache, route generation, etc.""" -import logging import os from pathlib import Path @@ -16,43 +15,6 @@ cache_dir_string = os.getenv('OSM_CACHE_DIR', './cache') OSM_CACHE_DIR = Path(cache_dir_string) -# if we are in a debug (local) session, set verbose and rich logging -debug = os.getenv('DEBUG', "false") == "true" -if os.getenv('KUBERNETES_SERVICE_HOST', None) is not None: - # in that case we want to log to stdout and also to loki - from loki_logger_handler.loki_logger_handler import LokiLoggerHandler - loki_url = os.getenv('LOKI_URL') - if loki_url is None: - raise ValueError("LOKI_URL environment variable is not set") - - loki_handler = LokiLoggerHandler( - url = loki_url, - labels = {'app': 'anyway', 'environment': 'production' if debug else 'staging'} - ) - print(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {debug=}") - if debug: - # we need to silence the debug logs made by the loki handler - logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) - - logging.basicConfig( - level=logging.DEBUG, - handlers=[loki_handler, logging.StreamHandler()] - ) - else: - logging.basicConfig( - level=logging.INFO, - handlers=[loki_handler, logging.StreamHandler()] - ) -else: - # in that case we are local and we want to log to stdout only, but make it pretty - from rich.logging import RichHandler - logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[RichHandler()] - ) - - MEMCACHED_HOST_PATH = os.getenv('MEMCACHED_HOST_PATH', None) if MEMCACHED_HOST_PATH == "none": MEMCACHED_HOST_PATH = None diff --git a/backend/src/structs/landmark.py b/backend/src/structs/landmark.py index a494896..60ef72a 100644 --- a/backend/src/structs/landmark.py +++ b/backend/src/structs/landmark.py @@ -139,4 +139,4 @@ class Toilets(BaseModel) : class Config: # This allows us to easily convert the model to and from dictionaries - orm_mode = True \ No newline at end of file + from_attributes = True \ No newline at end of file