From d9061388dde99e932bb45443f250bdb1cee70a82 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 28 Dec 2024 13:57:04 +0100 Subject: [PATCH 1/6] use additional loki logger --- backend/Pipfile | 1 + backend/Pipfile.lock | 11 +- backend/deployment | 2 +- backend/report.html | 1094 ---------------------- backend/src/{persistence.py => cache.py} | 0 backend/src/constants.py | 30 +- backend/src/main.py | 2 +- backend/src/tests/test_utils.py | 2 +- 8 files changed, 36 insertions(+), 1106 deletions(-) delete mode 100644 backend/report.html rename backend/src/{persistence.py => cache.py} (100%) diff --git a/backend/Pipfile b/backend/Pipfile index 4d06a94..1f9a632 100644 --- a/backend/Pipfile +++ b/backend/Pipfile @@ -25,3 +25,4 @@ pymemcache = "*" fastapi-cli = "*" scikit-learn = "*" pyqt6 = "*" +loki-logger-handler = "*" diff --git a/backend/Pipfile.lock b/backend/Pipfile.lock index 880b9a6..b13bd13 100644 --- a/backend/Pipfile.lock +++ b/backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bb22b4e28c7aa199c94b688ad93d3ab0ccf1089a172131f4aec03b78e7bd7f1c" + "sha256": "6edd6644586e8814a0b4526adb3352dfc17828ca129de7a68c1d5929efe94daa" }, "pipfile-spec": 6, "requires": {}, @@ -507,6 +507,15 @@ "markers": "python_version >= '3.8'", "version": "==1.4.7" }, + "loki-logger-handler": { + "hashes": [ + "sha256:aa1a9c933282c134a1e4271aba3cbaa2a3660eab6ea415bad7a072444ab98aa8", + "sha256:f6114727a9e5e6f3f2058b9b5324d1cab6d1a04e802079f7b57a8aeb7bd0a112" + ], + "index": "pypi", + "markers": "python_version >= '2.7'", + "version": "==1.0.2" + }, "lxml": { "hashes": [ "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e", diff --git a/backend/deployment b/backend/deployment index 718df09..4f0a028 160000 --- a/backend/deployment +++ b/backend/deployment @@ -1 +1 @@ -Subproject commit 718df09e88b63c9524c882ccbb8247ca1448d3ff +Subproject commit 4f0a0289fcf8a7755d7dc1a76b443fe1233685ae diff --git a/backend/report.html b/backend/report.html deleted file mode 100644 index d46b72d..0000000 --- a/backend/report.html +++ /dev/null @@ -1,1094 +0,0 @@ - - - - - Backend Testing Report - - - - -

Backend Testing Report

-

Report generated on 13-Dec-2024 at 08:58:37 by pytest-html - v4.1.1

-
-

Environment

-
-
- - - - - -
-
-

Summary

-
-
-

12 tests took 865 ms.

-

(Un)check the boxes to filter the results.

-
- -
-
-
-
- - 2 Failed, - - 10 Passed, - - 0 Skipped, - - 0 Expected failures, - - 0 Unexpected passes, - - 0 Errors, - - 0 Reruns -
-
-  /  -
-
-
-
-
-
-
-
- - - - - - - - - - - - -
ResultTestDetailed tripTrip DurationTarget DurationExecution timeLinks
- - - \ No newline at end of file diff --git a/backend/src/persistence.py b/backend/src/cache.py similarity index 100% rename from backend/src/persistence.py rename to backend/src/cache.py diff --git a/backend/src/constants.py b/backend/src/constants.py index 21ed016..303e7ed 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -1,4 +1,4 @@ -"""Module allowing to access the parameters of route generation""" +"""Module setting global parameters for the application, such as logging, cache, route generation, etc.""" import logging import os @@ -16,19 +16,33 @@ cache_dir_string = os.getenv('OSM_CACHE_DIR', './cache') OSM_CACHE_DIR = Path(cache_dir_string) -# if we are in a debug session, set verbose and rich logging -if os.getenv('DEBUG', "false") == "true": +# if we are in a debug (local) session, set verbose and rich logging +debug = os.getenv('DEBUG', "false") == "true" +if os.getenv('KUBERNETES_SERVICE_HOST', None) is not None: + # in that case we want to log to stdout and also to loki + from loki_logger_handler.loki_logger_handler import LokiLoggerHandler + loki_handler = LokiLoggerHandler( + url=os.getenv('LOKI_URL', 'http://loki:3100'), + labels={'app': 'anyway', 'env': 'production' if debug else 'staging'} + ) + if debug: + logging.basicConfig( + level=logging.DEBUG, + handlers=[loki_handler, logging.StreamHandler()] + ) + else: + logging.basicConfig( + level=logging.INFO, + handlers=[loki_handler, logging.StreamHandler()] + ) +else: + # in that case we are local and we want to log to stdout only, but make it pretty from rich.logging import RichHandler logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[RichHandler()] ) -else: - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) MEMCACHED_HOST_PATH = os.getenv('MEMCACHED_HOST_PATH', None) diff --git a/backend/src/main.py b/backend/src/main.py index a2efbf1..eea3bad 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -11,7 +11,7 @@ from .utils.landmarks_manager import LandmarkManager from .utils.toilets_manager import ToiletsManager from .utils.optimizer import Optimizer from .utils.refiner import Refiner -from .persistence import client as cache_client +from .cache import client as cache_client logger = logging.getLogger(__name__) diff --git a/backend/src/tests/test_utils.py b/backend/src/tests/test_utils.py index d5b69ad..73f4ec7 100644 --- a/backend/src/tests/test_utils.py +++ b/backend/src/tests/test_utils.py @@ -4,7 +4,7 @@ from fastapi import HTTPException from pydantic import ValidationError from ..structs.landmark import Landmark -from ..persistence import client as cache_client +from ..cache import client as cache_client def landmarks_to_osmid(landmarks: list[Landmark]) -> list[int] : -- 2.47.2 From c448e2dfb76ca0a42caeb50899b44a4d31a9ce78 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 28 Dec 2024 15:52:29 +0100 Subject: [PATCH 2/6] more verbose logger setup --- .vscode/settings.json | 3 --- backend/deployment | 2 +- backend/src/constants.py | 12 ++++++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 9ddf6b2..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "cmake.ignoreCMakeListsMissing": true -} \ No newline at end of file diff --git a/backend/deployment b/backend/deployment index 4f0a028..904f16b 160000 --- a/backend/deployment +++ b/backend/deployment @@ -1 +1 @@ -Subproject commit 4f0a0289fcf8a7755d7dc1a76b443fe1233685ae +Subproject commit 904f16bfc0624b6ab8569e0a70050aaa3bd64b3f diff --git a/backend/src/constants.py b/backend/src/constants.py index 303e7ed..208bfa6 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -21,11 +21,19 @@ debug = os.getenv('DEBUG', "false") == "true" if os.getenv('KUBERNETES_SERVICE_HOST', None) is not None: # in that case we want to log to stdout and also to loki from loki_logger_handler.loki_logger_handler import LokiLoggerHandler + loki_url = os.getenv('LOKI_URL') + if loki_url is None: + raise ValueError("LOKI_URL environment variable is not set") + loki_handler = LokiLoggerHandler( - url=os.getenv('LOKI_URL', 'http://loki:3100'), - labels={'app': 'anyway', 'env': 'production' if debug else 'staging'} + url = loki_url, + labels = {'app': 'anyway', 'environment': 'production' if debug else 'staging'} ) + print(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {debug=}") if debug: + # we need to silence the debug logs made by the loki handler + logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) + logging.basicConfig( level=logging.DEBUG, handlers=[loki_handler, logging.StreamHandler()] -- 2.47.2 From fa083a1080277cb68df711fd92cae6cad8eacf40 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 28 Dec 2024 22:25:42 +0100 Subject: [PATCH 3/6] logging cleanup --- .gitea/workflows/backend_run_lint.yaml | 2 - .gitea/workflows/backend_run_test.yaml | 1 - .vscode/launch.json | 10 +--- backend/Dockerfile | 4 +- backend/launcher.py | 82 ++++++++++++++++++++++++++ backend/logging_config.yaml | 29 +++++++++ backend/src/constants.py | 40 +------------ backend/src/structs/landmark.py | 2 +- 8 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 backend/launcher.py create mode 100644 backend/logging_config.yaml diff --git a/.gitea/workflows/backend_run_lint.yaml b/.gitea/workflows/backend_run_lint.yaml index e3fd418..5db2228 100644 --- a/.gitea/workflows/backend_run_lint.yaml +++ b/.gitea/workflows/backend_run_lint.yaml @@ -25,8 +25,6 @@ jobs: ls -la # only install dev-packages pipenv install --categories=dev-packages - pipenv run pip freeze - working-directory: backend - name: Run linter diff --git a/.gitea/workflows/backend_run_test.yaml b/.gitea/workflows/backend_run_test.yaml index 36b5ee8..82ad499 100644 --- a/.gitea/workflows/backend_run_test.yaml +++ b/.gitea/workflows/backend_run_test.yaml @@ -25,7 +25,6 @@ jobs: ls -la # install all packages, including dev-packages pipenv install --dev - pipenv run pip freeze working-directory: backend - name: Run Tests diff --git a/.vscode/launch.json b/.vscode/launch.json index 48d1c86..f5cc3b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,21 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - // backend - python using fastapi + // backend - python app that launches a uvicorn server { "name": "Backend - debug", "type": "debugpy", "request": "launch", - "module": "uvicorn", + "program": "launcher.py", "env": { "DEBUG": "true" }, - "args": [ - // "--app-dir", - // "src", - "src.main:app", - "--reload", - ], "jinja": true, "cwd": "${workspaceFolder}/backend" }, diff --git a/backend/Dockerfile b/backend/Dockerfile index 25a5e31..fae8d25 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,6 +7,7 @@ RUN pip install pipenv RUN pipenv install --deploy --system COPY src src +COPY launcher.py . EXPOSE 8000 @@ -14,5 +15,6 @@ EXPOSE 8000 ENV NUM_WORKERS=1 ENV OSM_CACHE_DIR=/cache ENV MEMCACHED_HOST_PATH=none +ENV LOKI_URL=none -CMD fastapi run src/main.py --port 8000 --workers $NUM_WORKERS +CMD ["python", "launcher.py"] diff --git a/backend/launcher.py b/backend/launcher.py new file mode 100644 index 0000000..269433b --- /dev/null +++ b/backend/launcher.py @@ -0,0 +1,82 @@ +"""Launcher for the FastAPI application. Fundametally this replicates the functionality of the uvicorn and fastapi CLI interfaces, but we need this to setup the logging correctly (and most importantly globally)""" + +import os +import uvicorn +import logging + +logger = logging.getLogger(__name__) +is_debug = os.getenv('DEBUG', "false") == "true" +is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST', None) is not None + +def logger_setup(): + """ + Setup the global logging configuration + """ + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + # Make uvicorn conform to the global logging configuration + uvicorn_logger = logging.getLogger('uvicorn') + uvicorn_logger.propagate = True + uvicorn_logger.handlers = [] # Remove default handlers to avoid duplicate logs + + if is_kubernetes: + # in that case we want to log to stdout and also to loki + from loki_logger_handler.loki_logger_handler import LokiLoggerHandler + loki_url = os.getenv('LOKI_URL') + if loki_url is None: + raise ValueError("LOKI_URL environment variable is not set") + + loki_handler = LokiLoggerHandler( + url = loki_url, + labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'} + ) + + logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}") + if is_debug: + logging.basicConfig( + format = logging_format, + level = logging.DEBUG, + handlers = [loki_handler, logging.StreamHandler()] + ) + # we need to silence the debug logs made by the loki handler + logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) + else: + logging.basicConfig( + format = logging_format, + level = logging.INFO, + handlers = [loki_handler, logging.StreamHandler()] + ) + else: + # if we are in a debug (local) session, set verbose and rich logging + from rich.logging import RichHandler + logging.basicConfig( + format = logging_format, + level = logging.DEBUG, + handlers = [RichHandler()] + ) + + +def uvicorn_run(): + """ + Run the FastAPI application using uvicorn + """ + num_workers = os.getenv('NUM_WORKERS', 1) + logger.info(f"Starting FastAPI+uvicorn with {num_workers=}, {is_debug=}, {is_kubernetes=}") + uvicorn.run( + # we could in theory directly import the app and pass it as an object + # this 'import string' is required for hot reloading and scaling + 'src.main:app', + host = '0.0.0.0', + port = 8000, + log_config = None, + access_log = True, + # Disable uvicorn's logging configuration so that it inherits the global one + workers = num_workers, + # Hot reload breaks logging, so we leave it disabled + # reload = is_debug + ) + + +if __name__ == '__main__': + logger_setup() + uvicorn_run() diff --git a/backend/logging_config.yaml b/backend/logging_config.yaml new file mode 100644 index 0000000..d6b7a1d --- /dev/null +++ b/backend/logging_config.yaml @@ -0,0 +1,29 @@ +version: 1 +disable_existing_loggers: False +formatters: + standard: + format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +handlers: + console: + class: logging.StreamHandler + formatter: standard + level: DEBUG + rich: + class: rich.logging.RichHandler + level: DEBUG + loki: + class: loki_logger_handler.loki_logger_handler.LokiLoggerHandler + level: DEBUG + formatter: standard + url: ${LOKI_URL} + labels: + app: anyway + environment: ${ENVIRONMENT} +loggers: + uvicorn: + handlers: [console, rich, loki] + level: DEBUG + propagate: False +root: + handlers: [console, rich, loki] + level: DEBUG \ No newline at end of file diff --git a/backend/src/constants.py b/backend/src/constants.py index 208bfa6..60a26c7 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -1,6 +1,5 @@ -"""Module setting global parameters for the application, such as logging, cache, route generation, etc.""" +"""Module setting global parameters for the application such as cache, route generation, etc.""" -import logging import os from pathlib import Path @@ -16,43 +15,6 @@ cache_dir_string = os.getenv('OSM_CACHE_DIR', './cache') OSM_CACHE_DIR = Path(cache_dir_string) -# if we are in a debug (local) session, set verbose and rich logging -debug = os.getenv('DEBUG', "false") == "true" -if os.getenv('KUBERNETES_SERVICE_HOST', None) is not None: - # in that case we want to log to stdout and also to loki - from loki_logger_handler.loki_logger_handler import LokiLoggerHandler - loki_url = os.getenv('LOKI_URL') - if loki_url is None: - raise ValueError("LOKI_URL environment variable is not set") - - loki_handler = LokiLoggerHandler( - url = loki_url, - labels = {'app': 'anyway', 'environment': 'production' if debug else 'staging'} - ) - print(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {debug=}") - if debug: - # we need to silence the debug logs made by the loki handler - logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) - - logging.basicConfig( - level=logging.DEBUG, - handlers=[loki_handler, logging.StreamHandler()] - ) - else: - logging.basicConfig( - level=logging.INFO, - handlers=[loki_handler, logging.StreamHandler()] - ) -else: - # in that case we are local and we want to log to stdout only, but make it pretty - from rich.logging import RichHandler - logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[RichHandler()] - ) - - MEMCACHED_HOST_PATH = os.getenv('MEMCACHED_HOST_PATH', None) if MEMCACHED_HOST_PATH == "none": MEMCACHED_HOST_PATH = None diff --git a/backend/src/structs/landmark.py b/backend/src/structs/landmark.py index a494896..60ef72a 100644 --- a/backend/src/structs/landmark.py +++ b/backend/src/structs/landmark.py @@ -139,4 +139,4 @@ class Toilets(BaseModel) : class Config: # This allows us to easily convert the model to and from dictionaries - orm_mode = True \ No newline at end of file + from_attributes = True \ No newline at end of file -- 2.47.2 From bc63b57154bb3cf002d8848f09254ff888a4ea0b Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 28 Dec 2024 22:34:14 +0100 Subject: [PATCH 4/6] dumb type conversion --- backend/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/launcher.py b/backend/launcher.py index 269433b..5de65ca 100644 --- a/backend/launcher.py +++ b/backend/launcher.py @@ -60,7 +60,7 @@ def uvicorn_run(): """ Run the FastAPI application using uvicorn """ - num_workers = os.getenv('NUM_WORKERS', 1) + num_workers = int(os.getenv('NUM_WORKERS', 1)) logger.info(f"Starting FastAPI+uvicorn with {num_workers=}, {is_debug=}, {is_kubernetes=}") uvicorn.run( # we could in theory directly import the app and pass it as an object -- 2.47.2 From 4e07c10969565c816a1ecd55f12126aabf35255f Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sun, 29 Dec 2024 14:45:41 +0100 Subject: [PATCH 5/6] actually use fastapi lifetime manager to setup logging --- .vscode/launch.json | 10 ++-- backend/Dockerfile | 3 +- backend/launcher.py | 82 --------------------------------- backend/logging_config.yaml | 29 ------------ backend/src/logging_config.py | 56 ++++++++++++++++++++++ backend/src/main.py | 17 ++++++- backend/src/structs/landmark.py | 10 ++-- backend/src/structs/trip.py | 8 ++-- 8 files changed, 88 insertions(+), 127 deletions(-) delete mode 100644 backend/launcher.py delete mode 100644 backend/logging_config.yaml create mode 100644 backend/src/logging_config.py diff --git a/.vscode/launch.json b/.vscode/launch.json index f5cc3b6..6f1ce86 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,17 +4,21 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - // backend - python app that launches a uvicorn server + // backend - python using fastapi { "name": "Backend - debug", "type": "debugpy", "request": "launch", - "program": "launcher.py", "env": { "DEBUG": "true" }, "jinja": true, - "cwd": "${workspaceFolder}/backend" + "cwd": "${workspaceFolder}/backend", + "module": "fastapi", + "args": [ + "dev", + "src/main.py" + ] }, { "name": "Backend - tester", diff --git a/backend/Dockerfile b/backend/Dockerfile index fae8d25..0f1f475 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,7 +7,6 @@ RUN pip install pipenv RUN pipenv install --deploy --system COPY src src -COPY launcher.py . EXPOSE 8000 @@ -17,4 +16,4 @@ ENV OSM_CACHE_DIR=/cache ENV MEMCACHED_HOST_PATH=none ENV LOKI_URL=none -CMD ["python", "launcher.py"] +CMD ["fastapi", "src/main.py", "--port", "8000", "--workers", "$NUM_WORKERS"] diff --git a/backend/launcher.py b/backend/launcher.py deleted file mode 100644 index 5de65ca..0000000 --- a/backend/launcher.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Launcher for the FastAPI application. Fundametally this replicates the functionality of the uvicorn and fastapi CLI interfaces, but we need this to setup the logging correctly (and most importantly globally)""" - -import os -import uvicorn -import logging - -logger = logging.getLogger(__name__) -is_debug = os.getenv('DEBUG', "false") == "true" -is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST', None) is not None - -def logger_setup(): - """ - Setup the global logging configuration - """ - logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - - # Make uvicorn conform to the global logging configuration - uvicorn_logger = logging.getLogger('uvicorn') - uvicorn_logger.propagate = True - uvicorn_logger.handlers = [] # Remove default handlers to avoid duplicate logs - - if is_kubernetes: - # in that case we want to log to stdout and also to loki - from loki_logger_handler.loki_logger_handler import LokiLoggerHandler - loki_url = os.getenv('LOKI_URL') - if loki_url is None: - raise ValueError("LOKI_URL environment variable is not set") - - loki_handler = LokiLoggerHandler( - url = loki_url, - labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'} - ) - - logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}") - if is_debug: - logging.basicConfig( - format = logging_format, - level = logging.DEBUG, - handlers = [loki_handler, logging.StreamHandler()] - ) - # we need to silence the debug logs made by the loki handler - logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) - else: - logging.basicConfig( - format = logging_format, - level = logging.INFO, - handlers = [loki_handler, logging.StreamHandler()] - ) - else: - # if we are in a debug (local) session, set verbose and rich logging - from rich.logging import RichHandler - logging.basicConfig( - format = logging_format, - level = logging.DEBUG, - handlers = [RichHandler()] - ) - - -def uvicorn_run(): - """ - Run the FastAPI application using uvicorn - """ - num_workers = int(os.getenv('NUM_WORKERS', 1)) - logger.info(f"Starting FastAPI+uvicorn with {num_workers=}, {is_debug=}, {is_kubernetes=}") - uvicorn.run( - # we could in theory directly import the app and pass it as an object - # this 'import string' is required for hot reloading and scaling - 'src.main:app', - host = '0.0.0.0', - port = 8000, - log_config = None, - access_log = True, - # Disable uvicorn's logging configuration so that it inherits the global one - workers = num_workers, - # Hot reload breaks logging, so we leave it disabled - # reload = is_debug - ) - - -if __name__ == '__main__': - logger_setup() - uvicorn_run() diff --git a/backend/logging_config.yaml b/backend/logging_config.yaml deleted file mode 100644 index d6b7a1d..0000000 --- a/backend/logging_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: 1 -disable_existing_loggers: False -formatters: - standard: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -handlers: - console: - class: logging.StreamHandler - formatter: standard - level: DEBUG - rich: - class: rich.logging.RichHandler - level: DEBUG - loki: - class: loki_logger_handler.loki_logger_handler.LokiLoggerHandler - level: DEBUG - formatter: standard - url: ${LOKI_URL} - labels: - app: anyway - environment: ${ENVIRONMENT} -loggers: - uvicorn: - handlers: [console, rich, loki] - level: DEBUG - propagate: False -root: - handlers: [console, rich, loki] - level: DEBUG \ No newline at end of file diff --git a/backend/src/logging_config.py b/backend/src/logging_config.py new file mode 100644 index 0000000..0f93fa7 --- /dev/null +++ b/backend/src/logging_config.py @@ -0,0 +1,56 @@ +"""Sets up global logging configuration for the application.""" + +import logging +import os + +logger = logging.getLogger(__name__) + + +def configure_logging(): + """ + Called at startup of a FastAPI application instance to setup logging. Depending on the environment, it will log to stdout or to Loki. + """ + + is_debug = os.getenv('DEBUG', "false") == "true" + is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST') is not None + + + if not is_kubernetes: + # in that case we want to log to stdout and also to loki + from loki_logger_handler.loki_logger_handler import LokiLoggerHandler + loki_url = os.getenv('LOKI_URL') + loki_url = "http://localhost:3100/loki/api/v1/push" + if loki_url is None: + raise ValueError("LOKI_URL environment variable is not set") + + loki_handler = LokiLoggerHandler( + url = loki_url, + labels = {'app': 'anyway', 'environment': 'staging' if is_debug else 'production'} + ) + + logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}") + logging_handlers = [loki_handler, logging.StreamHandler()] + logging_level = logging.DEBUG if is_debug else logging.INFO + # no need for time since it's added by loki or can be shown in kube logs + logging_format = '%(name)s - %(levelname)s - %(message)s' + + else: + # if we are in a debug (local) session, set verbose and rich logging + from rich.logging import RichHandler + logging_handlers = [RichHandler()] + logging_level = logging.DEBUG if is_debug else logging.INFO + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + + + logging.basicConfig( + level = logging_level, + format = logging_format, + handlers = logging_handlers + ) + + # also overwrite the uvicorn loggers + logging.getLogger('uvicorn').handlers = logging_handlers + logging.getLogger('uvicorn.access').handlers = logging_handlers + logging.getLogger('uvicorn.error').handlers = logging_handlers + diff --git a/backend/src/main.py b/backend/src/main.py index eea3bad..c8c1f40 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -2,7 +2,9 @@ import logging from fastapi import FastAPI, HTTPException, Query +from contextlib import asynccontextmanager +from .logging_config import configure_logging from .structs.landmark import Landmark, Toilets from .structs.preferences import Preferences from .structs.linked_landmarks import LinkedLandmarks @@ -13,15 +15,26 @@ from .utils.optimizer import Optimizer from .utils.refiner import Refiner from .cache import client as cache_client - logger = logging.getLogger(__name__) -app = FastAPI() manager = LandmarkManager() optimizer = Optimizer() refiner = Refiner(optimizer=optimizer) +@asynccontextmanager +async def lifespan(app: FastAPI): + """Function to run at the start of the app""" + logger.info("Setting up logging") + configure_logging() + yield + logger.info("Shutting down logging") + + +app = FastAPI(lifespan=lifespan) + + + @app.post("/trip/new") def new_trip(preferences: Preferences, start: tuple[float, float], diff --git a/backend/src/structs/landmark.py b/backend/src/structs/landmark.py index 60ef72a..da0f122 100644 --- a/backend/src/structs/landmark.py +++ b/backend/src/structs/landmark.py @@ -1,7 +1,7 @@ """Definition of the Landmark class to handle visitable objects across the world.""" from typing import Optional, Literal -from uuid import uuid4 +from uuid import uuid4, UUID from pydantic import BaseModel, Field @@ -29,12 +29,12 @@ class Landmark(BaseModel) : description (Optional[str]): A text description of the landmark. duration (Optional[int]): The estimated time to visit the landmark (in minutes). name_en (Optional[str]): The English name of the landmark. - uuid (str): A unique identifier for the landmark, generated by default using uuid4. + uuid (UUID): A unique identifier for the landmark, generated by default using uuid4. must_do (Optional[bool]): Whether the landmark is a "must-do" attraction. must_avoid (Optional[bool]): Whether the landmark should be avoided. is_secondary (Optional[bool]): Whether the landmark is secondary or less important. time_to_reach_next (Optional[int]): Estimated time (in minutes) to reach the next landmark. - next_uuid (Optional[str]): UUID of the next landmark in sequence (if applicable). + next_uuid (Optional[UUID]): UUID of the next landmark in sequence (if applicable). """ # Properties of the landmark @@ -52,7 +52,7 @@ class Landmark(BaseModel) : name_en : Optional[str] = None # Unique ID of a given landmark - uuid: str = Field(default_factory=uuid4) + uuid: UUID = Field(default_factory=uuid4) # Additional properties depending on specific tour must_do : Optional[bool] = False @@ -60,7 +60,7 @@ class Landmark(BaseModel) : is_secondary : Optional[bool] = False time_to_reach_next : Optional[int] = 0 - next_uuid : Optional[str] = None + next_uuid : Optional[UUID] = None def __str__(self) -> str: """ diff --git a/backend/src/structs/trip.py b/backend/src/structs/trip.py index c00f591..d9b19d7 100644 --- a/backend/src/structs/trip.py +++ b/backend/src/structs/trip.py @@ -1,6 +1,6 @@ """Definition of the Trip class.""" -import uuid +from uuid import uuid4, UUID from pydantic import BaseModel, Field from pymemcache.client.base import Client @@ -19,9 +19,9 @@ class Trip(BaseModel): Methods: from_linked_landmarks: create a Trip from LinkedLandmarks object. """ - uuid: str = Field(default_factory=uuid.uuid4) + uuid: UUID = Field(default_factory=uuid4) total_time: int - first_landmark_uuid: str + first_landmark_uuid: UUID @classmethod @@ -31,7 +31,7 @@ class Trip(BaseModel): """ trip = Trip( total_time = landmarks.total_time, - first_landmark_uuid = str(landmarks[0].uuid) + first_landmark_uuid = landmarks[0].uuid ) # Store the trip in the cache -- 2.47.2 From 86187d906976767e834eba4e4bab6172713ff7a8 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sun, 29 Dec 2024 14:51:28 +0100 Subject: [PATCH 6/6] launch adjustments --- backend/Dockerfile | 3 ++- backend/src/logging_config.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 0f1f475..b363917 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -16,4 +16,5 @@ ENV OSM_CACHE_DIR=/cache ENV MEMCACHED_HOST_PATH=none ENV LOKI_URL=none -CMD ["fastapi", "src/main.py", "--port", "8000", "--workers", "$NUM_WORKERS"] +# explicitly use a string instead of an argument list to force a shell and variable expansion +CMD fastapi run src/main.py --port 8000 --workers $NUM_WORKERS diff --git a/backend/src/logging_config.py b/backend/src/logging_config.py index 0f93fa7..c43a246 100644 --- a/backend/src/logging_config.py +++ b/backend/src/logging_config.py @@ -15,7 +15,7 @@ def configure_logging(): is_kubernetes = os.getenv('KUBERNETES_SERVICE_HOST') is not None - if not is_kubernetes: + if is_kubernetes: # in that case we want to log to stdout and also to loki from loki_logger_handler.loki_logger_handler import LokiLoggerHandler loki_url = os.getenv('LOKI_URL') @@ -31,6 +31,8 @@ def configure_logging(): logger.info(f"Logging to Loki at {loki_url} with {loki_handler.labels} and {is_debug=}") logging_handlers = [loki_handler, logging.StreamHandler()] logging_level = logging.DEBUG if is_debug else logging.INFO + # silence the chatty logs loki generates itself + logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) # no need for time since it's added by loki or can be shown in kube logs logging_format = '%(name)s - %(levelname)s - %(message)s' -- 2.47.2