Merge pull request 'added warning logs' (#2) from anyway/better-dashboard into main

Reviewed-on: #2
Reviewed-by: Remy Moll <me@moll.re>

dashboards/anyway_logs.json

@@ -27,9 +27,46 @@
         "type": "loki",
         "uid": "P8E80F9AEF21F6940"
       },
-      "description": "Total  Count of log lines in the specified time range",
+      "description": "Trips generated over the period of time",
       "fieldConfig": {
         "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "Trips",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "bars",
+            "fillOpacity": 100,
+            "gradientMode": "hue",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "stepBefore",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
           "mappings": [
             {
               "options": {
@@ -48,14 +85,6 @@
               {
                 "color": "rgb(31, 255, 7)",
                 "value": null
-              },
-              {
-                "color": "rgb(31, 255, 7)",
-                "value": 10
-              },
-              {
-                "color": "rgb(31, 255, 7)",
-                "value": 50
               }
             ]
           },
@@ -64,29 +93,28 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 3,
+        "h": 6,
-        "w": 12,
+        "w": 14,
         "x": 0,
         "y": 0
       },
       "id": 11,
       "maxDataPoints": 100,
       "options": {
-        "colorMode": "value",
+        "legend": {
-        "graphMode": "area",
+          "calcs": [],
-        "justifyMode": "auto",
+          "displayMode": "list",
-        "orientation": "horizontal",
+          "placement": "bottom",
-        "percentChangeColorMode": "standard",
+          "showLegend": false
-        "reduceOptions": {
-          "calcs": [
-            "sum"
-          ],
-          "fields": "",
-          "values": false
         },
-        "showPercentChange": false,
+        "timezone": [
-        "textMode": "auto",
+          "Europe/Zurich"
-        "wideLayout": true
+        ],
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "11.5.1",
       "targets": [
@@ -95,24 +123,42 @@
             "type": "loki",
             "uid": "P8E80F9AEF21F6940"
           },
-          "editorMode": "code",
+          "direction": "backward",
-          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"})[$__interval]))",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} |= `Generated a trip` [$__interval]))",
           "hide": false,
+          "legendFormat": "trips generated",
           "queryType": "range",
-          "refId": "A"
+          "refId": "generated trips"
-        }
-      ],
-      "title": "Total  Count of logs",
-      "type": "stat"
         },
         {
           "datasource": {
             "type": "loki",
             "uid": "P8E80F9AEF21F6940"
           },
-      "description": "Total Count: of $searchable_pattern in the specified time range",
+          "direction": "backward",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} |= `Trip generation failed` or `Exception in ASGI` [$__interval]))",
+          "hide": false,
+          "legendFormat": "trips failed",
+          "queryType": "range",
+          "refId": "failed trips"
+        }
+      ],
+      "title": "Trips generated",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Successfully generated trips",
       "fieldConfig": {
         "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
           "mappings": [
             {
               "options": {
@@ -130,10 +176,6 @@
               {
                 "color": "green",
                 "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
               }
             ]
           },
@@ -143,17 +185,17 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 12,
+        "w": 5,
-        "x": 12,
+        "x": 14,
         "y": 0
       },
-      "id": 6,
+      "id": 23,
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
-        "graphMode": "area",
+        "graphMode": "none",
         "justifyMode": "auto",
-        "orientation": "horizontal",
+        "orientation": "auto",
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
@@ -173,14 +215,249 @@
             "type": "loki",
             "uid": "P8E80F9AEF21F6940"
           },
-          "editorMode": "code",
+          "direction": "backward",
-          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"} |~ \"(?i)$searchable_pattern\")[$__interval]))",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Generated a trip` [$__interval]))",
           "hide": false,
           "queryType": "range",
           "refId": "A"
         }
       ],
-      "title": "Total Count: of $searchable_pattern",
+      "title": "Generated Trips",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Total count of log warnings",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "light-orange",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 6,
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.5.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"warning\"\n)[$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Total warnings",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Failed trip generations",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 14,
+        "y": 3
+      },
+      "id": 24,
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.5.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Trip generation failed` or `Exception in ASGI` [$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed Trips",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Total count of log errors",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "light-red",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 3
+      },
+      "id": 25,
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.5.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"error\"\n)[$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Total errors",
       "type": "stat"
     },
     {
@@ -194,10 +471,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 9,
+        "h": 11,
         "w": 24,
         "x": 0,
-        "y": 3
+        "y": 6
       },
       "id": 2,
       "options": {
@@ -220,7 +497,7 @@
           },
           "direction": "backward",
           "editorMode": "code",
-          "expr": "{app=\"anyway\", environment=~\"$environment\"} | json | name != \"uvicorn.access\" \n| detected_level=~\"(?i)^(debug|info|warn|error)$\" \n| detected_level=~\"(?i)$logLevel|info|warn|error\" | line_format \"{{ .message }}\"\n",
+          "expr": "{app=\"anyway\", environment=~\"$environment\"} \n| json\n| name != \"uvicorn.access\"\n| detected_level =~ \"${LogLevel}\"\n| line_format \"{{ .message }}\"\n",
           "hide": false,
           "queryType": "range",
           "refId": "A"
@@ -234,6 +511,7 @@
         "type": "loki",
         "uid": "P8E80F9AEF21F6940"
       },
+      "description": "Detailed distribution of warning types",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -251,35 +529,37 @@
               "options": {
                 "match": "null",
                 "result": {
-                  "color": "#299c46",
                   "text": "0"
                 }
               },
               "type": "special"
             }
-          ],
+          ]
-          "unit": "none"
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 7,
+        "h": 6,
-        "w": 9,
+        "w": 8,
         "x": 0,
-        "y": 12
+        "y": 17
       },
-      "id": 19,
+      "id": 26,
       "maxDataPoints": 100,
       "options": {
+        "displayLabels": [
+          "value"
+        ],
         "legend": {
           "displayMode": "list",
           "placement": "right",
-          "showLegend": true
+          "showLegend": true,
+          "values": []
         },
         "pieType": "pie",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "sum"
           ],
           "fields": "",
           "values": false
@@ -297,14 +577,69 @@
             "type": "loki",
             "uid": "P8E80F9AEF21F6940"
           },
+          "direction": "backward",
           "editorMode": "code",
-          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"})[$__interval])) by (stream)",
+          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"warn\"\n| function =~ \"solve_optimization\"\n)[$__interval]))",
           "hide": false,
+          "legendFormat": "Optimizer",
           "queryType": "range",
-          "refId": "A"
+          "refId": "optimization_warnings",
+          "step": ""
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Refiner failed` [$__interval]))",
+          "hide": false,
+          "legendFormat": "Refiner",
+          "queryType": "range",
+          "refId": "refiner_warnings"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} \n| json \n| detected_level =~ \"warn\"\n| module =~ \"cluster_manager\"\n[$__interval]))",
+          "hide": false,
+          "legendFormat": "Cluster Manager",
+          "queryType": "range",
+          "refId": "clusters_warning"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json \n| detected_level =~ `warn` \n| module !~ \"cluster_manager\"\n| module !~ \"landmarks_manager\"\n| function !~ `solve_optimization` != `Refiner failed` [$__interval]))",
+          "hide": false,
+          "legendFormat": "Other",
+          "queryType": "range",
+          "refId": "other"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} \n| json \n| detected_level =~ \"warn\"\n| module =~ \"landmarks_manager\"\n[$__interval]))",
+          "hide": false,
+          "legendFormat": "Landmarks Manager",
+          "queryType": "range",
+          "refId": "landmarks"
         }
       ],
-      "title": "Total count of stderr / stdout pie",
+      "title": "Warnings detail",
       "type": "piechart"
     },
     {
@@ -312,6 +647,7 @@
         "type": "loki",
         "uid": "P8E80F9AEF21F6940"
       },
+      "description": "Detailed distribution of error types",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -324,29 +660,42 @@
               "viz": false
             }
           },
-          "mappings": []
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ]
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 7,
+        "h": 6,
-        "w": 9,
+        "w": 8,
-        "x": 9,
+        "x": 8,
-        "y": 12
+        "y": 17
       },
-      "id": 20,
+      "id": 22,
-      "interval": "1m",
+      "maxDataPoints": 100,
-      "maxDataPoints": "",
       "options": {
+        "displayLabels": [
+          "value"
+        ],
         "legend": {
           "displayMode": "list",
           "placement": "right",
-          "showLegend": true
+          "showLegend": true,
+          "values": []
         },
         "pieType": "pie",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "sum"
           ],
           "fields": "",
           "values": false
@@ -364,13 +713,56 @@
             "type": "loki",
             "uid": "P8E80F9AEF21F6940"
           },
+          "direction": "backward",
           "editorMode": "code",
-          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (pod)",
+          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"error\"\n| function =~ \"solve_optimization\"\n)[$__interval]))",
+          "hide": false,
+          "legendFormat": "Optimizer",
           "queryType": "range",
-          "refId": "A"
+          "refId": "optimization_errors",
+          "step": ""
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"error\"\n| function =~ \"fill_cache\"\n)[$__interval]))",
+          "hide": false,
+          "legendFormat": "Cache",
+          "queryType": "range",
+          "refId": "cache_errors"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json | detected_level =~ `error` |= `Overpass API` or `overpass API` [$__interval]))",
+          "hide": false,
+          "legendFormat": "Overpass API",
+          "queryType": "range",
+          "refId": "overpass"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "direction": "backward",
+          "editorMode": "builder",
+          "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json | detected_level =~ `error` | function !~ `fill_cache` | function !~ `solve_optimization` != `Overpass API` or `overpass API` [$__interval]))",
+          "hide": false,
+          "legendFormat": "Other",
+          "queryType": "range",
+          "refId": "other"
         }
       ],
-      "title": "Matched word: \"$searchable_pattern\" donut",
+      "title": "Error detail",
       "type": "piechart"
     },
     {
@@ -409,10 +801,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 7,
+        "h": 6,
-        "w": 6,
+        "w": 8,
-        "x": 18,
+        "x": 16,
-        "y": 12
+        "y": 17
       },
       "id": 9,
       "maxDataPoints": 100,
@@ -514,7 +906,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 19
+        "y": 23
       },
       "id": 18,
       "interval": "1m",
@@ -617,7 +1009,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 19
+        "y": 23
       },
       "id": 10,
       "maxDataPoints": 100,
@@ -771,7 +1163,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 32
       },
       "id": 7,
       "maxDataPoints": 100,
@@ -819,7 +1211,7 @@
         "h": 11,
         "w": 24,
         "x": 0,
-        "y": 36
+        "y": 40
       },
       "id": 21,
       "options": {
@@ -894,29 +1286,40 @@
       },
       {
         "current": {
-          "text": "debug",
+          "text": "|debug|info|warn|error",
-          "value": "debug"
+          "value": "|debug|info|warn|error"
         },
+        "description": "Allows to filter the logs based on their level",
         "name": "LogLevel",
         "options": [
-          {
-            "selected": false,
-            "text": "info",
-            "value": "info"
-          },
           {
             "selected": true,
             "text": "debug",
-            "value": "debug"
+            "value": "|debug|info|warn|error"
+          },
+          {
+            "selected": false,
+            "text": "info",
+            "value": "|info|warn|error"
+          },
+          {
+            "selected": false,
+            "text": "warn",
+            "value": "|warn|error"
+          },
+          {
+            "selected": false,
+            "text": "error",
+            "value": "|error"
           }
         ],
-        "query": "info, debug",
+        "query": "debug : |debug|info|warn|error , info : |info|warn|error , warn :  |warn|error , error : |error",
         "type": "custom"
       }
     ]
   },
   "time": {
-    "from": "now-24h",
+    "from": "now-7d",
     "to": "now"
   },
   "timepicker": {
@@ -935,6 +1338,6 @@
   "timezone": "",
   "title": "AnyWay status",
   "uid": "fRIvzUZMz",
-  "version": 1,
+  "version": 3,
   "weekStart": ""
 }