diff --git a/dashboards/anyway_logs.json b/dashboards/anyway_logs.json
index d3f044c..09dd4b6 100644
--- a/dashboards/anyway_logs.json
+++ b/dashboards/anyway_logs.json
@@ -39,6 +39,7 @@
"axisColorMode": "text",
"axisLabel": "Trips",
"axisPlacement": "auto",
+ "axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
@@ -93,7 +94,7 @@
},
"gridPos": {
"h": 6,
- "w": 9,
+ "w": 14,
"x": 0,
"y": 0
},
@@ -106,6 +107,9 @@
"placement": "bottom",
"showLegend": false
},
+ "timezone": [
+ "Europe/Zurich"
+ ],
"tooltip": {
"hideZeros": false,
"mode": "single",
@@ -126,6 +130,19 @@
"legendFormat": "trips generated",
"queryType": "range",
"refId": "generated trips"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "builder",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} |= `Trip generation failed` or `Exception in ASGI` [$__interval]))",
+ "hide": false,
+ "legendFormat": "trips failed",
+ "queryType": "range",
+ "refId": "failed trips"
}
],
"title": "Trips generated",
@@ -168,8 +185,8 @@
},
"gridPos": {
"h": 3,
- "w": 4,
- "x": 9,
+ "w": 5,
+ "x": 14,
"y": 0
},
"id": 23,
@@ -246,8 +263,8 @@
},
"gridPos": {
"h": 3,
- "w": 4,
- "x": 13,
+ "w": 5,
+ "x": 19,
"y": 0
},
"id": 6,
@@ -284,9 +301,347 @@
"refId": "A"
}
],
- "title": "Total warning logs",
+ "title": "Total warnings",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Failed trip generations",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 5,
+ "x": 14,
+ "y": 3
+ },
+ "id": 24,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "sum"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.5.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "builder",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Trip generation failed` or `Exception in ASGI` [$__interval]))",
+ "hide": false,
+ "queryType": "range",
+ "refId": "A"
+ }
+ ],
+ "title": "Failed Trips",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Total count of log errors",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "light-red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 5,
+ "x": 19,
+ "y": 3
+ },
+ "id": 25,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "sum"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.5.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"error\"\n)[$__interval]))",
+ "hide": false,
+ "queryType": "range",
+ "refId": "A"
+ }
+ ],
+ "title": "Total errors",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Without access logs (see below)",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 24,
+ "x": 0,
+ "y": 6
+ },
+ "id": 2,
+ "options": {
+ "dedupStrategy": "none",
+ "enableInfiniteScrolling": false,
+ "enableLogDetails": true,
+ "prettifyLogMessage": false,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": true,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "pluginVersion": "11.5.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "{app=\"anyway\", environment=~\"$environment\"} \n| json\n| name != \"uvicorn.access\"\n| detected_level =~ \"${LogLevel}\"\n| line_format \"{{ .message }}\"\n",
+ "hide": false,
+ "queryType": "range",
+ "refId": "A"
+ }
+ ],
+ "title": "Live logs",
+ "type": "logs"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Detailed distribution of warning types",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ]
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 17
+ },
+ "id": 26,
+ "maxDataPoints": 100,
+ "options": {
+ "displayLabels": [
+ "value"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": []
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "sum"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.5.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"warn\"\n| function =~ \"solve_optimization\"\n)[$__interval]))",
+ "hide": false,
+ "legendFormat": "Optimizer",
+ "queryType": "range",
+ "refId": "optimization_warnings",
+ "step": ""
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "builder",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Refiner failed` [$__interval]))",
+ "hide": false,
+ "legendFormat": "Refiner",
+ "queryType": "range",
+ "refId": "refiner_warnings"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} \n| json \n| detected_level =~ \"warn\"\n| module =~ \"cluster_manager\"\n[$__interval]))",
+ "hide": false,
+ "legendFormat": "Cluster Manager",
+ "queryType": "range",
+ "refId": "clusters_warning"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json \n| detected_level =~ `warn` \n| module !~ \"cluster_manager\"\n| module !~ \"landmarks_manager\"\n| function !~ `solve_optimization` != `Refiner failed` [$__interval]))",
+ "hide": false,
+ "legendFormat": "Other",
+ "queryType": "range",
+ "refId": "other"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "direction": "backward",
+ "editorMode": "code",
+ "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} \n| json \n| detected_level =~ \"warn\"\n| module =~ \"landmarks_manager\"\n[$__interval]))",
+ "hide": false,
+ "legendFormat": "Landmarks Manager",
+ "queryType": "range",
+ "refId": "landmarks"
+ }
+ ],
+ "title": "Warnings detail",
+ "type": "piechart"
+ },
{
"datasource": {
"type": "loki",
@@ -321,9 +676,9 @@
},
"gridPos": {
"h": 6,
- "w": 7,
- "x": 17,
- "y": 0
+ "w": 8,
+ "x": 8,
+ "y": 17
},
"id": 22,
"maxDataPoints": 100,
@@ -410,409 +765,6 @@
"title": "Error detail",
"type": "piechart"
},
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "description": "Failed trip generations",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "0"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "red",
- "value": null
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 4,
- "x": 9,
- "y": 3
- },
- "id": 24,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "sum"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.5.1",
- "targets": [
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "builder",
- "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Trip generation failed` or `Exception in ASGI` [$__interval]))",
- "hide": false,
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Failed Trips",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "description": "Total count of log errors",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "0"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "light-red",
- "value": null
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 4,
- "x": 13,
- "y": 3
- },
- "id": 25,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "sum"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.5.1",
- "targets": [
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "code",
- "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"error\"\n)[$__interval]))",
- "hide": false,
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Total error logs",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "description": "Without access logs (see below)",
- "fieldConfig": {
- "defaults": {},
- "overrides": []
- },
- "gridPos": {
- "h": 11,
- "w": 24,
- "x": 0,
- "y": 6
- },
- "id": 2,
- "options": {
- "dedupStrategy": "none",
- "enableInfiniteScrolling": false,
- "enableLogDetails": true,
- "prettifyLogMessage": false,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": true,
- "sortOrder": "Descending",
- "wrapLogMessage": false
- },
- "pluginVersion": "11.5.1",
- "targets": [
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "code",
- "expr": "{app=\"anyway\", environment=~\"$environment\"} \n| json\n| name != \"uvicorn.access\"\n| detected_level =~ \"${LogLevel}\"\n| line_format \"{{ .message }}\"\n",
- "hide": false,
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Live logs",
- "type": "logs"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "description": "Detailed distribution of warning types",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "0"
- }
- },
- "type": "special"
- }
- ]
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 7,
- "x": 0,
- "y": 17
- },
- "id": 26,
- "maxDataPoints": 100,
- "options": {
- "displayLabels": [
- "value"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true,
- "values": []
- },
- "pieType": "pie",
- "reduceOptions": {
- "calcs": [
- "sum"
- ],
- "fields": "",
- "values": false
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.5.1",
- "targets": [
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "code",
- "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"}\n| json\n| detected_level =~ \"warn\"\n| function =~ \"solve_optimization\"\n)[$__interval]))",
- "hide": false,
- "legendFormat": "Optimizer",
- "queryType": "range",
- "refId": "optimization_errors",
- "step": ""
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "builder",
- "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json |= `Refiner failed` [$__interval]))",
- "hide": false,
- "legendFormat": "Refiner",
- "queryType": "range",
- "refId": "refiner_warnings"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "code",
- "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} \n| json \n| detected_level =~ \"warn\"\n| module =~ \"cluster_manager\"\n[$__interval]))",
- "hide": false,
- "legendFormat": "Cluster Manager",
- "queryType": "range",
- "refId": "clusters_warning"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "direction": "backward",
- "editorMode": "code",
- "expr": "sum(count_over_time({app=\"anyway\", environment=~\"$environment\"} | json \n| detected_level =~ `warn` \n| module !~ \"cluster_manager\"\n| function !~ `solve_optimization` != `Refiner failed` [$__interval]))",
- "hide": false,
- "legendFormat": "Other",
- "queryType": "range",
- "refId": "other"
- }
- ],
- "title": "Warnings detail",
- "type": "piechart"
- },
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "color": "#299c46",
- "text": "0"
- }
- },
- "type": "special"
- }
- ],
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 9,
- "x": 7,
- "y": 17
- },
- "id": 19,
- "maxDataPoints": 100,
- "options": {
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true
- },
- "pieType": "pie",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.5.1",
- "targets": [
- {
- "datasource": {
- "type": "loki",
- "uid": "P8E80F9AEF21F6940"
- },
- "editorMode": "code",
- "expr": "sum(count_over_time(({app=\"anyway\", environment=~\"$environment\"})[$__interval])) by (stream)",
- "hide": false,
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Total count of stderr / stdout pie",
- "type": "piechart"
- },
{
"datasource": {
"type": "loki",