monitoring fixes

apps/grafana/grafana-admin.sealedsecret.yaml

@@ -4,14 +4,14 @@ kind: SealedSecret
 metadata:
   creationTimestamp: null
   name: grafana-admin-secret
-  namespace: monitoring
+  namespace: grafana
 spec:
   encryptedData:
-    password: AgAwMLnsYN1y8JQSqgGQbNG/8jKensTDsEw6ogITdkhDRlJcg8HQ5t7a6xLzNCrLHLJiQW8YOoyLT4lvFkBRMOa2EYcrDvBiRD0PjygWLIscKa7dA+jpAUf/icD9zsiDnTym2yf+VUANcmEgE6DiNvlcsrcmYqiR4pKVUTDlKPNOjOpTJ3nXETb3/sbt69E0JSGwtkvusYQSXKLU9KLbciihv+ycdkdlC9xy9myd4+vYZYXSh/eAvyZeb/hsmdSX7yaASmupMvet6Qsdt99PNzFQxtbQH+LQvYalVZ8bjWZQvCN/p0bA4H15otKBfe8rtEwVthgvyEvo6TK0Mg0pFY/b3AOGFmImnT3rDmgG6S8KTZH0Jce17ksFqvELQmHjqHuYpQsPDl44glM8kWRJ9Mf/Z424LRwZlJNVcOkuVl4qFqPUjzd2rWIyF0RaD0BE012C0ThJxKn2l17lVJbNtdUiR3qNpW01ot2m0CgKd2kXbjDmgRgAll4WgrukfCIn9ZnE0gVCFLJuK3MOQAaipFYy/bDO0izwl9T8nldgcI8OfiC3NTk2O+Es5jJRXu0oJGaC3HrTB7wXiwOoELvAsxLTPxKBiN9mCHCMtZX0PEtrio0dFRQ6Pi5xPng0KVT0I9dvGNsPdhPETNOB913WEvbgP8Gt3cj016nCzk51eUsYbXPpNL2B4kmbIhecqW/8kwKQPwYjVlBSXj3NxjzwMY6PvOl1
+    password: AgAU6g/CwKj+1gPpt4DLvLsS0YCvJdVHWw4W4bRhibE9brVvcJtGB3D9MTJrSLVVwusaE6OR59og7oW5ge3yTd/9bbclXYLrxEi7OwvkQjCvo8MfD8yhJO9nV4Xs9Mjk2Z4SHGYuq6wvcssuJrpz5f0XEC7ocTRA+u0UaE+/b4FrYF71uyKGvj8GSXgLZUjGPFsGfPzwJn7cLBmlclVHx1xGbFpUc042m5Mulpn0QolFQnOwZiW4PL8pQyz1MXVRwCsz0RJd5apZL3XJ4X7BLMoAp+diHQ2xi3zoU9VScp+J2QgvFdRKgDa6v7Jz1f+HCwq5W/DoegwFXBrcMIfF2YrnvTnc1PCVwD9IHOeylO7J2hfi8teQiqTvvRlVgdBTLqoqlVovemf5k6ke6JfjTwnsJjTNnL7MKN5Qt0o7N2XRZ3ba9jp8cKbI7fyFQKaU2QEf2PIkp82kEnixmpA1aATgeA3W4E5Km7sKHUEB81+pwnOe54tzD2ShgQX/+UiswhWYTT+gdZKL1udBBemUDC0z9PSJNTPTy+hq+G4CIzVQUYxlioM3c+3geF7YLU8yXisj84pk44GN9KX3z5x+M2+LZL7agAWPUjxtrP2V+id7dNJQfCm0aSMeo57dVfb4zlBUAAgKIKjX+j1KqCVqE9zEO2F/QX7mY6MJTP2me3wmY7JAVRJ7d6bbkyyoDhs8JErLYLp0A+Eh+qx8nWgM9ErPVSA0
-    user: AgBqmjCYGMqy5zBE+vhtsynOvhWdHWDJDyl1D+laBtLjXTJwzRbNTdunHYo1ekwyqQ6Cr5pi4YMiLxAl1LIHF+Lfsp2QlY+ResAGzp9WgSBtNQDX3EmLDQofeWxMUDdMtMsE9wiKLCfNGDkRDsGquXTz+YFq03m1vH9cB8Bp+1ClWOTui+/Ce0MZlWsJZX1W8WXH7XTirtwUo0s53pc4AplUUH97ZEK3KSIxWa3gLCn0sAPDDLPX+JVA2xtpMq1XuVFiFifjzEtG2h0dejiF35FtSAR+rR4YmEfimk3QpRDfOqV5QUxvjCG+dTV49upSevF2mvbHW+o+lB6vEc6l9cZXvlbnMdaep3NmOsJcJ8wQIdFpFK4iVzFOTKSEbzLPlZ/J+sjS5vDXsfthorIO2faMA1iIf+I663zNxQU5btaK4TNYOZQlrFVjAmioRLkDhGZ6tDUPX/zMv+Crt+0HCwyEyhmvFZckDvezTZrxARSXXMKBVcvjHCyUNkz7ubZRiMU0PGM7fYuHr659e+XMRvj+LFA68ZaEIzCQpCFJenWWYAXgUdRG4LQ1LP2MwvRHpkOYSoRkHIpX7jOfhX82A60h/ta/CdbWifqNyL9OecvE3FKsZu/Kr0taw9W6nm6FBhQLgFkOnFrqp9dWnxfHruXuDBgcn0iE8nR7Ht2zS7hfQPeR4a3Y0xK3Plqbzdrb9HKnWQQhf14=
+    user: AgB8ZLG2EuERjg1nKdH/xadbUuIR2c8a9gF5fE8ctrp4DNDLLuuqmjyoHRiWpkrtfnE1yKg1rPP+asV9Lj5iVmE9J+OB3QUOeFS4MHciBNj7pa68zfFgnHP4kxMX6aXyKRQrYruYjHwfzCpOM1zyTEphuGlnokjQXxjF/mZsoM2NWn7WGReqfxqH95tJXfs9AUC5vVv/PHqd+KKRZH7+G1AnWVJ7RFQHedR7wyftO4/rkm8deMuZWtOLl25fAOyOr7+hSqT69s9/uTKSLJXjobSqtulqsR+v5lkwx2ThNKzmcEcuoenKG6lk8XLRSIscccZH3JTPh6IknQWUOC4nmYj+XUxE8Go0RX/4eL+D/6FrYrtp0gr3HOCLAGU4vAHMeKfJoyqykJVnvY6QY6bFgaziyOlWaoEHpg6g0vHHDwyX7HIDcQfJZGOLH9dhrWJ2sOkzyuuxfqWEgz/M2eBW4EUAudHwfTLPocSMUI+D6fjeciMojet5uxWMP7ZHh/E061f5+Vfk6CKYd9Kpi69Xah8KEyyHYP5NImkdIwjgllaEAd/FBE2+QJyTVZlUQC7y9ObagDMCUFaFbTS5QOLh5BOJDL5buEYFWG0IhoH47SC/pKeEOQH//uvoo27K9zvxTOQN1YOTrxCozmexMOsTIdhvU0dOnJDBrThSHKYLCeIokDOgUUT52FqDH51RoLoK3UkyGbMoq+M=
   template:
     metadata:
       creationTimestamp: null
       name: grafana-admin-secret
-      namespace: monitoring
+      namespace: grafana
     type: Opaque

apps/grafana/grafana-auth.sealedsecret.yaml

@@ -4,13 +4,13 @@ kind: SealedSecret
 metadata:
   creationTimestamp: null
   name: grafana-auth
-  namespace: monitoring
+  namespace: grafana
 spec:
   encryptedData:
-    client_secret: AgCcKsnS3u2eI+fNVC9hAZ3QRFOHFErAzs5aQgX51CSdJwM03SZUoTyrDi5JPcHUVyS3MbevFH5piMhDTARMI3bLOjYlcwMbpf77JCPa7o95Y9asA/FW3lXicYt3biN9xBXJBz7Ws3fVRtEzyf6DmbGedT9gaX8aPwrUVbP19RdyJiuu76oB1A/jdUkX4K+X6kVvmoP/BWdypk/kdQJrzBNt00DIXF4NHfYey36AuhpBtqYZs4faA/tBXMXLE4RxPNtcHwNfVjnRj3v3qzNufD1fnweJvLq2UfLMrQjoR9XDVnM0zkpautylkI7yrvcoEH7ljnf6b1FMogOEZUfH1BIdqTd/WwrrlCqE58OPfJWthIfN+pQ8LvdHsGo3jc9gXvfXS2cStyhP06eTZ4D79kG+RtDQGOsD/Wpx7EcM6hbB3+dIjcs3wEAIGjpIVtY9JayW8YeRnFApMuhDST1+hscm+LdoGvaSTlAuGzv9BbVrPX/Fo9XKeYHlbG/x71Er+vF8WbW0wUa46MHLvbEy376XIdJDYi+vjl4eqznZ6YhvPbawhoKXT8ZcKUcUAjVcMue/O/jCSPZplbn3vdSCeqPTiqVqDw9PTMIeWFUepgPMxiGpFRAqdwIecFBnYItq0dXoGlFrZpo0S6AECgZjxzUR5EgdkdPlDDs2CN+d9yP7f2S+gmL7AIlQr74NW1GrTGw2x/rD4IJhunh7
+    client_secret: AgCEdC1/ERlPQyQP+bd9gcW33Yrvl4uRbx+RF5AY4vYAquOzxmLTygMl/WZlB5wlCE5idIHgto6/fUWVZrQbmfClRqsW2pFoddKQAtS9cQNXwMjLCm7e0lXk9GM9O3ZwktmklFbCu8XewHmefGHhoJ28vPxPMaINv1fM4zYKvNz5RHf0dJfTHgxb68wRYjAbE/eJpRcVE3a29Yw6Gfa8Mb+cFI7RTHvjuv9LBgWqM6b3qvvJ4wYR2WKuiQrnJ5xAtHpMAI/2R80qq151wlaZueDZ1PwjRBHURkmPTmwZnrMrmIugNge7Tpww+ArZlG9kDfSu1aTJidbXbcpN6fyt1qARTCYrBlbn60PTYLnPL/NObvMCpjS6DsYsYz7MJ7WoOupu46Ib5paZHmak+CilC6lb9LjJj4EKfRsagZmWT07JavhHBW/tqjB3GToccIz4fOAOdA9aU51J4wCL2ctp2SgzCEKe2EaBK/f9nDd9ASmmon9PDwRDVtG8yTukrNcZHNzodi09Af81DB0RNa36Z3Sjt5xu94paN+mjiOWGf2JduVEq+60NbPvDbPE9e1aVH3DdQcij2WGZaTE8dAGLSsLoOkIq3m2E+Mbk1Re1gI9H18xJM72ivb5uDe7pzReyvO5DY4Pfq8JgQhPxWcDq9ScmWS6Bb+jdCKytFq5NafSAl+akPbbwN+1GFu33if/P5D9I2TwOA8V1wyVU
   template:
     metadata:
       creationTimestamp: null
       name: grafana-auth
-      namespace: monitoring
+      namespace: grafana
     type: Opaque

apps/grafana/grafana.values.yaml

@@ -35,13 +35,17 @@ datasources:
   datasources.yaml:
     apiVersion: 1
     datasources:
+      - name: Prometheus
+        type: prometheus
+        url: http://prometheus-server.monitoring.svc:80
+        isDefault: true
       - name: Thanos
         type: prometheus
         url: http://thanos-querier.monitoring.svc:10902
-        isDefault: true
+        isDefault: false
-      - name: Prometheus
+      - name: Loki
-        type: prometheus
+        type: loki
-        url: http://prometheus.monitoring.svc:9090
+        url: http://loki.monitoring.svc:3100
         isDefault: false
 
 dashboardProviders:
@@ -91,3 +95,4 @@ grafana.ini:
     tls_skip_verify_insecure: true
     auto_login: true
     use_pkce: true
+    role_attribute_path: contains(groups[*], 'apps_admin') && 'Admin' || 'Editor'

apps/grafana/kustomization.yaml

@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-namespace: monitoring
+namespace: grafana
 
 resources: 
   - namespace.yaml

infrastructure/monitoring/loki.values.yaml

@@ -5,11 +5,12 @@ loki:
     configs:
       - from: "2024-04-01"
         store: tsdb
-        object_store: s3
+        object_store: filesystem
         schema: v13
         index:
           prefix: loki_index_
           period: 24h
+  auth_enabled: false
   pattern_ingester:
     enabled: true
   limits_config:
@@ -19,6 +20,12 @@ loki:
   ruler:
     enable_api: true
   storage:
+    bucketNames:
+      # don't care since we use the filesystem
+      chunks: NOTUSED
+      ruler: NOTUSED
+      admin: NOTUSED
+
     type: filesystem
     filesystem:
       chunks_directory: /var/loki/chunks
@@ -46,6 +53,12 @@ singleBinary:
     # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
     storageClass: nfs-client
 
+
+# -- Section for configuring optional Helm test
+helm:
+  enabled: false
+
+
 # Zero out replica counts of other deployment modes
 backend:
   replicas: 0

infrastructure/monitoring/namespace.yaml

@@ -2,3 +2,5 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: placeholder
+  labels:
+    pod-security.kubernetes.io/enforce: privileged 

infrastructure/monitoring/prometheus.values.yaml

@@ -6,7 +6,7 @@ server:
     log.level: debug
     storage.tsdb.min-block-duration: 2h # Don't change this, see docs/components/sidecar.md
     storage.tsdb.max-block-duration: 2h # Don't change this, see docs/components/sidecar.md
-  retention: 4h
+  retention: 180d
   service:
     annotations:
       prometheus.io/scrape: "true"
@@ -16,68 +16,69 @@ server:
   podAnnotations:
     prometheus.io/scrape: "true"
     prometheus.io/port: "10902"
-  sidecarContainers:
+  # sidecarContainers:
-    thanos-sidecar:
+  #   thanos-sidecar:
-      image: thanos
+  #     image: thanos
-      resources:
+  #     resources:
-        requests:
+  #       requests:
-          memory: "512Mi"
+  #         memory: "512Mi"
-      env:
+  #     env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
+  #       - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/secret/sa
+  #         value: /etc/secret/sa
-      args:
+  #     args:
-        - "sidecar"
+  #       - "sidecar"
-        - "--log.level=debug"
+  #       - "--log.level=debug"
-        - "--tsdb.path=/data/"
+  #       - "--tsdb.path=/data/"
-        - "--prometheus.url=http://127.0.0.1:9090"
+  #       - "--prometheus.url=http://127.0.0.1:9090"
-        - "--objstore.config={type: GCS, config: {bucket: BUCKET_REPLACE_ME}}"
+  #       - "--objstore.config={type: GCS, config: {bucket: BUCKET_REPLACE_ME}}"
-        - "--reloader.config-file=/etc/prometheus-config/prometheus.yml"
+  #       - "--reloader.config-file=/etc/prometheus-config/prometheus.yml"
-        - "--reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yml"
+  #       - "--reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yml"
-        - "--reloader.rule-dir=/etc/prometheus-config/rules"
+  #       - "--reloader.rule-dir=/etc/prometheus-config/rules"
-      ports:
+  #     ports:
-        - name: sidecar-http
+  #       - name: sidecar-http
-          containerPort: 10902
+  #         containerPort: 10902
-        - name: grpc
+  #       - name: grpc
-          containerPort: 10901
+  #         containerPort: 10901
-        - name: cluster
+  #       - name: cluster
-          containerPort: 10900
+  #         containerPort: 10900
-      volumeMounts:
+  #     volumeMounts:
-        - name: storage-volume
+  #       - name: storage-volume
-          mountPath: /data
+  #         mountPath: /data
-        - name: thanos-storage-secret
+  #       - name: thanos-storage-secret
-          mountPath: /etc/secret
+  #         mountPath: /etc/secret
-        - name: config-volume
+  #       - name: config-volume
-          mountPath: /etc/prometheus-config
+  #         mountPath: /etc/prometheus-config
-          readOnly: false
+  #         readOnly: false
-        - name: prometheus-config-shared
+  #       - name: prometheus-config-shared
-          mountPath: /etc/prometheus-shared/
+  #         mountPath: /etc/prometheus-shared/
-          readOnly: false
+  #         readOnly: false
-  configPath: /etc/prometheus-shared/prometheus.yml
+  # # configPath: /etc/prometheus-shared/prometheus.yml
-  replicaCount: 1
+  # replicaCount: 1
-  persistentVolume:
+  # persistentVolume:
-    size: 20Gi
+  #   size: 20Gi
-  extraVolumes: # spec.template.spec.volumes
+  #   storageClass: nfs-client
-    - name: prometheus-config-shared
+  # extraVolumes: # spec.template.spec.volumes
-      emptyDir: {}
+  #   - name: prometheus-config-shared
-  extraVolumeMounts: # spec.template.spec.containers.volumeMounts for prometheus container
+  #     emptyDir: {}
-    - name: prometheus-config-shared
+  # extraVolumeMounts: # spec.template.spec.containers.volumeMounts for prometheus container
-      mountPath: /etc/prometheus-shared/
+  #   - name: prometheus-config-shared
-  resources:
+  #     mountPath: /etc/prometheus-shared/
-    requests:
+  # resources:
-      memory: 1Gi
+  #   requests:
-  global:
+  #     memory: 1Gi
-    scrape_interval: 5s
+  # global:
-    scrape_timeout: 4s
+  #   scrape_interval: 5s
-    external_labels:
+  #   scrape_timeout: 4s
-      prometheus_group: KLUSTER
+  #   external_labels:
-      prometheus_replica: '$(HOSTNAME)'
+  #     prometheus_group: KLUSTER
-    evaluation_interval: 5s
+  #     prometheus_replica: '$(HOSTNAME)'
-  extraSecretMounts:
+  #   evaluation_interval: 5s
-    - name: thanos-objstore-config
+  # extraSecretMounts:
-      mountPath: /etc/secret/
+  #   - name: thanos-storage-secret
-      subPath: sa
+  #     mountPath: /etc/secret/
-      readOnly: false
+  #     subPath: sa
-      secretName: thanos-storage-secret
+  #     readOnly: false
+  #     secretName: thanos-objstore-config
 
 # as thanos sidecar is taking care of the config reload
 # we can disable the prometheus configmap reload

kluster-deployments/grafana/application.yaml

@@ -16,3 +16,5 @@ spec:
     automated:
       prune: true
       selfHeal: true
+    syncOptions:
+      - Replace=true

kluster-deployments/monitoring/application.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: prometheus-application
+  name: monitoring-application
   namespace: argocd
 
 spec:
@@ -9,7 +9,7 @@ spec:
   source:
     repoURL: git@github.com:moll-re/bootstrap-k3s-infra.git
     targetRevision: main
-    path: infrastructure/prometheus
+    path: infrastructure/monitoring
   destination:
     server: https://kubernetes.default.svc
     namespace: monitoring