Add node exporter

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

charts/meta-monitoring/templates/agent/config.yaml

@@ -37,6 +37,8 @@ data:
       }
     }
 
+    // Logs
+
     {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     loki.source.kubernetes "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
@@ -58,6 +60,8 @@ data:
     {{- end }}
     {{- end }}
 
+    // Metrics
+
     {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
     prometheus.scrape "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
@@ -70,8 +74,117 @@ data:
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
     {{- end }}
+
+    // cAdvisor and Kubelete metrics
+    // Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
+    discovery.kubernetes "all_nodes" {
+      role = "node"
+    }
+
+    discovery.relabel "all_nodes" {
+      targets = discovery.kubernetes.all_nodes.targets
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        target_label = "node"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterName -}}"
+      }
+    }
+
+    prometheus.scrape "cadvisor" {
+      targets    = discovery.relabel.all_nodes.output
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      scrape_interval = "15s"
+      metrics_path = "/metrics/cadvisor"
+      scheme = "https"
+
+      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      tls_config {
+        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+      }
+    }
+
+    prometheus.scrape "kubelet" {
+      targets    = discovery.relabel.all_nodes.output
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      scrape_interval = "15s"
+      metrics_path = "/metrics"
+      scheme = "https"
+
+      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      tls_config {
+        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+      }
+    }
+
+    prometheus.exporter.unix {}
+
+    prometheus.scrape "node_exporter" {
+      targets = prometheus.exporter.unix.targets
+      forward_to = [prometheus.relabel.node_exporter.receiver]
+
+      job_name = "node-exporter"
+      scrape_interval = "15s"
+    }
+
+    prometheus.relabel "node_exporter" {
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      rule {
+        replacement = env("HOSTNAME")
+        target_label = "nodename"
+      }
+      rule {
+        replacement = "node-exporter"
+        target_label = "job"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        target_label = "node"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterName -}}"
+      }
+    }
     {{- end }}
 
+    // Traces
+
     {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {

charts/meta-monitoring/templates/grafana/configmap-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if .Values.local.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if .Values.local.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-3.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-4.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-5.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-6.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-7.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,16 +1,16 @@
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: dashboards-provisioning
+  name: loki-dashboards-provisioning
   namespace: {{ $.Release.Namespace }}
 data:
   dashboards.yaml: |
     ---
     apiVersion: 1
     providers:
-{{- if .Values.dashboards.logs.enabled }}
+{{- if .Values.local.logs.enabled }}
       - disableDeletion: true
         editable: false
         folder: Loki
@@ -28,7 +28,7 @@ data:
         orgId: 1
         type: file
 {{- end }}
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if .Values.local.metrics.enabled }}
       - disableDeletion: true
         editable: false
         folder: Mimir
@@ -70,14 +70,4 @@ data:
         orgId: 1
         type: file
 {{- end }}
-{{- if .Values.dashboards.traces.enabled }}
-      - disableDeletion: true
-        editable: false
-        folder: Tempo
-        name: tempo-1
-        options:
-          path: /var/lib/grafana/dashboards/tempo-1
-        orgId: 1
-        type: file
-{{- end }}
 {{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -65,17 +65,15 @@ spec:
               name: grafana-pv
             - mountPath: /etc/grafana/provisioning/datasources
               name: datasources-provisioning
-            {{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
             - mountPath: /etc/grafana/provisioning/dashboards
-              name: dashboards-provisioning
-            {{- end }}
-            {{- if .Values.dashboards.logs.enabled }}
+              name: loki-dashboards-provisioning
+            {{- if .Values.local.logs.enabled }}
             - mountPath: /var/lib/grafana/dashboards/loki-1
               name: loki-dashboards-1
             - mountPath: /var/lib/grafana/dashboards/loki-2
               name: loki-dashboards-2
             {{- end }}
-            {{- if .Values.dashboards.metrics.enabled }}
+            {{- if .Values.local.metrics.enabled }}
             - mountPath: /var/lib/grafana/dashboards/mimir-1
               name: mimir-dashboards-1
             - mountPath: /var/lib/grafana/dashboards/mimir-2
@@ -87,10 +85,6 @@ spec:
             - mountPath: /var/lib/grafana/dashboards/mimir-5
               name: mimir-dashboards-5
             {{- end }}
-            {{- if .Values.dashboards.traces.enabled }}
-            - mountPath: /var/lib/grafana/dashboards/tempo-1
-              name: tempo-dashboards-1
-            {{- end }}
       volumes:
         - name: grafana-pv
           persistentVolumeClaim:
@@ -98,10 +92,10 @@ spec:
         - name: datasources-provisioning
           configMap:
             name: datasources-provisioning
-        - name: dashboards-provisioning
+        {{- if .Values.local.logs.enabled }}
+        - name: loki-dashboards-provisioning
           configMap:
-            name: dashboards-provisioning
-        {{- if .Values.dashboards.logs.enabled }}
+            name: loki-dashboards-provisioning
         - name: loki-dashboards-1
           configMap:
             name: loki-dashboards-1
@@ -109,7 +103,10 @@ spec:
           configMap:
             name: loki-dashboards-2
         {{- end }}
-        {{- if .Values.dashboards.metrics.enabled }}
+        {{- if .Values.local.metrics.enabled }}
+        - name: mimir-dashboards-provisioning
+          configMap:
+            name: mimir-dashboards-provisioning
         - name: mimir-dashboards-1
           configMap:
             name: mimir-dashboards-1
@@ -126,11 +123,6 @@ spec:
           configMap:
             name: mimir-dashboards-5
         {{- end }}
-        {{- if .Values.dashboards.traces.enabled }}
-        - name: tempo-dashboards-1
-          configMap:
-            name: tempo-dashboards-1
-        {{- end }}
 
 ---
 apiVersion: v1
@@ -146,4 +138,4 @@ spec:
     app: grafana
   sessionAffinity: None
   type: ClusterIP  # Make this configurable
-{{- end }}
+{{- end }}

charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml

@@ -1,21 +0,0 @@
-{{- if .Values.dashboards.traces.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: tempo-dashboards-1
-  namespace: {{ $.Release.Namespace }}
-data:
-  "tempo-operational.json": |
-    {{ $.Files.Get "src/dashboards/tempo-operational.json" | fromJson | toJson }}
-  "tempo-reads.json": |
-    {{ $.Files.Get "src/dashboards/tempo-reads.json" | fromJson | toJson }}
-  "tempo-resources.json": |
-    {{ $.Files.Get "src/dashboards/tempo-resources.json" | fromJson | toJson }}
-  "tempo-rollout-progress.json": |
-    {{ $.Files.Get "src/dashboards/tempo-rollout-progress.json" | fromJson | toJson }}
-  "tempo-tenants.json": |
-    {{ $.Files.Get "src/dashboards/tempo-tenants.json" | fromJson | toJson }}
-  "tempo-writes.json": |
-    {{ $.Files.Get "src/dashboards/tempo-writes.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/values.yaml

@@ -14,6 +14,7 @@ local:
   minio:
     enabled: false  # This should be set to true if any of the previous is enabled
 
+
 cloud:
   logs:
     enabled: true
@@ -40,15 +41,6 @@ logs:
   #   source: ""         # Empty uses the log message
   #   replace: "*****""
 
-# Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
-dashboards:
-  logs:
-    enabled: true
-  metrics:
-    enabled: true
-  traces:
-    enabled: true
-
 global:
   minio:
     rootUser: "rootuser"