Add Loki retention

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

charts/meta-monitoring/templates/agent/config.yaml

@@ -37,6 +37,8 @@ data:
       }
     }
 
+    // Logs
+
     {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     loki.source.kubernetes "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
@@ -58,6 +60,8 @@ data:
     {{- end }}
     {{- end }}
 
+    // Metrics
+
     {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
     prometheus.scrape "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
@@ -70,8 +74,117 @@ data:
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
     {{- end }}
+
+    // cAdvisor and Kubelete metrics
+    // Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
+    discovery.kubernetes "all_nodes" {
+      role = "node"
+    }
+
+    discovery.relabel "all_nodes" {
+      targets = discovery.kubernetes.all_nodes.targets
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        target_label = "node"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterName -}}"
+      }
+    }
+
+    prometheus.scrape "cadvisor" {
+      targets    = discovery.relabel.all_nodes.output
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      scrape_interval = "15s"
+      metrics_path = "/metrics/cadvisor"
+      scheme = "https"
+
+      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      tls_config {
+        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+      }
+    }
+
+    prometheus.scrape "kubelet" {
+      targets    = discovery.relabel.all_nodes.output
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      scrape_interval = "15s"
+      metrics_path = "/metrics"
+      scheme = "https"
+
+      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      tls_config {
+        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+      }
+    }
+
+    prometheus.exporter.unix {}
+
+    prometheus.scrape "node_exporter" {
+      targets = prometheus.exporter.unix.targets
+      forward_to = [prometheus.relabel.node_exporter.receiver]
+
+      job_name = "node-exporter"
+      scrape_interval = "15s"
+    }
+
+    prometheus.relabel "node_exporter" {
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+
+      rule {
+        replacement = env("HOSTNAME")
+        target_label = "nodename"
+      }
+      rule {
+        replacement = "node-exporter"
+        target_label = "job"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_node_name"]
+        target_label = "node"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterName -}}"
+      }
+    }
     {{- end }}
 
+    // Traces
+
     {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {

charts/meta-monitoring/values.yaml

@@ -71,13 +71,22 @@ loki:
     storage:
       type: "s3"
       s3:
-        endpoint: "meta-minio.meta.svc:9000"
-        access_key_id: rootuser
-        secret_access_key: rootpassword
         insecure: true
+        s3ForcePathStyle: true
       bucketNames:
         chunks: loki-chunks
         ruler: loki-ruler
+    structuredConfig:
+      common:
+        storage:
+          s3:
+            access_key_id: "{{ .Values.global.minio.rootUser }}"
+            endpoint: "{{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000"
+            secret_access_key: "{{ .Values.global.minio.rootPassword }}"
+      compactor:
+        retention_enabled: true
+      limits_config:
+        retention_period: 24h
   monitoring:
     dashboards:
       enabled: false
@@ -128,6 +137,8 @@ mimir-distributed:
           endpoint: "{{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000"
           secret_access_key: "{{ .Values.global.minio.rootPassword }}"
           insecure: true
+      limits:
+        compactor_blocks_retention_period: 24h
 
 tempo-distributed:
   tempo:
@@ -141,6 +152,9 @@ tempo-distributed:
             access_key: "{{ .Values.global.minio.rootUser }}"
             secret_key: "{{ .Values.global.minio.rootPassword }}"
             insecure: true
+      compactor:
+        compaction:
+          block_retention: 24h
   traces:
     otlp:
       http: