Scrape agent metrics as well

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

charts/meta-monitoring/Chart.lock

@@ -1,18 +1,18 @@
 dependencies:
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: 5.8.0
+  version: 5.47.2
 - name: grafana-agent
   repository: https://grafana.github.io/helm-charts
-  version: 0.15.0
+  version: 0.37.0
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 4.4.1
+  version: 5.2.0
 - name: tempo-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 1.4.7
+  version: 1.9.1
 - name: minio
   repository: https://charts.min.io
   version: 5.0.11
-digest: sha256:da0e744b5046eb7972e0bf82d1d0ba4786e9600af63b65f35b16118105248074
-generated: "2023-08-18T10:58:08.978123+01:00"
+digest: sha256:7b7e62e08d9a56e63fdb12ce3fd4d1fda4887545546ac3e98c7886be714fd763
+generated: "2024-04-02T15:09:13.121195+01:00"

charts/meta-monitoring/Chart.yaml

@@ -26,18 +26,18 @@ appVersion: "0.0.1"
 dependencies:
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: "5.8.0"
+  version: "5.47.2"
   condition: local.logs.enabled
 - name: grafana-agent
   repository: https://grafana.github.io/helm-charts
-  version: "0.15.0"
+  version: "0.37.0"
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
-  version: "4.4.1"
+  version: "5.2.0"
   condition: local.metrics.enabled
 - name: tempo-distributed
   repository: https://grafana.github.io/helm-charts
-  version: "1.4.7"
+  version: "1.9.1"
   condition: local.traces.enabled
 - name: minio
   repository: https://charts.min.io

charts/meta-monitoring/src/rules/loki-rules.yaml

@@ -1,53 +1,53 @@
 groups:
 - name: "loki_rules"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[1m]))
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[5m]))
       by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)
+      / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job,
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job,
       route)"
     record: "cluster_job_route:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
+      job, route) / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate"

charts/meta-monitoring/src/rules/mimir-rules.yaml

@@ -1,322 +1,317 @@
 groups:
 - name: "mimir_api_1"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[5m]))
       by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_api_2"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job, route)
+      / sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, job,
       route)"
     record: "cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)"
+  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_api_3"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, namespace,
+      job, route) / sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_querier_api"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+      job) / sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[5m])) by
       (cluster, job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[5m]))
       by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate"
 - name: "mimir_cache"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
-      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[5m])) by (cluster,
+      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[5m]))
       by (cluster, job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_bucket[5m])) by (le, cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[5m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_count[5m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_cache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_cache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job)
+      / sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_cache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
-      method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job,
+      method) / sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le, cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job,
       method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate"
 - name: "mimir_storage"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_kv_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_kv_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[5m])) by (cluster, job)
+      / sum(rate(cortex_kv_request_duration_seconds_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_bucket[5m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_count:sum_rate"
 - name: "mimir_queries"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_retries:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_retries:50quantile"
-  - expr: "sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
+  - expr: "sum(rate(cortex_query_frontend_retries_sum[5m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[5m]))
       by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries:avg"
-  - expr: "sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_bucket[5m])) by (le, cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_bucket:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_sum:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[5m])) by (cluster,
+      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[5m])) by
       (cluster, job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:avg"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m])) by (le,
       cluster, job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_count[5m])) by (cluster,
       job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate"
 - name: "mimir_ingester_queries"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_series:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_series:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
+  - expr: "sum(rate(cortex_ingester_queried_series_sum[5m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[5m]))
       by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series:avg"
-  - expr: "sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_bucket[5m])) by (le, cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_samples:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_samples:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
+  - expr: "sum(rate(cortex_ingester_queried_samples_sum[5m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[5m]))
       by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples:avg"
-  - expr: "sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_bucket[5m])) by (le, cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_exemplars:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[5m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_exemplars:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
-      sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[5m])) by (cluster, job) /
+      sum(rate(cortex_ingester_queried_exemplars_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars:avg"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_bucket[5m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_count[5m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_count:sum_rate"
 - name: "mimir_received_samples"
   rules:
-  - expr: "|
-      sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))"
+  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_received_samples:rate5m"
 - name: "mimir_exemplars_in"
   rules:
-  - expr: "|
-      sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))"
+  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_exemplars_in:rate5m"
 - name: "mimir_received_exemplars"
   rules:
-  - expr: "|
-      sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))"
+  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_received_exemplars:rate5m"
 - name: "mimir_exemplars_ingested"
   rules:
-  - expr: "|
-      sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))"
+  - expr: "sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))"
     record: "cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m"
 - name: "mimir_exemplars_appended"
   rules:
-  - expr: "|
-      sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))"
+  - expr: "sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))"
     record: "cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m"
 - name: "mimir_scaling_rules"
   rules:
-  - expr: "|
+  - expr: |
       # Convenience rule to get the number of replicas for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
       sum by (cluster, namespace, deployment) (
         label_replace(
           kube_deployment_spec_replicas,
-          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
         )
       )
       or
       sum by (cluster, namespace, deployment) (
-        label_replace(kube_statefulset_replicas, \"deployment\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\")
-      )"
+        label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
+      )
     record: "cluster_namespace_deployment:actual_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
         quantile_over_time(0.99,
           sum by (cluster, namespace) (
@@ -324,21 +319,21 @@ groups:
           )[24h:]
         )
         / 240000
-      )"
+      )
     labels:
       deployment: "distributor"
       reason: "sample_rate"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"ingestion_rate\"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
         * 0.59999999999999998 / 240000
-      )"
+      )
     labels:
       deployment: "distributor"
       reason: "sample_rate_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
         quantile_over_time(0.99,
           sum by (cluster, namespace) (
@@ -346,12 +341,12 @@ groups:
           )[24h:]
         )
         * 3 / 80000
-      )"
+      )
     labels:
       deployment: "ingester"
       reason: "sample_rate"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
         quantile_over_time(0.99,
           sum by(cluster, namespace) (
@@ -359,59 +354,59 @@ groups:
           )[24h:]
         )
         / 1500000
-      )"
+      )
     labels:
       deployment: "ingester"
       reason: "active_series"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"max_global_series_per_user\"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
         * 3 * 0.59999999999999998 / 1500000
-      )"
+      )
     labels:
       deployment: "ingester"
       reason: "active_series_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"ingestion_rate\"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
         * 0.59999999999999998 / 80000
-      )"
+      )
     labels:
       deployment: "ingester"
       reason: "sample_rate_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       ceil(
         (sum by (cluster, namespace) (
-          cortex_ingester_tsdb_storage_blocks_bytes{job=~\".+/ingester.*\"}
+          cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
         ) / 4)
           /
         avg by (cluster, namespace) (
-          memcached_limit_bytes{job=~\".+/memcached\"}
+          memcached_limit_bytes{job=~".+/memcached"}
         )
-      )"
+      )
     labels:
       deployment: "memcached"
       reason: "active_series"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       sum by (cluster, namespace, deployment) (
         label_replace(
           label_replace(
-            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
-            \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[5m])),
+            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
           ),
-          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
         )
-      )"
+      )
     record: "cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate"
-  - expr: "|
+  - expr: |
       # Convenience rule to get the CPU request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
       # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
       # that remove resource metrics, ref:
       # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
@@ -424,11 +419,11 @@ groups:
           label_replace(
             label_replace(
               kube_pod_container_resource_requests_cpu_cores,
-              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
             ),
-            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
           )
         )
       )
@@ -439,17 +434,17 @@ groups:
         sum by (cluster, namespace, deployment) (
           label_replace(
             label_replace(
-              kube_pod_container_resource_requests{resource=\"cpu\"},
-              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+              kube_pod_container_resource_requests{resource="cpu"},
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
             ),
-            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
           )
         )
-      )"
+      )
     record: "cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum"
-  - expr: "|
+  - expr: |
       # Jobs should be sized to their CPU usage.
       # We do this by comparing 99th percentile usage over the last 24hrs to
       # their current provisioned #replicas and resource requests.
@@ -459,28 +454,28 @@ groups:
         quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
           /
         cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
-      )"
+      )
     labels:
       reason: "cpu_usage"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: "|
+  - expr: |
       # Convenience rule to get the Memory utilization for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
       sum by (cluster, namespace, deployment) (
         label_replace(
           label_replace(
-            container_memory_usage_bytes{image!=\"\"},
-            \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+            container_memory_usage_bytes{image!=""},
+            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
           ),
-          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
         )
-      )"
+      )
     record: "cluster_namespace_deployment:container_memory_usage_bytes:sum"
-  - expr: "|
+  - expr: |
       # Convenience rule to get the Memory request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
       # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
       # that remove resource metrics, ref:
       # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
@@ -493,11 +488,11 @@ groups:
           label_replace(
             label_replace(
               kube_pod_container_resource_requests_memory_bytes,
-              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
             ),
-            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
           )
         )
       )
@@ -508,17 +503,17 @@ groups:
         sum by (cluster, namespace, deployment) (
           label_replace(
             label_replace(
-              kube_pod_container_resource_requests{resource=\"memory\"},
-              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
+              kube_pod_container_resource_requests{resource="memory"},
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
             ),
-            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
           )
         )
-      )"
+      )
     record: "cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum"
-  - expr: "|
+  - expr: |
       # Jobs should be sized to their Memory usage.
       # We do this by comparing 99th percentile usage over the last 24hrs to
       # their current provisioned #replicas and resource requests.
@@ -528,44 +523,33 @@ groups:
         quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
           /
         cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
-      )"
+      )
     labels:
       reason: "memory_usage"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
 - name: "mimir_alertmanager_rules"
   rules:
-  - expr: "|
-      sum by (cluster, job, pod) (cortex_alertmanager_alerts)"
+  - expr: "sum by (cluster, job, pod) (cortex_alertmanager_alerts)"
     record: "cluster_job_pod:cortex_alertmanager_alerts:sum"
-  - expr: "|
-      sum by (cluster, job, pod) (cortex_alertmanager_silences)"
+  - expr: "sum by (cluster, job, pod) (cortex_alertmanager_silences)"
     record: "cluster_job_pod:cortex_alertmanager_silences:sum"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))"
     record: "cluster_job:cortex_alertmanager_alerts_received_total:rate5m"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))"
     record: "cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m"
-  - expr: "|
-      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))"
+  - expr: "sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))"
     record: "cluster_job_integration:cortex_alertmanager_notifications_total:rate5m"
-  - expr: "|
-      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))"
+  - expr: "sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))"
     record: "cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))"
     record: "cluster_job:cortex_alertmanager_state_replication_total:rate5m"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))"
     record: "cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))"
     record: "cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m"
-  - expr: "|
-      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))"
+  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))"
     record: "cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m"
 - name: "mimir_ingester_rules"
   rules:
-  - expr: "|
-      sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))"
+  - expr: "sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[5m]))"
     record: "cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m"

charts/meta-monitoring/src/rules/tempo-rules.yaml

@@ -1,15 +1,15 @@
 groups:
 - name: "tempo_rules"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))"
+  - expr: "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))"
+  - expr: "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:50quantile"
-  - expr: "sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_sum[5m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[5m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:avg"
-  - expr: "sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_sum[5m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_count[5m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_count:sum_rate"

charts/meta-monitoring/templates/_helpers.tpl

@@ -0,0 +1,33 @@
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "ingress.apiVersion" -}}
+  {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}}
+      {{- print "networking.k8s.io/v1" -}}
+  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
+    {{- print "networking.k8s.io/v1beta1" -}}
+  {{- else -}}
+    {{- print "extensions/v1beta1" -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+Return if ingress is stable.
+*/}}
+{{- define "ingress.isStable" -}}
+  {{- eq (include "ingress.apiVersion" .) "networking.k8s.io/v1" -}}
+{{- end -}}
+
+{{/*
+Return if ingress supports ingressClassName.
+*/}}
+{{- define "ingress.supportsIngressClassName" -}}
+  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
+{{- end -}}
+
+{{/*
+Return if ingress supports pathType.
+*/}}
+{{- define "ingress.supportsPathType" -}}
+  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
+{{- end -}}

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -18,10 +18,10 @@
 {{- end }}
 
 {{- define "agent.loki_process_targets" -}}
-{{- if empty .Values.logs.piiRegexes }}
+{{- if and (empty .Values.logs.piiRegexes) (empty .Values.logs.retain) }}
 {{- include "agent.loki_write_targets" . }}
 {{- else }}
-{{- printf "loki.process.PII.receiver" }}
+{{- printf "loki.process.filter.receiver" }}
 {{- end }}
 {{- end }}
 

charts/meta-monitoring/templates/agent/config.yaml

@@ -8,7 +8,7 @@ data:
     discovery.kubernetes "pods" {
       role = "pod"
       namespaces {
-        own_namespace = false
+        own_namespace = true
         names = [ {{ include "agent.namespaces" . }} ]
       }
     }
@@ -33,22 +33,38 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
+        replacement = "{{- .Values.clusterLabelValue -}}"
       }
     }
 
+    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     // Logs
 
-    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
+    remote.kubernetes.secret "logs_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.logs.secret -}}"
+    }
+
     loki.source.kubernetes "pods" {
+      clustering {
+        enabled = true
+      }
       targets    = discovery.relabel.rename_meta_labels.output
       forward_to = [ {{ include "agent.loki_process_targets" . }} ]
     }
 
-    {{- if not (empty .Values.logs.piiRegexes) }}
-    loki.process "PII" {
+    {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.piiRegexes)) }}
+    loki.process "filter" {
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
+      {{- if not (empty .Values.logs.retain) }}
+      stage.match {
+        selector = "{cluster=\"{{- .Values.clusterLabelValue -}}\", namespace=~\"{{- join "|" .Values.namespacesToMonitor -}}|{{- $.Release.Namespace -}}\", pod=~\"loki.*\"} !~ \"{{ join "|" .Values.logs.retain }}\""
+        action   = "drop"
+      }
+      {{- end }}
+
+      {{- if not (empty .Values.logs.piiRegexes) }}
       {{- range .Values.logs.piiRegexes }}
       stage.replace {
         expression = "{{ .expression }}"
@@ -56,26 +72,85 @@ data:
         replace = "{{ .replace }}"
       }
       {{- end }}
+      {{- end }}
     }
     {{- end }}
     {{- end }}
 
+    {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
     // Metrics
 
-    {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
+    remote.kubernetes.secret "metrics_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.metrics.secret -}}"
+    }
+
+    discovery.kubernetes "metric_pods" {
+      role = "pod"
+      namespaces {
+        own_namespace = true
+        names = [ {{ include "agent.namespaces" . }} ]
+      }
+    }
+
+    discovery.relabel "only_http_metrics" {
+      targets = discovery.kubernetes.metric_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterLabelValue -}}"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_port_number"]
+        action = "drop"
+        regex = "9095"
+      }
+    }
+
     prometheus.scrape "pods" {
-      targets    = discovery.relabel.rename_meta_labels.output
+      clustering {
+        enabled = true
+      }
+      targets    = discovery.relabel.only_http_metrics.output
+      forward_to = [ prometheus.relabel.filter.receiver ]
+    }
+
+    prometheus.relabel "filter" {
+      rule {
+        source_labels = ["__name__"]
+        regex = "({{ join "|" .Values.metrics.retain }})"
+        action = "keep"
+      }
+
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
     {{- if .Values.kubeStateMetrics.enabled }}
 
     prometheus.scrape "kubeStateMetrics" {
+      clustering {
+        enabled = true
+      }
       targets    = [ { "__address__" = "{{ .Values.kubeStateMetrics.endpoint }}" } ]
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+      forward_to = [ prometheus.relabel.filter.receiver ]
     }
     {{- end }}
 
-    // cAdvisor and Kubelete metrics
+    // cAdvisor and Kubelet metrics
     // Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
     discovery.kubernetes "all_nodes" {
       role = "node"
@@ -104,15 +179,17 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
+        replacement = "{{- .Values.clusterLabelValue -}}"
       }
     }
 
     prometheus.scrape "cadvisor" {
+      clustering {
+        enabled = true
+      }
       targets    = discovery.relabel.all_nodes.output
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+      forward_to = [ prometheus.relabel.filter.receiver ]
 
-      scrape_interval = "15s"
       metrics_path = "/metrics/cadvisor"
       scheme = "https"
 
@@ -123,10 +200,12 @@ data:
     }
 
     prometheus.scrape "kubelet" {
+      clustering {
+        enabled = true
+      }
       targets    = discovery.relabel.all_nodes.output
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+      forward_to = [ prometheus.relabel.filter.receiver ]
 
-      scrape_interval = "15s"
       metrics_path = "/metrics"
       scheme = "https"
 
@@ -136,18 +215,20 @@ data:
       }
     }
 
-    prometheus.exporter.unix {}
+    prometheus.exporter.unix "promexporter" {}
 
     prometheus.scrape "node_exporter" {
-      targets = prometheus.exporter.unix.targets
+      clustering {
+        enabled = true
+      }
+      targets = prometheus.exporter.unix.promexporter.targets
       forward_to = [prometheus.relabel.node_exporter.receiver]
 
       job_name = "node-exporter"
-      scrape_interval = "15s"
     }
 
     prometheus.relabel "node_exporter" {
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+      forward_to = [ prometheus.relabel.filter.receiver ]
 
       rule {
         replacement = env("HOSTNAME")
@@ -178,14 +259,19 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
+        replacement = "{{- .Values.clusterLabelValue -}}"
       }
     }
     {{- end }}
 
+    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // Traces
 
-    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
+    remote.kubernetes.secret "traces_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.traces.secret -}}"
+    }
+
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {
       // We don't technically need this, but it shows how to change listen address and incoming port.
@@ -254,11 +340,10 @@ data:
     {{- if .Values.cloud.logs.enabled }}
     loki.write "cloud" {
       endpoint {
-        url = "{{- .Values.cloud.logs.endpoint -}}/loki/api/v1/push"
-
+        url = nonsensitive(remote.kubernetes.secret.logs_credentials.data["endpoint"])
         basic_auth {
-          username = "{{- .Values.cloud.logs.username -}}"
-          password = "{{- .Values.cloud.logs.password -}}"
+          username = nonsensitive(remote.kubernetes.secret.logs_credentials.data["username"])
+          password = remote.kubernetes.secret.logs_credentials.data["password"]
         }
       }
     }
@@ -267,11 +352,10 @@ data:
     {{- if .Values.cloud.metrics.enabled }}
     prometheus.remote_write "cloud" {
       endpoint {
-        url = "{{- .Values.cloud.metrics.endpoint -}}/api/prom/push"
-
+        url = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["endpoint"])
         basic_auth {
-          username = "{{- .Values.cloud.metrics.username -}}"
-          password = "{{- .Values.cloud.metrics.password -}}"
+          username = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["username"])
+          password = remote.kubernetes.secret.metrics_credentials.data["password"]
         }
       }
     }
@@ -280,13 +364,13 @@ data:
     {{- if .Values.cloud.traces.enabled }}
     otelcol.exporter.otlp "cloud" {
         client {
-            endpoint = "{{- .Values.cloud.traces.endpoint -}}"
+            endpoint = nonsensitive(remote.kubernetes.secret.traces_credentials.data["endpoint"])
             auth = otelcol.auth.basic.creds.handler
         }
     }
 
     otelcol.auth.basic "creds" {
-      username = "{{- .Values.cloud.traces.username -}}"
-      password = "{{- .Values.cloud.traces.password -}}"
+      username = nonsensitive(remote.kubernetes.secret.traces_credentials.data["username"])
+      password = remote.kubernetes.secret.traces_credentials.data["password"]
     }
     {{- end }}

charts/meta-monitoring/templates/grafana/agent-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.traces.enabled }}
+{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,4 +1,4 @@
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -1,4 +1,4 @@
-{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
+{{- if .Values.local.grafana.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/grafana-ingress.yaml

@@ -0,0 +1,57 @@
+{{- if and .Values.local.grafana.enabled .Values.grafana.ingress.enabled -}}
+{{- $ingressApiIsStable := eq (include "ingress.isStable" .) "true" -}}
+{{- $ingressSupportsIngressClassName := eq (include "ingress.supportsIngressClassName" .) "true" -}}
+{{- $ingressSupportsPathType := eq (include "ingress.supportsPathType" .) "true" -}}
+apiVersion: {{ include "ingress.apiVersion" . }}
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: grafana
+    {{- range $labelKey, $labelValue := .Values.grafana.ingress.labels }}
+    {{ $labelKey }}: {{ $labelValue | toYaml }}
+    {{- end }}
+  {{- with .Values.grafana.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and $ingressSupportsIngressClassName .Values.grafana.ingress.ingressClassName }}
+  ingressClassName: {{ .Values.grafana.ingress.ingressClassName }}
+  {{- end -}}
+  {{- if .Values.grafana.ingress.tls }}
+  tls:
+    {{- range .Values.grafana.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ tpl . $ | quote }}
+        {{- end }}
+      {{- with .secretName }}
+      secretName: {{ . }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.grafana.ingress.hosts }}
+    - host: {{ tpl .host $ | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if $ingressSupportsPathType }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if $ingressApiIsStable }}
+              service:
+                name: grafana
+                port:
+                  number: 3000
+              {{- else }}
+              serviceName: grafana
+              servicePort: 3000
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -1,4 +1,4 @@
-{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
+{{- if .Values.local.grafana.enabled }}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim

charts/meta-monitoring/templates/grafana/loki-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/loki-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-5.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.traces.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/ruler/ruler.yaml

@@ -1,9 +1,9 @@
-{{- if .Values.local.metrics.enabled }}
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if .Values.local.grafana.enabled }}
+{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: meta-mimir-ruler-for-dashboards
+  name: {{ $.Release.Namespace }}-mimir-ruler-for-dashboards
   namespace: {{ $.Release.Namespace }}
 spec:
   progressDeadlineSeconds: 600
@@ -108,11 +108,11 @@ spec:
             items:
               - key: mimir.yaml
                 path: mimir.yaml
-            name: meta-mimir-config
+            name: {{ $.Release.Namespace }}-mimir-config
           name: config
         - configMap:
             defaultMode: 420
-            name: meta-mimir-runtime
+            name: {{ $.Release.Namespace }}-mimir-runtime
           name: runtime-config
         - emptyDir: {}
           name: storage

charts/meta-monitoring/templates/ruler/rules-configmap.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.local.metrics.enabled }}
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/validate.yaml

@@ -3,20 +3,20 @@
 {{- end -}}
 
 {{- if eq .Values.cloud.logs.enabled true -}}
-  {{- if or (empty .Values.cloud.logs.endpoint) (or (empty .Values.cloud.logs.username) (empty .Values.cloud.logs.password)) -}}
-    {{- fail "if cloud.logs is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- if empty .Values.cloud.logs.secret -}}
+    {{- fail "if cloud.logs is enabled then the secret has to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
 {{- if eq .Values.cloud.metrics.enabled true -}}
-  {{- if or (empty .Values.cloud.metrics.endpoint) (or (empty .Values.cloud.metrics.username) (empty .Values.cloud.metrics.password)) -}}
-    {{- fail "if cloud.metrics is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- if empty .Values.cloud.metrics.secret -}}
+    {{- fail "if cloud.metrics is enabled then the secret has to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
 {{- if eq .Values.cloud.traces.enabled true -}}
-  {{- if or (empty .Values.cloud.traces.endpoint) (or (empty .Values.cloud.traces.username) (empty .Values.cloud.traces.password)) -}}
-    {{- fail "if cloud.traces is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- if empty .Values.cloud.traces.secret -}}
+    {{- fail "if cloud.traces is enabled then the secret has to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
@@ -37,3 +37,7 @@
 {{- if empty .Values.namespacesToMonitor -}}
   {{- fail "No namespaces have been specified in namespacesToMonitor" -}}
 {{- end -}}
+
+{{- if empty .Values.metrics.retain -}}
+  {{- fail "All metrics will be collected, please specify some in metrics.retain" -}}
+{{- end -}}

charts/meta-monitoring/values.yaml

@@ -4,28 +4,24 @@ namespacesToMonitor:
 - mimir
 - tempo
 # The name of the cluster where this will be installed
-clusterName: "meta-monitoring"
+clusterLabelValue: "meta-monitoring"
 
 # Set to true to write logs, metrics or traces to Grafana Cloud
 cloud:
   logs:
     enabled: true
-    endpoint: to_be_changed
-    username: to_be_changed
-    password: to_be_changed
+    secret: "logs"
   metrics:
     enabled: true
-    endpoint: to_be_changed
-    username: to_be_changed
-    password: to_be_changed
+    secret: "metrics"
   traces:
     enabled: true
-    endpoint: to_be_changed
-    username: to_be_changed
-    password: to_be_changed
+    secret: "traces"
 
 # Set to true for a local version of logs, metrics or traces
 local:
+  grafana:
+    enabled: false
   logs:
     enabled: false
   metrics:
@@ -35,15 +31,132 @@ local:
   minio:
     enabled: false  # This should be set to true if any of the previous is enabled
 
-# Adding regexes here will add a stage.replace block for logs. For more information see
-# https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
+grafana:
+  # Gateway ingress configuration
+  ingress:
+    # -- Specifies whether an ingress for the gateway should be created
+    enabled: true
+    # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
+    ingressClassName: ""
+    # -- Annotations for the gateway ingress
+    annotations: { }
+    # -- Labels for the gateway ingress
+    labels: { }
+    # -- Hosts configuration for the gateway ingress, passed through the `tpl` function to allow templating
+    hosts:
+      - host: monitoring.example.com
+        paths:
+          - path: /
+            # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
+            # pathType: Prefix
+    # -- TLS configuration for the gateway ingress. Hosts passed through the `tpl` function to allow templating
+    #tls:
+    #  - secretName: grafana-tls
+    #    hosts:
+    #      - monitoring.example.com
+
+
 logs:
+  # Adding regexes here will add a stage.replace block for logs. For more information see
+  # https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
   piiRegexes:
   # This example replaces the word after password with *****
   # - expression: "password (\\\\S+)"
   #   source: ""         # Empty uses the log message
   #   replace: "*****""
 
+  # The lines matching these will be kept in Loki
+  retain:
+  # This shows the queries
+  - caller=metrics.go
+  # This shows any errors
+  - level=error
+  # This shows the ingest requests and is very noisy. Uncomment to include.
+  # - caller=push.go
+  # Log lines for delete requests
+  - delete request for user added
+  - Started processing delete request
+  - delete request for user marked as processed
+
+metrics:
+  # The list of metrics to retain for logging dashboards
+  retain:
+  - agent_config_last_load_success_timestamp_seconds
+  - agent_config_last_load_successful
+  - agent_config_load_failures_total
+  - container_cpu_usage_seconds_total
+  - container_fs_writes_bytes_total
+  - container_memory_working_set_bytes
+  - container_network_receive_bytes_total
+  - container_network_transmit_bytes_total
+  - container_spec_cpu_period
+  - container_spec_cpu_quota
+  - container_spec_memory_limit_bytes
+  - cortex_ingester_flush_queue_length
+  - go_gc_duration_seconds
+  - go_goroutines
+  - go_memstats_heap_inuse_bytes
+  - kubelet_volume_stats_used_bytes
+  - kubelet_volume_stats_capacity_bytes
+  - kube_persistentvolumeclaim_labels
+  - kube_pod_container_resource_requests
+  - kube_pod_container_status_last_terminated_reason
+  - kube_pod_container_status_restarts_total
+  - loki_boltdb_shipper_compact_tables_operation_duration_seconds
+  - loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds
+  - loki_boltdb_shipper_retention_marker_count_total
+  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket
+  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count
+  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum
+  - loki_boltdb_shipper_retention_marker_table_processed_total
+  - loki_boltdb_shipper_request_duration_seconds_bucket
+  - loki_boltdb_shipper_request_duration_seconds_count
+  - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count
+  - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum
+  - loki_boltdb_shipper_retention_sweeper_marker_files_current
+  - loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time
+  - loki_build_info
+  - loki_chunk_store_index_entries_per_chunk_count
+  - loki_chunk_store_index_entries_per_chunk_sum
+  - loki_compactor_delete_requests_processed_total
+  - loki_compactor_delete_requests_received_total
+  - loki_compactor_deleted_lines
+  - loki_compactor_oldest_pending_delete_request_age_seconds
+  - loki_compactor_pending_delete_requests_count
+  - loki_distributor_lines_received_total
+  - loki_ingester_chunk_age_seconds_bucket
+  - loki_ingester_chunk_age_seconds_count
+  - loki_ingester_chunk_age_seconds_sum
+  - loki_ingester_chunk_bounds_hours_bucket
+  - loki_ingester_chunk_bounds_hours_count
+  - loki_ingester_chunk_bounds_hours_sum
+  - loki_ingester_chunk_entries_bucket
+  - loki_ingester_chunk_entries_count
+  - loki_ingester_chunk_entries_sum
+  - loki_ingester_chunk_size_bytes_bucket
+  - loki_ingester_chunk_utilization_bucket
+  - loki_ingester_chunk_utilization_sum
+  - loki_ingester_chunks_flushed_total
+  - loki_ingester_memory_chunks
+  - loki_ingester_memory_streams
+  - loki_request_duration_seconds_count
+  - loki_ruler_wal_appender_ready
+  - loki_ruler_wal_disk_size
+  - loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds
+  - loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds
+  - loki_ruler_wal_prometheus_remote_storage_samples_pending
+  - loki_ruler_wal_prometheus_remote_storage_samples_total
+  - loki_ruler_wal_samples_appended_total
+  - loki_ruler_wal_storage_created_series_total
+  - loki_write_batch_retries_total
+  - loki_write_dropped_bytes_total
+  - loki_write_dropped_entries_total
+  - loki_write_sent_bytes_total
+  - loki_write_sent_entries_total
+  - node_disk_read_bytes_total
+  - node_disk_written_bytes_total
+  - promtail_custom_bad_words_total
+
 # Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
 dashboards:
   logs:
@@ -109,10 +222,26 @@ loki:
 
 grafana-agent:
   agent:
+    clustering:
+      enabled: true
     configMap:
       create: false
       name: "agent-configmap"
       key: 'config.river'
+    resources:
+      requests:
+        cpu: '1000m'
+        memory: '600Mi'
+      limits:
+        memory: '4Gi'
+  controller:
+    type: "statefulset"
+    autoscaling:
+      enabled: true
+      minReplicas: 3
+      maxReplicas: 30
+      targetMemoryUtilizationPercentage: 90
+      targetCPUUtilizationPercentage: 90
 
 mimir-distributed:
   minio:

docs/installation.md

@@ -6,7 +6,26 @@
    kubectl create namespace meta
    ```
 
-1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml).
+1. Create secrets with credentials and the endpoint when sending logs, metrics or traces to Grafana Cloud.
+
+   ```
+   kubectl create secret generic logs -n meta \
+    --from-literal=username=<logs username> \
+    --from-literal=password=<logs password>
+    --from-literal=endpoint='https://logs-prod-us-central1.grafana.net/loki/api/v1/push'
+
+   kubectl create secret generic metrics -n meta \
+    --from-literal=username=<metrics username> \
+    --from-literal=password=<metrics password>
+    --from-literal=endpoint='https://prometheus-us-central1.grafana.net/api/prom/push'
+
+   kubectl create secret generic traces -n meta \
+    --from-literal=username=<traces username> \
+    --from-literal=password=<traces password>
+    --from-literal=endpoint='https://tempo-us-central1.grafana.net/tempo'
+   ```
+
+1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml). Fill in the names of the secrets created above as needed.
 
 1. Install this helm chart