Add agent dashboards

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

charts/meta-monitoring/src/dashboards/agent.json

@@ -0,0 +1,786 @@
+{
+   "annotations": {
+      "list": [ ]
+   },
+   "editable": true,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "links": [ ],
+   "refresh": "30s",
+   "rows": [
+      {
+         "collapse": false,
+         "height": "250px",
+         "panels": [
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 1,
+               "id": 1,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 1,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 12,
+               "stack": false,
+               "steppedLine": false,
+               "styles": [
+                  {
+                     "alias": "Time",
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "pattern": "Time",
+                     "type": "hidden"
+                  },
+                  {
+                     "alias": "Count",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "link": false,
+                     "linkTargetBlank": false,
+                     "linkTooltip": "Drill down",
+                     "linkUrl": "",
+                     "pattern": "Value #A",
+                     "thresholds": [ ],
+                     "type": "hidden",
+                     "unit": "short"
+                  },
+                  {
+                     "alias": "Uptime",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "link": false,
+                     "linkTargetBlank": false,
+                     "linkTooltip": "Drill down",
+                     "linkUrl": "",
+                     "pattern": "Value #B",
+                     "thresholds": [ ],
+                     "type": "number",
+                     "unit": "short"
+                  },
+                  {
+                     "alias": "Container",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "link": false,
+                     "linkTargetBlank": false,
+                     "linkTooltip": "Drill down",
+                     "linkUrl": "",
+                     "pattern": "container",
+                     "thresholds": [ ],
+                     "type": "number",
+                     "unit": "short"
+                  },
+                  {
+                     "alias": "Pod",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "link": false,
+                     "linkTargetBlank": false,
+                     "linkTooltip": "Drill down",
+                     "linkUrl": "",
+                     "pattern": "pod",
+                     "thresholds": [ ],
+                     "type": "number",
+                     "unit": "short"
+                  },
+                  {
+                     "alias": "Version",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "link": false,
+                     "linkTargetBlank": false,
+                     "linkTooltip": "Drill down",
+                     "linkUrl": "",
+                     "pattern": "version",
+                     "thresholds": [ ],
+                     "type": "number",
+                     "unit": "short"
+                  },
+                  {
+                     "alias": "",
+                     "colorMode": null,
+                     "colors": [ ],
+                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
+                     "decimals": 2,
+                     "pattern": "/.*/",
+                     "thresholds": [ ],
+                     "type": "string",
+                     "unit": "short"
+                  }
+               ],
+               "targets": [
+                  {
+                     "expr": "count by (pod, container, version) (agent_build_info{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"})",
+                     "format": "table",
+                     "instant": true,
+                     "intervalFactor": 2,
+                     "legendFormat": "",
+                     "refId": "A",
+                     "step": 10
+                  },
+                  {
+                     "expr": "max by (pod, container) (time() - process_start_time_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"})",
+                     "format": "table",
+                     "instant": true,
+                     "intervalFactor": 2,
+                     "legendFormat": "",
+                     "refId": "B",
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Agent Stats",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "transform": "table",
+               "type": "table",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Agent Stats",
+         "titleSize": "h6"
+      },
+      {
+         "collapse": false,
+         "height": "250px",
+         "panels": [
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 1,
+               "id": 2,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 1,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 6,
+               "stack": false,
+               "steppedLine": false,
+               "targets": [
+                  {
+                     "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])) by (pod, scrape_job) * 1e3",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "{{pod}}/{{scrape_job}}",
+                     "legendLink": null,
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Target Sync",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "type": "graph",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "ms",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            },
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 10,
+               "id": 3,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 0,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 6,
+               "stack": true,
+               "steppedLine": false,
+               "targets": [
+                  {
+                     "expr": "sum by (pod) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"})",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "{{pod}}",
+                     "legendLink": null,
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Targets",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "type": "graph",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Prometheus Discovery",
+         "titleSize": "h6"
+      },
+      {
+         "collapse": false,
+         "height": "250px",
+         "panels": [
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 1,
+               "id": 4,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 1,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 4,
+               "stack": false,
+               "steppedLine": false,
+               "targets": [
+                  {
+                     "expr": "rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n/\nrate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n* 1e3\n",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "{{pod}} {{interval}} configured",
+                     "legendLink": null,
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Average Scrape Interval Duration",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "type": "graph",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "ms",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            },
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 10,
+               "id": 5,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 0,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 4,
+               "stack": true,
+               "steppedLine": false,
+               "targets": [
+                  {
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "exceeded sample limit: {{job}}",
+                     "legendLink": null,
+                     "step": 10
+                  },
+                  {
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "duplicate timestamp: {{job}}",
+                     "legendLink": null,
+                     "step": 10
+                  },
+                  {
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "out of bounds: {{job}}",
+                     "legendLink": null,
+                     "step": 10
+                  },
+                  {
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "out of order: {{job}}",
+                     "legendLink": null,
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Scrape failures",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "type": "graph",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            },
+            {
+               "aliasColors": { },
+               "bars": false,
+               "dashLength": 10,
+               "dashes": false,
+               "datasource": "$datasource",
+               "fill": 10,
+               "id": 6,
+               "legend": {
+                  "avg": false,
+                  "current": false,
+                  "max": false,
+                  "min": false,
+                  "show": true,
+                  "total": false,
+                  "values": false
+               },
+               "lines": true,
+               "linewidth": 0,
+               "links": [ ],
+               "nullPointMode": "null as zero",
+               "percentage": false,
+               "pointradius": 5,
+               "points": false,
+               "renderer": "flot",
+               "seriesOverrides": [ ],
+               "spaceLength": 10,
+               "span": 4,
+               "stack": true,
+               "steppedLine": false,
+               "targets": [
+                  {
+                     "expr": "sum by (job, instance_group_name) (rate(agent_wal_samples_appended_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))",
+                     "format": "time_series",
+                     "intervalFactor": 2,
+                     "legendFormat": "{{job}} {{instance_group_name}}",
+                     "legendLink": null,
+                     "step": 10
+                  }
+               ],
+               "thresholds": [ ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Appended Samples",
+               "tooltip": {
+                  "shared": true,
+                  "sort": 2,
+                  "value_type": "individual"
+               },
+               "type": "graph",
+               "xaxis": {
+                  "buckets": null,
+                  "mode": "time",
+                  "name": null,
+                  "show": true,
+                  "values": [ ]
+               },
+               "yaxes": [
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": 0,
+                     "show": true
+                  },
+                  {
+                     "format": "short",
+                     "label": null,
+                     "logBase": 1,
+                     "max": null,
+                     "min": null,
+                     "show": false
+                  }
+               ]
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Prometheus Retrieval",
+         "titleSize": "h6"
+      }
+   ],
+   "schemaVersion": 14,
+   "style": "dark",
+   "tags": [
+      "grafana-agent-mixin"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data Source",
+            "name": "datasource",
+            "options": [ ],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "allValue": ".+",
+            "current": {
+               "selected": true,
+               "text": "All",
+               "value": "$__all"
+            },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": "cluster",
+            "multi": true,
+            "name": "cluster",
+            "options": [ ],
+            "query": "label_values(agent_build_info, cluster)",
+            "refresh": 1,
+            "regex": "",
+            "sort": 2,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         },
+         {
+            "allValue": ".+",
+            "current": {
+               "selected": true,
+               "text": "All",
+               "value": "$__all"
+            },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": "namespace",
+            "multi": true,
+            "name": "namespace",
+            "options": [ ],
+            "query": "label_values(agent_build_info, namespace)",
+            "refresh": 1,
+            "regex": "",
+            "sort": 2,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         },
+         {
+            "allValue": ".+",
+            "current": {
+               "selected": true,
+               "text": "All",
+               "value": "$__all"
+            },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": "container",
+            "multi": true,
+            "name": "container",
+            "options": [ ],
+            "query": "label_values(agent_build_info, container)",
+            "refresh": 1,
+            "regex": "",
+            "sort": 2,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         },
+         {
+            "allValue": "grafana-agent-.*",
+            "current": {
+               "selected": true,
+               "text": "All",
+               "value": "$__all"
+            },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": "pod",
+            "multi": true,
+            "name": "pod",
+            "options": [ ],
+            "query": "label_values(agent_build_info{container=~\"$container\"}, pod)",
+            "refresh": 1,
+            "regex": "",
+            "sort": 2,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "",
+   "title": "Agent",
+   "uid": "",
+   "version": 0
+}

charts/meta-monitoring/src/rules/loki-rules.yaml

@@ -0,0 +1,53 @@
+groups:
+- name: loki_rules
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:loki_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:loki_request_duration_seconds:50quantile
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[1m]))
+      by (cluster, job)
+    record: cluster_job:loki_request_duration_seconds:avg
+  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job)
+    record: cluster_job:loki_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job)
+    record: cluster_job:loki_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job)
+    record: cluster_job:loki_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:loki_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:loki_request_duration_seconds:50quantile
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
+      / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
+    record: cluster_job_route:loki_request_duration_seconds:avg
+  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job,
+      route)
+    record: cluster_job_route:loki_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
+    record: cluster_job_route:loki_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
+    record: cluster_job_route:loki_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:loki_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:loki_request_duration_seconds:50quantile
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
+      job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster,
+      namespace, job, route)
+    record: cluster_namespace_job_route:loki_request_duration_seconds:avg
+  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate

charts/meta-monitoring/src/rules/mimir-rules.yaml

@@ -0,0 +1,571 @@
+groups:
+- name: mimir_api_1
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
+      by (cluster, job)
+    record: cluster_job:cortex_request_duration_seconds:avg
+  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
+    record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+- name: mimir_api_2
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:cortex_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:cortex_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
+      / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
+    record: cluster_job_route:cortex_request_duration_seconds:avg
+  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
+      route)
+    record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
+    record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
+    record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
+- name: mimir_api_3
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
+      job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
+      namespace, job, route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
+  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
+      job, route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
+- name: mimir_querier_api
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_querier_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_querier_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_querier_request_duration_seconds:avg
+  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job)
+    record: cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_querier_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:cortex_querier_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, route))
+    record: cluster_job_route:cortex_querier_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
+      (cluster, job, route)
+    record: cluster_job_route:cortex_querier_request_duration_seconds:avg
+  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job, route)
+    record: cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      job, route)
+    record: cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+      job, route)
+    record: cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+      by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
+      by (cluster, namespace, job, route)
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg
+  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+      namespace, job, route)
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+      namespace, job, route)
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+      namespace, job, route)
+    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate
+- name: mimir_cache
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, method))
+    record: cluster_job_method:cortex_memcache_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, method))
+    record: cluster_job_method:cortex_memcache_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
+      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
+      by (cluster, job, method)
+    record: cluster_job_method:cortex_memcache_request_duration_seconds:avg
+  - expr: sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job, method)
+    record: cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
+      job, method)
+    record: cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
+      job, method)
+    record: cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_cache_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_cache_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
+      / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_cache_request_duration_seconds:avg
+  - expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job)
+    record: cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_cache_request_duration_seconds_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, method))
+    record: cluster_job_method:cortex_cache_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job, method))
+    record: cluster_job_method:cortex_cache_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
+      method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+      job, method)
+    record: cluster_job_method:cortex_cache_request_duration_seconds:avg
+  - expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job, method)
+    record: cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
+      method)
+    record: cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
+      job, method)
+    record: cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate
+- name: mimir_storage
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_kv_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_kv_request_duration_seconds:50quantile
+  - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
+      / sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_kv_request_duration_seconds:avg
+  - expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
+      job)
+    record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
+- name: mimir_queries
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_query_frontend_retries:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_query_frontend_retries:50quantile
+  - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
+      by (cluster, job)
+    record: cluster_job:cortex_query_frontend_retries:avg
+  - expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
+    record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
+  - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
+  - expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_query_frontend_retries_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
+  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
+      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
+      (cluster, job)
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
+  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
+      cluster, job)
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
+  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
+      job)
+    record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
+- name: mimir_ingester_queries
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_series:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_series:50quantile
+  - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
+      by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_series:avg
+  - expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
+    record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_series_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_samples:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_samples:50quantile
+  - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
+      by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_samples:avg
+  - expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
+    record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
+  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_exemplars:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+      by (le, cluster, job))
+    record: cluster_job:cortex_ingester_queried_exemplars:50quantile
+  - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
+      sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_exemplars:avg
+  - expr: sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
+      job)
+    record: cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate
+  - expr: sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
+    record: cluster_job:cortex_ingester_queried_exemplars_count:sum_rate
+- name: mimir_received_samples
+  rules:
+  - expr: |
+      sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
+    record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
+- name: mimir_exemplars_in
+  rules:
+  - expr: |
+      sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))
+    record: cluster_namespace_job:cortex_distributor_exemplars_in:rate5m
+- name: mimir_received_exemplars
+  rules:
+  - expr: |
+      sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))
+    record: cluster_namespace_job:cortex_distributor_received_exemplars:rate5m
+- name: mimir_exemplars_ingested
+  rules:
+  - expr: |
+      sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))
+    record: cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m
+- name: mimir_exemplars_appended
+  rules:
+  - expr: |
+      sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))
+    record: cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m
+- name: mimir_scaling_rules
+  rules:
+  - expr: |
+      # Convenience rule to get the number of replicas for both a deployment and a statefulset.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      sum by (cluster, namespace, deployment) (
+        label_replace(
+          kube_deployment_spec_replicas,
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # always matches everything and the (optional) zone is not removed.
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+        )
+      )
+      or
+      sum by (cluster, namespace, deployment) (
+        label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
+      )
+    record: cluster_namespace_deployment:actual_replicas:count
+  - expr: |
+      ceil(
+        quantile_over_time(0.99,
+          sum by (cluster, namespace) (
+            cluster_namespace_job:cortex_distributor_received_samples:rate5m
+          )[24h:]
+        )
+        / 240000
+      )
+    labels:
+      deployment: distributor
+      reason: sample_rate
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+        * 0.59999999999999998 / 240000
+      )
+    labels:
+      deployment: distributor
+      reason: sample_rate_limits
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        quantile_over_time(0.99,
+          sum by (cluster, namespace) (
+            cluster_namespace_job:cortex_distributor_received_samples:rate5m
+          )[24h:]
+        )
+        * 3 / 80000
+      )
+    labels:
+      deployment: ingester
+      reason: sample_rate
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        quantile_over_time(0.99,
+          sum by(cluster, namespace) (
+            cortex_ingester_memory_series
+          )[24h:]
+        )
+        / 1500000
+      )
+    labels:
+      deployment: ingester
+      reason: active_series
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
+        * 3 * 0.59999999999999998 / 1500000
+      )
+    labels:
+      deployment: ingester
+      reason: active_series_limits
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+        * 0.59999999999999998 / 80000
+      )
+    labels:
+      deployment: ingester
+      reason: sample_rate_limits
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      ceil(
+        (sum by (cluster, namespace) (
+          cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
+        ) / 4)
+          /
+        avg by (cluster, namespace) (
+          memcached_limit_bytes{job=~".+/memcached"}
+        )
+      )
+    labels:
+      deployment: memcached
+      reason: active_series
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      sum by (cluster, namespace, deployment) (
+        label_replace(
+          label_replace(
+            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
+            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+          ),
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # always matches everything and the (optional) zone is not removed.
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+        )
+      )
+    record: cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate
+  - expr: |
+      # Convenience rule to get the CPU request for both a deployment and a statefulset.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
+      # that remove resource metrics, ref:
+      # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
+      # - https://github.com/kubernetes/kube-state-metrics/pull/1004
+      #
+      # This is the old expression, compatible with kube-state-metrics < v2.0.0,
+      # where kube_pod_container_resource_requests_cpu_cores was removed:
+      (
+        sum by (cluster, namespace, deployment) (
+          label_replace(
+            label_replace(
+              kube_pod_container_resource_requests_cpu_cores,
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            ),
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # always matches everything and the (optional) zone is not removed.
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          )
+        )
+      )
+      or
+      # This expression is compatible with kube-state-metrics >= v1.4.0,
+      # where kube_pod_container_resource_requests was introduced.
+      (
+        sum by (cluster, namespace, deployment) (
+          label_replace(
+            label_replace(
+              kube_pod_container_resource_requests{resource="cpu"},
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            ),
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # always matches everything and the (optional) zone is not removed.
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          )
+        )
+      )
+    record: cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
+  - expr: |
+      # Jobs should be sized to their CPU usage.
+      # We do this by comparing 99th percentile usage over the last 24hrs to
+      # their current provisioned #replicas and resource requests.
+      ceil(
+        cluster_namespace_deployment:actual_replicas:count
+          *
+        quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
+          /
+        cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
+      )
+    labels:
+      reason: cpu_usage
+    record: cluster_namespace_deployment_reason:required_replicas:count
+  - expr: |
+      # Convenience rule to get the Memory utilization for both a deployment and a statefulset.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      sum by (cluster, namespace, deployment) (
+        label_replace(
+          label_replace(
+            container_memory_usage_bytes{image!=""},
+            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+          ),
+          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # always matches everything and the (optional) zone is not removed.
+          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+        )
+      )
+    record: cluster_namespace_deployment:container_memory_usage_bytes:sum
+  - expr: |
+      # Convenience rule to get the Memory request for both a deployment and a statefulset.
+      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
+      # that remove resource metrics, ref:
+      # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
+      # - https://github.com/kubernetes/kube-state-metrics/pull/1004
+      #
+      # This is the old expression, compatible with kube-state-metrics < v2.0.0,
+      # where kube_pod_container_resource_requests_memory_bytes was removed:
+      (
+        sum by (cluster, namespace, deployment) (
+          label_replace(
+            label_replace(
+              kube_pod_container_resource_requests_memory_bytes,
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            ),
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # always matches everything and the (optional) zone is not removed.
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          )
+        )
+      )
+      or
+      # This expression is compatible with kube-state-metrics >= v1.4.0,
+      # where kube_pod_container_resource_requests was introduced.
+      (
+        sum by (cluster, namespace, deployment) (
+          label_replace(
+            label_replace(
+              kube_pod_container_resource_requests{resource="memory"},
+              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            ),
+            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # always matches everything and the (optional) zone is not removed.
+            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          )
+        )
+      )
+    record: cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
+  - expr: |
+      # Jobs should be sized to their Memory usage.
+      # We do this by comparing 99th percentile usage over the last 24hrs to
+      # their current provisioned #replicas and resource requests.
+      ceil(
+        cluster_namespace_deployment:actual_replicas:count
+          *
+        quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
+          /
+        cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
+      )
+    labels:
+      reason: memory_usage
+    record: cluster_namespace_deployment_reason:required_replicas:count
+- name: mimir_alertmanager_rules
+  rules:
+  - expr: |
+      sum by (cluster, job, pod) (cortex_alertmanager_alerts)
+    record: cluster_job_pod:cortex_alertmanager_alerts:sum
+  - expr: |
+      sum by (cluster, job, pod) (cortex_alertmanager_silences)
+    record: cluster_job_pod:cortex_alertmanager_silences:sum
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))
+    record: cluster_job:cortex_alertmanager_alerts_received_total:rate5m
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))
+    record: cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m
+  - expr: |
+      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))
+    record: cluster_job_integration:cortex_alertmanager_notifications_total:rate5m
+  - expr: |
+      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))
+    record: cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))
+    record: cluster_job:cortex_alertmanager_state_replication_total:rate5m
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))
+    record: cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))
+    record: cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m
+  - expr: |
+      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))
+    record: cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m
+- name: mimir_ingester_rules
+  rules:
+  - expr: |
+      sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))
+    record: cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m

charts/meta-monitoring/src/rules/tempo-rules.yaml

@@ -0,0 +1,15 @@
+groups:
+- name: tempo_rules
+  rules:
+  - expr: histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:tempo_request_duration_seconds:99quantile
+  - expr: histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
+    record: cluster_namespace_job_route:tempo_request_duration_seconds:50quantile
+  - expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
+    record: cluster_namespace_job_route:tempo_request_duration_seconds:avg
+  - expr: sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route)
+    record: cluster_namespace_job_route:tempo_request_duration_seconds_bucket:sum_rate
+  - expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route)
+    record: cluster_namespace_job_route:tempo_request_duration_seconds_sum:sum_rate
+  - expr: sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
+    record: cluster_namespace_job_route:tempo_request_duration_seconds_count:sum_rate

charts/meta-monitoring/templates/grafana/agent-dashboards-1.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.dashboards.traces.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-dashboards-1
+  namespace: {{ $.Release.Namespace }}
+data:
+  "agent-logs-pipeline.json": |
+    {{ $.Files.Get "src/dashboards/agent-logs-pipeline.json" | fromJson | toJson }}
+  "agent-operational.json": |
+    {{ $.Files.Get "src/dashboards/agent-operational.json" | fromJson | toJson }}
+  "agent-remote-write.json": |
+    {{ $.Files.Get "src/dashboards/agent-remote-write.json" | fromJson | toJson }}
+  "agent-tracing-pipeline.json": |
+    {{ $.Files.Get "src/dashboards/agent-tracing-pipeline.json" | fromJson | toJson }}
+  "agent.json": |
+    {{ $.Files.Get "src/dashboards/agent.json" | fromJson | toJson }}
+{{- end }}

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,16 +1,16 @@
-{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: loki-dashboards-provisioning
+  name: dashboards-provisioning
   namespace: {{ $.Release.Namespace }}
 data:
   dashboards.yaml: |
     ---
     apiVersion: 1
     providers:
-{{- if .Values.local.logs.enabled }}
+{{- if .Values.dashboards.logs.enabled }}
       - disableDeletion: true
         editable: false
         folder: Loki
@@ -28,7 +28,7 @@ data:
         orgId: 1
         type: file
 {{- end }}
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
       - disableDeletion: true
         editable: false
         folder: Mimir
@@ -70,4 +70,22 @@ data:
         orgId: 1
         type: file
 {{- end }}
+{{- if .Values.dashboards.traces.enabled }}
+      - disableDeletion: true
+        editable: false
+        folder: Tempo
+        name: tempo-1
+        options:
+          path: /var/lib/grafana/dashboards/tempo-1
+        orgId: 1
+        type: file
+{{- end }}
+      - disableDeletion: true
+        editable: false
+        folder: Agent
+        name: agent-1
+        options:
+          path: /var/lib/grafana/dashboards/agent-1
+        orgId: 1
+        type: file
 {{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -65,15 +65,17 @@ spec:
               name: grafana-pv
             - mountPath: /etc/grafana/provisioning/datasources
               name: datasources-provisioning
+            {{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
             - mountPath: /etc/grafana/provisioning/dashboards
-              name: loki-dashboards-provisioning
-            {{- if .Values.local.logs.enabled }}
+              name: dashboards-provisioning
+            {{- end }}
+            {{- if .Values.dashboards.logs.enabled }}
             - mountPath: /var/lib/grafana/dashboards/loki-1
               name: loki-dashboards-1
             - mountPath: /var/lib/grafana/dashboards/loki-2
               name: loki-dashboards-2
             {{- end }}
-            {{- if .Values.local.metrics.enabled }}
+            {{- if .Values.dashboards.metrics.enabled }}
             - mountPath: /var/lib/grafana/dashboards/mimir-1
               name: mimir-dashboards-1
             - mountPath: /var/lib/grafana/dashboards/mimir-2
@@ -85,6 +87,12 @@ spec:
             - mountPath: /var/lib/grafana/dashboards/mimir-5
               name: mimir-dashboards-5
             {{- end }}
+            {{- if .Values.dashboards.traces.enabled }}
+            - mountPath: /var/lib/grafana/dashboards/tempo-1
+              name: tempo-dashboards-1
+            {{- end }}
+            - mountPath: /var/lib/grafana/dashboards/agent-1
+              name: agent-dashboards-1
       volumes:
         - name: grafana-pv
           persistentVolumeClaim:
@@ -92,10 +100,10 @@ spec:
         - name: datasources-provisioning
           configMap:
             name: datasources-provisioning
-        {{- if .Values.local.logs.enabled }}
-        - name: loki-dashboards-provisioning
+        - name: dashboards-provisioning
           configMap:
-            name: loki-dashboards-provisioning
+            name: dashboards-provisioning
+        {{- if .Values.dashboards.logs.enabled }}
         - name: loki-dashboards-1
           configMap:
             name: loki-dashboards-1
@@ -103,10 +111,7 @@ spec:
           configMap:
             name: loki-dashboards-2
         {{- end }}
-        {{- if .Values.local.metrics.enabled }}
-        - name: mimir-dashboards-provisioning
-          configMap:
-            name: mimir-dashboards-provisioning
+        {{- if .Values.dashboards.metrics.enabled }}
         - name: mimir-dashboards-1
           configMap:
             name: mimir-dashboards-1
@@ -123,6 +128,14 @@ spec:
           configMap:
             name: mimir-dashboards-5
         {{- end }}
+        {{- if .Values.dashboards.traces.enabled }}
+        - name: tempo-dashboards-1
+          configMap:
+            name: tempo-dashboards-1
+        {{- end }}
+        - name: agent-dashboards-1
+          configMap:
+            name: agent-dashboards-1
 
 ---
 apiVersion: v1
@@ -138,4 +151,4 @@ spec:
     app: grafana
   sessionAffinity: None
   type: ClusterIP  # Make this configurable
-{{- end }}
+{{- end }}

charts/meta-monitoring/templates/grafana/loki-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.logs.enabled }}
+{{- if .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/loki-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.logs.enabled }}
+{{- if .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dahboards-5.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.dashboards.traces.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tempo-dashboards-1
+  namespace: {{ $.Release.Namespace }}
+data:
+  "tempo-operational.json": |
+    {{ $.Files.Get "src/dashboards/tempo-operational.json" | fromJson | toJson }}
+  "tempo-reads.json": |
+    {{ $.Files.Get "src/dashboards/tempo-reads.json" | fromJson | toJson }}
+  "tempo-resources.json": |
+    {{ $.Files.Get "src/dashboards/tempo-resources.json" | fromJson | toJson }}
+  "tempo-rollout-progress.json": |
+    {{ $.Files.Get "src/dashboards/tempo-rollout-progress.json" | fromJson | toJson }}
+  "tempo-tenants.json": |
+    {{ $.Files.Get "src/dashboards/tempo-tenants.json" | fromJson | toJson }}
+  "tempo-writes.json": |
+    {{ $.Files.Get "src/dashboards/tempo-writes.json" | fromJson | toJson }}
+{{- end }}

charts/meta-monitoring/templates/ruler/ruler.yaml

@@ -0,0 +1,126 @@
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: meta-mimir-ruler-for-dashboards
+  namespace: meta
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: ruler-for-dashboards
+      app.kubernetes.io/instance: meta
+      app.kubernetes.io/name: mimir
+  strategy:
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: ruler-for-dashboards
+        app.kubernetes.io/instance: meta
+        app.kubernetes.io/name: mimir
+      namespace: meta
+    spec:
+      containers:
+        - args:
+            - -target=ruler
+            - -log.level=debug
+            - -ruler-storage.backend=local
+            - -ruler-storage.local.directory=/etc/rules
+            - -ruler.ring.prefix=dashboards/
+            - -config.expand-env=true
+            - -config.file=/etc/mimir/mimir.yaml
+          image: grafana/mimir:2.8.0
+          imagePullPolicy: IfNotPresent
+          name: ruler
+          ports:
+            - containerPort: 8080
+              name: http-metrics
+              protocol: TCP
+            - containerPort: 9095
+              name: grpc
+              protocol: TCP
+            - containerPort: 7946
+              name: memberlist
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /ready
+              port: http-metrics
+              scheme: HTTP
+            initialDelaySeconds: 45
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/mimir
+              name: config
+            - mountPath: /var/mimir
+              name: runtime-config
+            - mountPath: /data
+              name: storage
+            - mountPath: /active-query-tracker
+              name: active-queries
+            - mountPath: /etc/rules/anonymous
+              name: rules
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        fsGroup: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        runAsUser: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccount: meta-mimir
+      serviceAccountName: meta-mimir
+      terminationGracePeriodSeconds: 180
+      topologySpreadConstraints:
+        - labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: ruler
+              app.kubernetes.io/instance: meta
+              app.kubernetes.io/name: mimir
+          maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+      volumes:
+        - configMap:
+            defaultMode: 420
+            items:
+              - key: mimir.yaml
+                path: mimir.yaml
+            name: meta-mimir-config
+          name: config
+        - configMap:
+            defaultMode: 420
+            name: meta-mimir-runtime
+          name: runtime-config
+        - emptyDir: {}
+          name: storage
+        - emptyDir: {}
+          name: active-queries
+        - configMap:
+            defaultMode: 420
+            name: rules
+          name: rules
+{{- end }}

charts/meta-monitoring/templates/ruler/rules-configmap.yaml

@@ -0,0 +1,18 @@
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rules
+  namespace: {{ $.Release.Namespace }}
+data:
+{{- if .Values.dashboards.logs.enabled }}
+{{ ($.Files.Glob "src/rules/loki-rules.yaml").AsConfig | indent 2 }}
+{{- end }}
+{{- if .Values.dashboards.metrics.enabled }}
+{{ ($.Files.Glob "src/rules/mimir-rules.yaml").AsConfig | indent 2 }}
+{{- end }}
+{{- if .Values.dashboards.traces.enabled }}
+{{ ($.Files.Glob "src/rules/tempo-rules.yaml").AsConfig | indent 2 }}
+{{- end }}
+{{- end }}

charts/meta-monitoring/values.yaml

@@ -14,7 +14,6 @@ local:
   minio:
     enabled: false  # This should be set to true if any of the previous is enabled
 
-
 cloud:
   logs:
     enabled: true
@@ -41,6 +40,15 @@ logs:
   #   source: ""         # Empty uses the log message
   #   replace: "*****""
 
+# Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
+dashboards:
+  logs:
+    enabled: true
+  metrics:
+    enabled: true
+  traces:
+    enabled: true
+
 global:
   minio:
     rootUser: "rootuser"