From 54eda36ec3cc7b6af2a2d0de97553972a58e79ad Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Thu, 2 May 2024 13:27:26 +0100 Subject: [PATCH 1/3] Cleanup dashboards we won't ship Signed-off-by: Michel Hollands --- README.md | 4 +- .../templates/grafana/dashboard.yaml | 52 ------------------- .../templates/grafana/grafana.yaml | 38 -------------- .../templates/grafana/mimir-dashboards-1.yaml | 19 ------- .../templates/grafana/mimir-dashboards-2.yaml | 19 ------- .../templates/grafana/mimir-dashboards-3.yaml | 19 ------- .../templates/grafana/mimir-dashboards-4.yaml | 19 ------- .../templates/grafana/mimir-dashboards-5.yaml | 19 ------- .../templates/grafana/tempo-dashboards-1.yaml | 21 -------- 9 files changed, 1 insertion(+), 209 deletions(-) delete mode 100644 charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml delete mode 100644 charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml delete mode 100644 charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml delete mode 100644 charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml delete mode 100644 charts/meta-monitoring/templates/grafana/mimir-dashboards-5.yaml delete mode 100644 charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml diff --git a/README.md b/README.md index a085d41..d0a9cf8 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ In the cloud mode the logs, metrics and/or traces are sent to Grafana Cloud. To enable cloud mode set `cloud..enabled` to true. The `endpoint`, `username` and `password` settings for your Grafana Cloud logs, metrics and traces instances have to be filled in as well. -Both modes can be enabled at the same time. +Both modes can be enabled at the same time. Cloud mode is preferred. ## Installation @@ -33,8 +33,6 @@ For more instructions including how to update the chart go to the [installation] - Specify PII regexes that are applied to logs before they are sent to Loki (cloud or local). The capture group in the regex is replaced with *****. - a Grafana instance is installed (when local mode is used) with the relevant datasources installed. The following dashboards are installed: - logs dashboards - - metrics dashboards - - traces dashboards - agent dashboards - Retention is set to 24 hours diff --git a/charts/meta-monitoring/templates/grafana/dashboard.yaml b/charts/meta-monitoring/templates/grafana/dashboard.yaml index 1a49653..aa63bf3 100644 --- a/charts/meta-monitoring/templates/grafana/dashboard.yaml +++ b/charts/meta-monitoring/templates/grafana/dashboard.yaml @@ -27,58 +27,6 @@ data: path: /var/lib/grafana/dashboards/loki-2 orgId: 1 type: file -{{- end }} -{{- if .Values.dashboards.metrics.enabled }} - - disableDeletion: true - editable: false - folder: Mimir - name: mimir-1 - options: - path: /var/lib/grafana/dashboards/mimir-1 - orgId: 1 - type: file - - disableDeletion: true - editable: false - folder: Mimir - name: mimir-2 - options: - path: /var/lib/grafana/dashboards/mimir-2 - orgId: 1 - type: file - - disableDeletion: true - editable: false - folder: Mimir - name: mimir-3 - options: - path: /var/lib/grafana/dashboards/mimir-3 - orgId: 1 - type: file - - disableDeletion: true - editable: false - folder: Mimir - name: mimir-4 - options: - path: /var/lib/grafana/dashboards/mimir-4 - orgId: 1 - type: file - - disableDeletion: true - editable: false - folder: Mimir - name: mimir-5 - options: - path: /var/lib/grafana/dashboards/mimir-5 - orgId: 1 - type: file -{{- end }} -{{- if .Values.dashboards.traces.enabled }} - - disableDeletion: true - editable: false - folder: Tempo - name: tempo-1 - options: - path: /var/lib/grafana/dashboards/tempo-1 - orgId: 1 - type: file {{- end }} - disableDeletion: true editable: false diff --git a/charts/meta-monitoring/templates/grafana/grafana.yaml b/charts/meta-monitoring/templates/grafana/grafana.yaml index 8687066..e30c61c 100644 --- a/charts/meta-monitoring/templates/grafana/grafana.yaml +++ b/charts/meta-monitoring/templates/grafana/grafana.yaml @@ -75,22 +75,6 @@ spec: - mountPath: /var/lib/grafana/dashboards/loki-2 name: loki-dashboards-2 {{- end }} - {{- if .Values.dashboards.metrics.enabled }} - - mountPath: /var/lib/grafana/dashboards/mimir-1 - name: mimir-dashboards-1 - - mountPath: /var/lib/grafana/dashboards/mimir-2 - name: mimir-dashboards-2 - - mountPath: /var/lib/grafana/dashboards/mimir-3 - name: mimir-dashboards-3 - - mountPath: /var/lib/grafana/dashboards/mimir-4 - name: mimir-dashboards-4 - - mountPath: /var/lib/grafana/dashboards/mimir-5 - name: mimir-dashboards-5 - {{- end }} - {{- if .Values.dashboards.traces.enabled }} - - mountPath: /var/lib/grafana/dashboards/tempo-1 - name: tempo-dashboards-1 - {{- end }} - mountPath: /var/lib/grafana/dashboards/agent-1 name: agent-dashboards-1 volumes: @@ -111,28 +95,6 @@ spec: configMap: name: loki-dashboards-2 {{- end }} - {{- if .Values.dashboards.metrics.enabled }} - - name: mimir-dashboards-1 - configMap: - name: mimir-dashboards-1 - - name: mimir-dashboards-2 - configMap: - name: mimir-dashboards-2 - - name: mimir-dashboards-3 - configMap: - name: mimir-dashboards-3 - - name: mimir-dashboards-4 - configMap: - name: mimir-dashboards-4 - - name: mimir-dashboards-5 - configMap: - name: mimir-dashboards-5 - {{- end }} - {{- if .Values.dashboards.traces.enabled }} - - name: tempo-dashboards-1 - configMap: - name: tempo-dashboards-1 - {{- end }} - name: agent-dashboards-1 configMap: name: agent-dashboards-1 diff --git a/charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml b/charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml deleted file mode 100644 index 202a60e..0000000 --- a/charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mimir-dashboards-1 - namespace: {{ $.Release.Namespace }} -data: - "mimir-alertmanager-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-alertmanager-resources.json" | fromJson | toJson }} - "mimir-alertmanager.json": | - {{ $.Files.Get "src/dashboards/mimir-alertmanager.json" | fromJson | toJson }} - "mimir-compactor-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-compactor-resources.json" | fromJson | toJson }} - "mimir-compactor.json": | - {{ $.Files.Get "src/dashboards/mimir-compactor.json" | fromJson | toJson }} - "mimir-config.json": | - {{ $.Files.Get "src/dashboards/mimir-config.json" | fromJson | toJson }} -{{- end }} \ No newline at end of file diff --git a/charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml b/charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml deleted file mode 100644 index 6a3dcdf..0000000 --- a/charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mimir-dashboards-2 - namespace: {{ $.Release.Namespace }} -data: - "mimir-object-store.json": | - {{ $.Files.Get "src/dashboards/mimir-object-store.json" | fromJson | toJson }} - "mimir-overrides.json": | - {{ $.Files.Get "src/dashboards/mimir-overrides.json" | fromJson | toJson }} - "mimir-overview-networking.json": | - {{ $.Files.Get "src/dashboards/mimir-overview-networking.json" | fromJson | toJson }} - "mimir-overview-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-overview-resources.json" | fromJson | toJson }} - "mimir-overview.json": | - {{ $.Files.Get "src/dashboards/mimir-overview.json" | fromJson | toJson }} -{{- end }} \ No newline at end of file diff --git a/charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml b/charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml deleted file mode 100644 index f2770b3..0000000 --- a/charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mimir-dashboards-3 - namespace: {{ $.Release.Namespace }} -data: - "mimir-queries.json": | - {{ $.Files.Get "src/dashboards/mimir-queries.json" | fromJson | toJson }} - "mimir-reads-networking.json": | - {{ $.Files.Get "src/dashboards/mimir-reads-networking.json" | fromJson | toJson }} - "mimir-reads-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-reads-resources.json" | fromJson | toJson }} - "mimir-reads.json": | - {{ $.Files.Get "src/dashboards/mimir-reads.json" | fromJson | toJson }} - "mimir-remote-ruler-reads-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-remote-ruler-reads-resources.json" | fromJson | toJson }} -{{- end }} \ No newline at end of file diff --git a/charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml b/charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml deleted file mode 100644 index e7a160c..0000000 --- a/charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mimir-dashboards-4 - namespace: {{ $.Release.Namespace }} -data: - "mimir-remote-ruler-reads.json": | - {{ $.Files.Get "src/dashboards/mimir-remote-ruler-reads.json" | fromJson | toJson }} - "mimir-rollout-progress.json": | - {{ $.Files.Get "src/dashboards/mimir-rollout-progress.json" | fromJson | toJson }} - "mimir-ruler.json": | - {{ $.Files.Get "src/dashboards/mimir-ruler.json" | fromJson | toJson }} - "mimir-scaling.json": | - {{ $.Files.Get "src/dashboards/mimir-scaling.json" | fromJson | toJson }} - "mimir-slow-queries.json": | - {{ $.Files.Get "src/dashboards/mimir-slow-queries.json" | fromJson | toJson }} -{{- end }} \ No newline at end of file diff --git a/charts/meta-monitoring/templates/grafana/mimir-dashboards-5.yaml b/charts/meta-monitoring/templates/grafana/mimir-dashboards-5.yaml deleted file mode 100644 index bcaeb7c..0000000 --- a/charts/meta-monitoring/templates/grafana/mimir-dashboards-5.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mimir-dashboards-5 - namespace: {{ $.Release.Namespace }} -data: - "mimir-tenants.json": | - {{ $.Files.Get "src/dashboards/mimir-tenants.json" | fromJson | toJson }} - "mimir-top-tenants.json": | - {{ $.Files.Get "src/dashboards/mimir-top-tenants.json" | fromJson | toJson }} - "mimir-writes-networking.json": | - {{ $.Files.Get "src/dashboards/mimir-writes-networking.json" | fromJson | toJson }} - "mimir-writes-resources.json": | - {{ $.Files.Get "src/dashboards/mimir-writes-resources.json" | fromJson | toJson }} - "mimir-writes.json": | - {{ $.Files.Get "src/dashboards/mimir-writes.json" | fromJson | toJson }} -{{- end }} diff --git a/charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml b/charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml deleted file mode 100644 index 57a8814..0000000 --- a/charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.local.grafana.enabled .Values.dashboards.traces.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: tempo-dashboards-1 - namespace: {{ $.Release.Namespace }} -data: - "tempo-operational.json": | - {{ $.Files.Get "src/dashboards/tempo-operational.json" | fromJson | toJson }} - "tempo-reads.json": | - {{ $.Files.Get "src/dashboards/tempo-reads.json" | fromJson | toJson }} - "tempo-resources.json": | - {{ $.Files.Get "src/dashboards/tempo-resources.json" | fromJson | toJson }} - "tempo-rollout-progress.json": | - {{ $.Files.Get "src/dashboards/tempo-rollout-progress.json" | fromJson | toJson }} - "tempo-tenants.json": | - {{ $.Files.Get "src/dashboards/tempo-tenants.json" | fromJson | toJson }} - "tempo-writes.json": | - {{ $.Files.Get "src/dashboards/tempo-writes.json" | fromJson | toJson }} -{{- end }} \ No newline at end of file From 3c187def4713ffc09eeeb4f41f1bef4139ef93cf Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Thu, 2 May 2024 14:24:09 +0100 Subject: [PATCH 2/3] Fix error in query Signed-off-by: Michel Hollands --- charts/meta-monitoring/src/dashboards/loki-retention.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/meta-monitoring/src/dashboards/loki-retention.json b/charts/meta-monitoring/src/dashboards/loki-retention.json index addbde8..e9336d9 100644 --- a/charts/meta-monitoring/src/dashboards/loki-retention.json +++ b/charts/meta-monitoring/src/dashboards/loki-retention.json @@ -266,7 +266,7 @@ "span": 4, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/\"(.*compactor|(loki|enterprise-logs)-backend.*|loki-single-binary)\"\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(.*compactor|(loki|enterprise-logs)-backend.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null From c70ef27e486e024aba01adf831223ee5ba840466 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Thu, 2 May 2024 14:24:36 +0100 Subject: [PATCH 3/3] Set scrape interval to 1m Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/grafana/datasources.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/meta-monitoring/templates/grafana/datasources.yaml b/charts/meta-monitoring/templates/grafana/datasources.yaml index 285be9a..fe3ff7e 100644 --- a/charts/meta-monitoring/templates/grafana/datasources.yaml +++ b/charts/meta-monitoring/templates/grafana/datasources.yaml @@ -61,6 +61,10 @@ data: # Allows users to edit data sources from the # Grafana UI. editable: true + # Extra config. + jsonData: + # Scrape interval + timeInterval: 1m {{- end }} {{- if .Values.local.traces.enabled }} - name: Tempo