Merge pull request #89 from grafana/add_extra_metrics

Add way to gather extra metrics and logs

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -54,4 +54,26 @@
 {{- $list = append $list ("otelcol.exporter.otlphttp.cloud.input") }}
 {{- end }}
 {{- join ", " $list }}
+{{- end }}
+
+{{- define "agent.all_logs" -}}
+{{- $list := list }}
+{{- range .Values.logs.retain }}
+{{- $list = append $list . }}
+{{- end }}
+{{- range .Values.logs.extraLogs }}
+{{- $list = append $list . }}
+{{- end }}
+{{- join "|" $list }}
+{{- end }}
+
+{{- define "agent.all_metrics" -}}
+{{- $list := list }}
+{{- range .Values.metrics.retain }}
+{{- $list = append $list . }}
+{{- end }}
+{{- range .Values.metrics.extraMetrics }}
+{{- $list = append $list . }}
+{{- end }}
+{{- join "|" $list }}
 {{- end }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -59,9 +59,9 @@ data:
     loki.process "filter" {
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
-      {{- if not (empty .Values.logs.retain) }}
+      {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.extraLogs)) }}
       stage.match {
-        selector = "{cluster=\"{{- .Values.clusterLabelValue -}}\", namespace=~\"{{- join "|" .Values.namespacesToMonitor -}}|{{- $.Release.Namespace -}}\", pod=~\"loki.*\"} !~ \"{{ join "|" .Values.logs.retain }}\""
+        selector = "{cluster=\"{{- .Values.clusterLabelValue -}}\", namespace=~\"{{- join "|" .Values.namespacesToMonitor -}}|{{- $.Release.Namespace -}}\", pod=~\"loki.*\"} !~ \"{{ include "agent.all_logs" . }}\""
         action   = "drop"
       }
       {{- end }}
@@ -137,7 +137,7 @@ data:
     prometheus.relabel "filter" {
       rule {
         source_labels = ["__name__"]
-        regex = "({{ join "|" .Values.metrics.retain }})"
+        regex = "({{ include "agent.all_metrics" . }})"
         action = "keep"
       }
 

charts/meta-monitoring/values.yaml

@@ -70,12 +70,14 @@ logs:
   - caller=metrics.go
   # This shows any errors
   - level=error
-  # This shows the ingest requests and is very noisy. Uncomment to include.
-  # - caller=push.go
   # Log lines for delete requests
   - delete request for user added
   - Started processing delete request
   - delete request for user marked as processed
+  # This shows the ingest requests and is very noisy. Uncomment to include.
+  # - caller=push.go
+  # Additional log lines to retain
+  extraLogs: []
 
 metrics:
   # The list of metrics to retain for logging dashboards
@@ -175,6 +177,8 @@ metrics:
   - node_disk_read_bytes_total
   - node_disk_written_bytes_total
   - promtail_custom_bad_words_total
+  # Additional metrics to retain
+  extraMetrics: []
 
 # Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
 dashboards: