Merge pull request #4 from grafana/add_regexes_to_filter_log_lines

Add regexes to filter log lines

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -17,6 +17,14 @@
 {{- join ", " $list }}
 {{- end }}
 
+{{- define "agent.loki_process_targets" -}}
+{{- if empty .Values.logs.piiRegexes }}
+{{- include "agent.loki_write_targets" . }}
+{{- else }}
+{{- printf "loki.process.PII.receiver" }}
+{{- end }}
+{{- end }}
+
 {{- define "agent.prometheus_write_targets" -}}
 {{- $list := list }}
 {{- if .Values.local.metrics.enabled }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -40,8 +40,22 @@ data:
     {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     loki.source.kubernetes "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
-      forward_to = [ {{ include "agent.loki_write_targets" . }} ]
+      forward_to = [ {{ include "agent.loki_process_targets" . }} ]
     }
+
+    {{- if not (empty .Values.logs.piiRegexes) }}
+    loki.process "PII" {
+      forward_to = [ {{ include "agent.loki_write_targets" . }} ]
+
+      {{- range .Values.logs.piiRegexes }}
+      stage.replace {
+        expression = "{{ .expression }}"
+        source = "{{ .source }}"
+        replace = "{{ .replace }}"
+      }
+      {{- end }}
+    }
+    {{- end }}
     {{- end }}
 
     {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}

charts/meta-monitoring/values.yaml

@@ -32,6 +32,15 @@ cloud:
     username:
     password:
 
+# Adding regexes here will add a stage.replace block. For more information see
+# https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
+logs:
+  piiRegexes:
+  # This example replaces the word after password with *****
+  # - expression: "password (\\\\S+)"
+  #   source: ""         # Empty uses the log message
+  #   replace: "*****""
+
 global:
   minio:
     rootUser: "rootuser"