From fa2b01708c56bbded6d3da15e76757a334cd75be Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Fri, 7 Jul 2023 14:04:56 +0100 Subject: [PATCH 1/4] Add regexes to filter out PII Signed-off-by: Michel Hollands --- .../templates/agent/_helpers-agent.tpl | 8 ++++++++ .../meta-monitoring/templates/agent/config.yaml | 16 +++++++++++++++- charts/meta-monitoring/values.yaml | 3 +++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl index aa4225a..906217b 100644 --- a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl +++ b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl @@ -17,6 +17,14 @@ {{- join ", " $list }} {{- end }} +{{- define "agent.loki_process_targets" -}} +{{- if empty .Values.logs.piiregexes }} +{{- include "agent.loki_write_targets" . }} +{{- else }} +{{- printf "loki.process.PII.receiver" }} +{{- end }} +{{- end }} + {{- define "agent.prometheus_write_targets" -}} {{- $list := list }} {{- if .Values.local.metrics.enabled }} diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index c73d3e2..36424ee 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -40,8 +40,22 @@ data: {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }} loki.source.kubernetes "pods" { targets = discovery.relabel.rename_meta_labels.output - forward_to = [ {{ include "agent.loki_write_targets" . }} ] + forward_to = [ {{ include "agent.loki_process_targets" . }} ] } + + {{- if not (empty .Values.logs.piiregexes) }} + loki.process "PII" { + forward_to = [ {{ include "agent.loki_write_targets" . }} ] + + {{- range .Values.logs.piiregexes }} + stage.replace { + expression = "{{ . }}" + replace = "*****" + } + {{- end }} + } + {{- end }} + {{- end }} {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} diff --git a/charts/meta-monitoring/values.yaml b/charts/meta-monitoring/values.yaml index 0eea3df..8336126 100644 --- a/charts/meta-monitoring/values.yaml +++ b/charts/meta-monitoring/values.yaml @@ -32,6 +32,9 @@ cloud: username: password: +logs: + PIIregexes: + global: minio: rootUser: "rootuser" From 9f54397e8390a57bdf30b14c8bdf3f693ac0f9c9 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Fri, 7 Jul 2023 14:23:23 +0100 Subject: [PATCH 2/4] Add documentation for the regex Signed-off-by: Michel Hollands --- charts/meta-monitoring/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/meta-monitoring/values.yaml b/charts/meta-monitoring/values.yaml index 8336126..10b578c 100644 --- a/charts/meta-monitoring/values.yaml +++ b/charts/meta-monitoring/values.yaml @@ -32,6 +32,12 @@ cloud: username: password: +# Adding regexes here will add a stage.replace block that parses the log lines using a regular expression +# and replaces the capture group in the regex with "*****". +# For example the regex "password (\\S+)" will replace the line +# we should not be logging the password xyzabc at all +# with: +# we should not be logging the password ***** at all logs: PIIregexes: From 04cf591478e9291bb6dd37b6f85c11b4f91f878d Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Fri, 7 Jul 2023 14:26:26 +0100 Subject: [PATCH 3/4] cleanup and fix Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/agent/config.yaml | 1 - charts/meta-monitoring/values.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 36424ee..b9e9313 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -55,7 +55,6 @@ data: {{- end }} } {{- end }} - {{- end }} {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} diff --git a/charts/meta-monitoring/values.yaml b/charts/meta-monitoring/values.yaml index 10b578c..6289867 100644 --- a/charts/meta-monitoring/values.yaml +++ b/charts/meta-monitoring/values.yaml @@ -39,7 +39,7 @@ cloud: # with: # we should not be logging the password ***** at all logs: - PIIregexes: + piiregexes: global: minio: From 9dd6584bee1b7264ba8e9601c14afecabf7033f0 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Mon, 24 Jul 2023 13:42:55 +0100 Subject: [PATCH 4/4] Add full object with stage replace block Signed-off-by: Michel Hollands --- .../templates/agent/_helpers-agent.tpl | 2 +- charts/meta-monitoring/templates/agent/config.yaml | 9 +++++---- charts/meta-monitoring/values.yaml | 14 +++++++------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl index 906217b..5994c4e 100644 --- a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl +++ b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl @@ -18,7 +18,7 @@ {{- end }} {{- define "agent.loki_process_targets" -}} -{{- if empty .Values.logs.piiregexes }} +{{- if empty .Values.logs.piiRegexes }} {{- include "agent.loki_write_targets" . }} {{- else }} {{- printf "loki.process.PII.receiver" }} diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index b9e9313..34a663d 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -43,14 +43,15 @@ data: forward_to = [ {{ include "agent.loki_process_targets" . }} ] } - {{- if not (empty .Values.logs.piiregexes) }} + {{- if not (empty .Values.logs.piiRegexes) }} loki.process "PII" { forward_to = [ {{ include "agent.loki_write_targets" . }} ] - {{- range .Values.logs.piiregexes }} + {{- range .Values.logs.piiRegexes }} stage.replace { - expression = "{{ . }}" - replace = "*****" + expression = "{{ .expression }}" + source = "{{ .source }}" + replace = "{{ .replace }}" } {{- end }} } diff --git a/charts/meta-monitoring/values.yaml b/charts/meta-monitoring/values.yaml index 6289867..25f48e9 100644 --- a/charts/meta-monitoring/values.yaml +++ b/charts/meta-monitoring/values.yaml @@ -32,14 +32,14 @@ cloud: username: password: -# Adding regexes here will add a stage.replace block that parses the log lines using a regular expression -# and replaces the capture group in the regex with "*****". -# For example the regex "password (\\S+)" will replace the line -# we should not be logging the password xyzabc at all -# with: -# we should not be logging the password ***** at all +# Adding regexes here will add a stage.replace block. For more information see +# https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block logs: - piiregexes: + piiRegexes: + # This example replaces the word after password with ***** + # - expression: "password (\\\\S+)" + # source: "" # Empty uses the log message + # replace: "*****"" global: minio: