From 3f200115f97a82cc30f73fdac613fc3747ba17dc Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 1 May 2024 10:16:57 +0100 Subject: [PATCH 1/4] Consider the meta monitoring namespace for logs as well Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/agent/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 8822957..498ce69 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -9,7 +9,7 @@ data: role = "pod" namespaces { own_namespace = true - names = [ {{ include "agent.namespaces" . }} ] + names = [ {{ include "agent.all_namespaces" . }} ] } } From 5d553e50f609d234ab90b7a7f30b106a03249775 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 1 May 2024 16:54:51 +0100 Subject: [PATCH 2/4] Add config for Loki traces to OTEL Signed-off-by: Michel Hollands --- .../templates/agent/_helpers-agent.tpl | 2 +- .../templates/agent/config.yaml | 37 +++++++------------ charts/meta-monitoring/values.yaml | 9 +++++ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl index a6f5661..475d4e8 100644 --- a/charts/meta-monitoring/templates/agent/_helpers-agent.tpl +++ b/charts/meta-monitoring/templates/agent/_helpers-agent.tpl @@ -51,7 +51,7 @@ {{- $list = append $list ("otelcol.exporter.otlp.local.input") }} {{- end }} {{- if .Values.cloud.traces.enabled }} -{{- $list = append $list ("otelcol.exporter.otlp.cloud.input") }} +{{- $list = append $list ("otelcol.exporter.otlphttp.cloud.input") }} {{- end }} {{- join ", " $list }} {{- end }} \ No newline at end of file diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 498ce69..b8e51e3 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -9,7 +9,7 @@ data: role = "pod" namespaces { own_namespace = true - names = [ {{ include "agent.all_namespaces" . }} ] + names = [ {{ include "agent.namespaces" . }} ] } } @@ -295,13 +295,23 @@ data: // In this case, the Agent is listening on all available bindable addresses on port 4317 (which is the // default OTLP gRPC port) for the OTLP protocol. grpc { - endpoint = "0.0.0.0:4317" + endpoint = "0.0.0.0:4317" } // We define where to send the output of all ingested traces. In this case, to the OpenTelemetry batch processor // named 'default'. output { - traces = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } + } + + otelcol.receiver.jaeger "jaeger" { + protocols { + thrift_http {} + } + + output { + traces = [otelcol.processor.batch.default.input] } } @@ -335,25 +345,6 @@ data: } {{- end }} - {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }} - // The OpenTelemetry exporter exports processed trace spans to another target that is listening for OTLP format traces. - // A unique label, 'local', is added to uniquely identify this exporter. - otelcol.exporter.otlp "local" { - // Define the client for exporting. - client { - // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC). - endpoint = "meta-tempo-distributor:4317" - // Configure TLS settings for communicating with the endpoint. - tls { - // The connection is insecure. - insecure = true - // Do not verify TLS certificates when connecting. - insecure_skip_verify = true - } - } - } - {{- end }} - {{- if .Values.cloud.logs.enabled }} loki.write "cloud" { endpoint { @@ -379,7 +370,7 @@ data: {{- end }} {{- if .Values.cloud.traces.enabled }} - otelcol.exporter.otlp "cloud" { + otelcol.exporter.otlphttp "cloud" { client { endpoint = nonsensitive(remote.kubernetes.secret.traces_credentials.data["endpoint"]) auth = otelcol.auth.basic.creds.handler diff --git a/charts/meta-monitoring/values.yaml b/charts/meta-monitoring/values.yaml index 89dddee..d01bd05 100644 --- a/charts/meta-monitoring/values.yaml +++ b/charts/meta-monitoring/values.yaml @@ -267,6 +267,15 @@ alloy: memory: '600Mi' limits: memory: '4Gi' + extraPorts: + - name: "otel" + port: 4317 + targetPort: 4317 + protocol: "TCP" + - name: "thrifthttp" + port: 14268 + targetPort: 14268 + protocol: "TCP" controller: type: "statefulset" autoscaling: From 636b6548283628b6348b520ee4b9cda463cbd022 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 1 May 2024 17:20:00 +0100 Subject: [PATCH 3/4] Add docs on how to setup Loki Signed-off-by: Michel Hollands --- docs/installation.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index c671725..15bcb17 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -163,4 +163,23 @@ For each of the dashboard files in charts/meta-monitoring/src/dashboards folder ``` mimirtool rules print --address= --id= --key= - ``` \ No newline at end of file + ``` + +## Configure Loki to send traces + +1. In the Loki config enable tracing: + + ``` + loki: + tracing: + enabled: true + ``` + +1. Add the following environment variables to your Loki binaries. When using the Loki Helm chart these can be added using the `extraEnv` setting for the Loki components. + + 1. JAEGER_ENDPOINT: http address of the mmc-alloy service installed by the meta-monitoring chart, for example "http://mmc-alloy:14268/api/traces" + 1. JAEGER_AGENT_TAGS: extra tags you would like to add to the spans, for example 'cluster="abc",namespace="def"' + 1. JAEGER_SAMPLER_TYPE: the sampling strategy, for example to sample all use 'const' with a value of 1 for the next environment variable + 1. JAEGER_SAMPLER_PARAM: 1 + +1. If Loki is installed in a different namespace you can create an [ExternalName service](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) in Kubernetes to point to the mmc-alloy service in the meta monitoring namespace From a01992194b7cd6cf02a5a745708f0c8aec84915d Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 1 May 2024 17:24:38 +0100 Subject: [PATCH 4/4] Use OpenTelemetry token for traces Signed-off-by: Michel Hollands --- docs/installation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 15bcb17..e827d5c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -34,12 +34,12 @@ --from-literal=endpoint='https://prometheus-us-central1.grafana.net/api/prom/push' kubectl create secret generic traces -n meta \ - --from-literal=username= \ + --from-literal=username= \ --from-literal=password= - --from-literal=endpoint='https://tempo-us-central1.grafana.net/tempo' + --from-literal=endpoint='https://otlp-gateway-prod-us-east-0.grafana.net/otlp' ``` - The logs, metrics and traces usernames are the `User / Username / Instance IDs` of the Loki, Prometheus/Mimir and Tempo instances in Grafana Cloud. From `Home` in Grafana click on `Stacks`. Then go to the `Details` pages of Loki, Prometheus/Mimir and Tempo. + The logs, metrics and traces usernames are the `User / Username / Instance IDs` of the Loki, Prometheus/Mimir and OpenTelemetry instances in Grafana Cloud. From `Home` in Grafana click on `Stacks`. Then go to the `Details` pages of Loki and Prometheus/Mimir. For OpenTelemetry go to the `Configure` page. 1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml). Fill in the names of the secrets created above as needed. An example minimal values.yaml looks like this: