Add tempo

Signed-off-by: Michel Hollands <michel.hollands@grafana.com>

charts/meta-monitoring/Chart.lock

@@ -8,8 +8,11 @@ dependencies:
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
   version: 4.4.1
+- name: tempo-distributed
+  repository: https://grafana.github.io/helm-charts
+  version: 1.4.7
 - name: minio
   repository: https://charts.min.io
   version: 5.0.11
-digest: sha256:a010b5c0c7e82b65bee8c6b4b039b6bdcb294f8bd545afe6462221693a487291
+digest: sha256:4b04084e6fe821c4d481017b2430f7c8cd782a5d60830dd3a24eb8f10a9ece09
-generated: "2023-06-28T10:37:38.589479+01:00"
+generated: "2023-06-29T14:25:07.247853+01:00"

charts/meta-monitoring/Chart.yaml

@@ -35,6 +35,10 @@ dependencies:
   repository:  https://grafana.github.io/helm-charts
   version: "4.4.1"
   condition: local.enabled
+- name: tempo-distributed
+  repository:  https://grafana.github.io/helm-charts
+  version: "1.4.7"
+  condition: local.enabled
 - name: minio
   repository: https://charts.min.io
   version: "5.0.11"

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -26,4 +26,15 @@
 {{- $list = append $list ("prometheus.remote_write.cloud.receiver") }}
 {{- end }}
 {{- join ", " $list }}
+{{- end }}
+
+{{- define "agent.tempo_write_targets" -}}
+{{- $list := list }}
+{{- if .Values.local.enabled }}
+{{- $list = append $list ("otelcol.exporter.otlp.local.input") }}
+{{- end }}
+{{- if .Values.cloud.enabled }}
+{{- $list = append $list ("otelcol.exporter.otlp.cloud.input") }}
+{{- end }}
+{{- join ", " $list }}
 {{- end }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -47,6 +47,34 @@ data:
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
 
+    // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
+    otelcol.receiver.otlp "otlp_receiver" {
+      // We don't technically need this, but it shows how to change listen address and incoming port.
+      // In this case, the Agent is listening on all available bindable addresses on port 4317 (which is the
+      // default OTLP gRPC port) for the OTLP protocol.
+      grpc {
+          endpoint = "0.0.0.0:4317"
+      }
+
+      // We define where to send the output of all ingested traces. In this case, to the OpenTelemetry batch processor
+      // named 'default'.
+      output {
+          traces = [otelcol.processor.batch.default.input]
+      }
+    }
+
+    // The OpenTelemetry batch processor collects trace spans until a batch size or timeout is met, before sending those
+    // spans onto another target. This processor is labeled 'default'.
+    otelcol.processor.batch "default" {
+        // Wait until we've received 16K of data.
+        send_batch_size = 16384
+        // Or until 2 seconds have elapsed.
+        timeout = "2s"
+        // When the Agent has enough batched data, send it to the OpenTelemetry exporter named 'local'.
+        output {
+            traces = [ {{ include "agent.tempo_write_targets" . }} ]
+        }
+    }
 
     {{- if .Values.local.enabled }}
     loki.write "local" {
@@ -60,6 +88,23 @@ data:
         url = "http://{{- .Release.Name -}}-mimir-nginx.{{- .Release.Namespace -}}.svc:80/api/v1/push"
       }
     }
+
+    // The OpenTelemetry exporter exports processed trace spans to another target that is listening for OTLP format traces.
+    // A unique label, 'local', is added to uniquely identify this exporter.
+    otelcol.exporter.otlp "local" {
+        // Define the client for exporting.
+        client {
+            // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC).
+            endpoint = "meta-tempo-distributor:4317"
+            // Configure TLS settings for communicating with the endpoint.
+            tls {
+                // The connection is insecure.
+                insecure = true
+                // Do not verify TLS certificates when connecting.
+                insecure_skip_verify = true
+            }
+        }
+    }
     {{- end }}
 
     {{- if .Values.cloud.enabled }}
@@ -84,4 +129,16 @@ data:
         }
       }
     }
+
+    otelcol.exporter.otlp "cloud" {
+        client {
+            endpoint = "{{- .Values.cloud.traces.endpoint -}}"
+            auth = otelcol.auth.basic.creds.handler
+        }
+    }
+
+    otelcol.auth.basic "creds" {
+      username = "{{- .Values.cloud.traces.username -}}"
+      password = "{{- .Values.cloud.traces.password -}}"
+    }
     {{- end }}

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -58,4 +58,23 @@ data:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
         editable: true
+      - name: Tempo
+        # <string, required> Sets the data source type.
+        type: tempo
+        # <int> Sets the organization id. Defaults to orgId 1.
+        orgId: 1
+        # <string> Sets a custom UID to reference this
+        # data source in other parts of the configuration.
+        # If not specified, Grafana generates one.
+        uid: tempo_ds
+        # <string> Sets the data source's URL, including the
+        # port.
+        url: http://{{- $.Release.Name -}}-tempo-query-frontend.{{- $.Release.Namespace -}}.svc:3100
+        # <bool> Toggles whether the data source is pre-selected
+        # for new panels. You can set only one default
+        # data source per organization.
+        isDefault:
+        # <bool> Allows users to edit data sources from the
+        # Grafana UI.
+        editable: true
 {{- end }}

charts/meta-monitoring/values.yaml

@@ -15,6 +15,10 @@ cloud:
     endpoint:
     username:
     password:
+  traces:
+    endpoint:
+    username:
+    password:
 
 global:
   minio:
@@ -27,8 +31,6 @@ global:
 loki:
   loki:
     auth_enabled: false
-    commonConfig:
-      replication_factor: 1
     storage:
       type: "s3"
       s3:
@@ -90,6 +92,25 @@ mimir-distributed:
           secret_access_key: "{{ .Values.global.minio.rootPassword }}"
           insecure: true
 
+tempo-distributed:
+  tempo:
+    structuredConfig:
+      storage:
+        trace:
+          backend: s3
+          s3:
+            bucket: tempo
+            endpoint: "{{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000"
+            access_key: "{{ .Values.global.minio.rootUser }}"
+            secret_key: "{{ .Values.global.minio.rootPassword }}"
+            insecure: true
+  traces:
+    otlp:
+      http:
+        enabled: true
+      grpc:
+        enabled: true
+
 minio:
   rootUser: rootuser
   rootPassword: rootpassword