Merge pull request #85 from grafana/open_extra_ports_for_otlp

Enabled traces from Loki

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -51,7 +51,7 @@
 {{- $list = append $list ("otelcol.exporter.otlp.local.input") }}
 {{- end }}
 {{- if .Values.cloud.traces.enabled }}
-{{- $list = append $list ("otelcol.exporter.otlp.cloud.input") }}
+{{- $list = append $list ("otelcol.exporter.otlphttp.cloud.input") }}
 {{- end }}
 {{- join ", " $list }}
 {{- end }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -295,13 +295,23 @@ data:
       // In this case, the Agent is listening on all available bindable addresses on port 4317 (which is the
       // default OTLP gRPC port) for the OTLP protocol.
       grpc {
-          endpoint = "0.0.0.0:4317"
+        endpoint = "0.0.0.0:4317"
       }
 
       // We define where to send the output of all ingested traces. In this case, to the OpenTelemetry batch processor
       // named 'default'.
       output {
-          traces = [otelcol.processor.batch.default.input]
+        traces = [otelcol.processor.batch.default.input]
+      }
+    }
+
+    otelcol.receiver.jaeger "jaeger" {
+      protocols {
+        thrift_http {}
+      }
+
+      output {
+        traces = [otelcol.processor.batch.default.input]
       }
     }
 
@@ -335,25 +345,6 @@ data:
     }
     {{- end }}
 
-    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
-    // The OpenTelemetry exporter exports processed trace spans to another target that is listening for OTLP format traces.
-    // A unique label, 'local', is added to uniquely identify this exporter.
-    otelcol.exporter.otlp "local" {
-        // Define the client for exporting.
-        client {
-            // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC).
-            endpoint = "meta-tempo-distributor:4317"
-            // Configure TLS settings for communicating with the endpoint.
-            tls {
-                // The connection is insecure.
-                insecure = true
-                // Do not verify TLS certificates when connecting.
-                insecure_skip_verify = true
-            }
-        }
-    }
-    {{- end }}
-
     {{- if .Values.cloud.logs.enabled }}
     loki.write "cloud" {
       endpoint {
@@ -379,7 +370,7 @@ data:
     {{- end }}
 
     {{- if .Values.cloud.traces.enabled }}
-    otelcol.exporter.otlp "cloud" {
+    otelcol.exporter.otlphttp "cloud" {
         client {
             endpoint = nonsensitive(remote.kubernetes.secret.traces_credentials.data["endpoint"])
             auth = otelcol.auth.basic.creds.handler

charts/meta-monitoring/values.yaml

@@ -267,6 +267,15 @@ alloy:
         memory: '600Mi'
       limits:
         memory: '4Gi'
+    extraPorts:
+    - name: "otel"
+      port: 4317
+      targetPort: 4317
+      protocol: "TCP"
+    - name: "thrifthttp"
+      port: 14268
+      targetPort: 14268
+      protocol: "TCP"
   controller:
     type: "statefulset"
     autoscaling:

docs/installation.md

@@ -34,12 +34,12 @@
     --from-literal=endpoint='https://prometheus-us-central1.grafana.net/api/prom/push'
 
    kubectl create secret generic traces -n meta \
-    --from-literal=username=<traces username> \
+    --from-literal=username=<OTLP instance ID> \
     --from-literal=password=<token>
-    --from-literal=endpoint='https://tempo-us-central1.grafana.net/tempo'
+    --from-literal=endpoint='https://otlp-gateway-prod-us-east-0.grafana.net/otlp'
    ```
 
-   The logs, metrics and traces usernames are the `User / Username / Instance IDs` of the Loki, Prometheus/Mimir and Tempo instances in Grafana Cloud. From `Home` in Grafana click on `Stacks`. Then go to the `Details` pages of Loki, Prometheus/Mimir and Tempo.
+   The logs, metrics and traces usernames are the `User / Username / Instance IDs` of the Loki, Prometheus/Mimir and OpenTelemetry instances in Grafana Cloud. From `Home` in Grafana click on `Stacks`. Then go to the `Details` pages of Loki and Prometheus/Mimir. For OpenTelemetry go to the `Configure` page.
 
 1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml). Fill in the names of the secrets created above as needed. An example minimal values.yaml looks like this:
 
@@ -164,3 +164,22 @@ For each of the dashboard files in charts/meta-monitoring/src/dashboards folder
   ```
   mimirtool rules print --address=<your_cloud_prometheus_endpoint> --id=<your_instance_id> --key=<your_cloud_access_policy_token>
   ```
+
+## Configure Loki to send traces
+
+1. In the Loki config enable tracing:
+
+   ```
+   loki:
+     tracing:
+       enabled: true
+   ```
+
+1. Add the following environment variables to your Loki binaries. When using the Loki Helm chart these can be added using the `extraEnv` setting for the Loki components.
+
+   1. JAEGER_ENDPOINT: http address of the mmc-alloy service installed by the meta-monitoring chart, for example "http://mmc-alloy:14268/api/traces"
+   1. JAEGER_AGENT_TAGS: extra tags you would like to add to the spans, for example  'cluster="abc",namespace="def"'
+   1. JAEGER_SAMPLER_TYPE: the sampling strategy, for example to sample all use 'const' with a value of 1 for the next environment variable
+   1. JAEGER_SAMPLER_PARAM: 1
+
+1. If Loki is installed in a different namespace you can create an [ExternalName service](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) in Kubernetes to point to the mmc-alloy service in the meta monitoring namespace