Add local.enabled setting and make it work

Signed-off-by: Michel Hollands <michel.hollands@grafana.com>

charts/meta-monitoring/Chart.yaml

@@ -27,9 +27,11 @@ dependencies:
 - name: loki
   repository:  https://grafana.github.io/helm-charts
   version: "5.8.0"
+  condition: local.enabled
 - name: grafana-agent
   repository:  https://grafana.github.io/helm-charts
   version: "0.15.0"
 - name: mimir-distributed
   repository:  https://grafana.github.io/helm-charts
-  version: "4.4.1"
+  version: "4.4.1"
+  condition: local.enabled

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -5,3 +5,25 @@
 {{- end }}
 {{- join ", " $list }}
 {{- end }}
+
+{{- define "agent.loki_write_targets" -}}
+{{- $list := list }}
+{{- if .Values.local.enabled }}
+{{- $list = append $list ("loki.write.local.receiver") }}
+{{- end }}
+{{- if .Values.cloud.enabled }}
+{{- $list = append $list ("loki.write.cloud.receiver") }}
+{{- end }}
+{{- join ", " $list }}
+{{- end }}
+
+{{- define "agent.prometheus_write_targets" -}}
+{{- $list := list }}
+{{- if .Values.local.enabled }}
+{{- $list = append $list ("prometheus.remote_write.local.receiver") }}
+{{- end }}
+{{- if .Values.cloud.enabled }}
+{{- $list = append $list ("prometheus.remote_write.cloud.receiver") }}
+{{- end }}
+{{- join ", " $list }}
+{{- end }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -32,7 +32,7 @@ data:
     }
 
     loki.process "loki" {
-      forward_to = [loki.write.local.receiver]
+      forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
       stage.static_labels {
         values = {
@@ -47,19 +47,44 @@ data:
 
     prometheus.scrape "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
-      forward_to = [prometheus.remote_write.default.receiver]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
 
+
+    {{- if .Values.local.enabled }}
     loki.write "local" {
       endpoint {
         url = "http://{{- .Release.Name -}}-loki.{{- .Release.Namespace -}}.svc.cluster.local:3100/loki/api/v1/push"
       }
     }
 
-    prometheus.remote_write "default" {
+    prometheus.remote_write "local" {
       endpoint {
         url = "http://{{- .Release.Name -}}-mimir-nginx.{{- .Release.Namespace -}}.svc:80/api/v1/push"
       }
     }
+    {{- end }}
 
+    {{- if .Values.cloud.enabled }}
+    loki.write "cloud" {
+      endpoint {
+        url = "{{- .Values.cloud.logs.endpoint -}}/loki/api/v1/push"
 
+        basic_auth {
+          username = "{{- .Values.cloud.logs.username -}}"
+          password = "{{- .Values.cloud.logs.password -}}"
+        }
+      }
+    }
+
+    prometheus.remote_write "cloud" {
+      endpoint {
+        url = "{{- .Values.cloud.metrics.endpoint -}}/api/prom/push"
+
+        basic_auth {
+          username = "{{- .Values.cloud.metrics.username -}}"
+          password = "{{- .Values.cloud.metrics.password -}}"
+        }
+      }
+    }
+    {{- end }}

charts/meta-monitoring/templates/grafana/configmap-1.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.local.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -14,4 +15,5 @@ data:
   "loki-mixin-recording-rules.json": |
     {{ $.Files.Get "src/dashboards/loki-mixin-recording-rules.json" | fromJson | toJson }}
   "loki-operational.json": |
-    {{ $.Files.Get "src/dashboards/loki-operational.json" | fromJson | toJson }}
+    {{ $.Files.Get "src/dashboards/loki-operational.json" | fromJson | toJson }}
+{{- end }}

charts/meta-monitoring/templates/grafana/configmap-2.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.local.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -14,4 +15,5 @@ data:
   "loki-writes-resources.json": |
     {{ $.Files.Get "src/dashboards/loki-writes-resources.json" | fromJson | toJson }}
   "loki-writes.json": |
-    {{ $.Files.Get "src/dashboards/loki-writes.json" | fromJson | toJson }}
+    {{ $.Files.Get "src/dashboards/loki-writes.json" | fromJson | toJson }}
+{{- end }}

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.local.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -24,4 +25,5 @@ data:
         options:
           path: /var/lib/grafana/dashboards/loki-2
         orgId: 1
-        type: file
+        type: file
+{{- end }}

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.local.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -56,4 +57,5 @@ data:
         isDefault:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
-        editable: true
+        editable: true
+{{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.local.enabled }}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -100,4 +101,5 @@ spec:
   selector:
     app: grafana
   sessionAffinity: None
-  type: ClusterIP  # Make this configurable
+  type: ClusterIP  # Make this configurable
+{{- end }}

charts/meta-monitoring/values.yaml

@@ -3,6 +3,18 @@ namespacesToMonitor:
 - mimir
 - tempo
 clusterName: "observability"  # TODO check if this can be derived
+local:
+  enabled: false
+cloud:
+  enabled: true
+  logs:
+    endpoint:
+    username:
+    password:
+  metrics:
+    endpoint:
+    username:
+    password:
 
 # The following are configuration for the dependencies.
 # These should not be changed.