Add scraping of kube state metrics

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

charts/meta-monitoring/Chart.yaml

@@ -27,19 +27,19 @@ dependencies:
 - name: loki
   repository:  https://grafana.github.io/helm-charts
   version: "5.8.0"
-  condition: local.enabled
+  condition: local.logs.enabled
 - name: grafana-agent
   repository:  https://grafana.github.io/helm-charts
   version: "0.15.0"
 - name: mimir-distributed
   repository:  https://grafana.github.io/helm-charts
   version: "4.4.1"
-  condition: local.enabled
+  condition: local.metrics.enabled
 - name: tempo-distributed
   repository:  https://grafana.github.io/helm-charts
   version: "1.4.7"
-  condition: local.enabled
+  condition: local.traces.enabled
 - name: minio
   repository: https://charts.min.io
   version: "5.0.11"
-  condition: local.enabled
+  condition: local.minio.enabled

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -8,10 +8,10 @@
 
 {{- define "agent.loki_write_targets" -}}
 {{- $list := list }}
-{{- if .Values.local.enabled }}
+{{- if .Values.local.logs.enabled }}
 {{- $list = append $list ("loki.write.local.receiver") }}
 {{- end }}
-{{- if .Values.cloud.enabled }}
+{{- if .Values.cloud.logs.enabled }}
 {{- $list = append $list ("loki.write.cloud.receiver") }}
 {{- end }}
 {{- join ", " $list }}
@@ -19,10 +19,10 @@
 
 {{- define "agent.prometheus_write_targets" -}}
 {{- $list := list }}
-{{- if .Values.local.enabled }}
+{{- if .Values.local.metrics.enabled }}
 {{- $list = append $list ("prometheus.remote_write.local.receiver") }}
 {{- end }}
-{{- if .Values.cloud.enabled }}
+{{- if .Values.cloud.metrics.enabled }}
 {{- $list = append $list ("prometheus.remote_write.cloud.receiver") }}
 {{- end }}
 {{- join ", " $list }}
@@ -30,10 +30,10 @@
 
 {{- define "agent.tempo_write_targets" -}}
 {{- $list := list }}
-{{- if .Values.local.enabled }}
+{{- if .Values.local.traces.enabled }}
 {{- $list = append $list ("otelcol.exporter.otlp.local.input") }}
 {{- end }}
-{{- if .Values.cloud.enabled }}
+{{- if .Values.cloud.traces.enabled }}
 {{- $list = append $list ("otelcol.exporter.otlp.cloud.input") }}
 {{- end }}
 {{- join ", " $list }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -37,16 +37,28 @@ data:
       }
     }
 
+    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     loki.source.kubernetes "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
     }
+    {{- end }}
 
+    {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
     prometheus.scrape "pods" {
       targets    = discovery.relabel.rename_meta_labels.output
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
+    {{- if .Values.kubeStateMetrics.enabled }}
 
+    prometheus.scrape "kubeStateMetrics" {
+      targets    = [ { "__address__" = "{{ .Values.kubeStateMetrics.endpoint }}" } ]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
+    }
+    {{- end }}
+    {{- end }}
+
+    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {
       // We don't technically need this, but it shows how to change listen address and incoming port.
@@ -75,20 +87,25 @@ data:
             traces = [ {{ include "agent.tempo_write_targets" . }} ]
         }
     }
+    {{- end }}
 
-    {{- if .Values.local.enabled }}
+    {{- if .Values.local.logs.enabled }}
     loki.write "local" {
       endpoint {
         url = "http://loki-gateway.{{- .Release.Namespace -}}.svc.cluster.local:80/loki/api/v1/push"
       }
     }
+    {{- end }}
 
+    {{- if .Values.local.metrics.enabled }}
     prometheus.remote_write "local" {
       endpoint {
         url = "http://{{- .Release.Name -}}-mimir-nginx.{{- .Release.Namespace -}}.svc:80/api/v1/push"
       }
     }
+    {{- end }}
 
+    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // The OpenTelemetry exporter exports processed trace spans to another target that is listening for OTLP format traces.
     // A unique label, 'local', is added to uniquely identify this exporter.
     otelcol.exporter.otlp "local" {
@@ -107,7 +124,7 @@ data:
     }
     {{- end }}
 
-    {{- if .Values.cloud.enabled }}
+    {{- if .Values.cloud.logs.enabled }}
     loki.write "cloud" {
       endpoint {
         url = "{{- .Values.cloud.logs.endpoint -}}/loki/api/v1/push"
@@ -118,7 +135,9 @@ data:
         }
       }
     }
+    {{- end }}
 
+    {{- if .Values.cloud.metrics.enabled }}
     prometheus.remote_write "cloud" {
       endpoint {
         url = "{{- .Values.cloud.metrics.endpoint -}}/api/prom/push"
@@ -129,7 +148,9 @@ data:
         }
       }
     }
+    {{- end }}
 
+    {{- if .Values.cloud.traces.enabled }}
     otelcol.exporter.otlp "cloud" {
         client {
             endpoint = "{{- .Values.cloud.traces.endpoint -}}"

charts/meta-monitoring/templates/grafana/configmap-1.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.local.enabled }}
+{{- if .Values.local.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/configmap-2.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.local.enabled }}
+{{- if .Values.local.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.local.enabled }}
+{{- if .Values.local.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.local.enabled }}
+{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -20,6 +20,7 @@ data:
     datasources:
       # <string, required> Sets the name you use to refer to
       # the data source in panels and queries.
+      {{- if .Values.local.logs.enabled }}
       - name: Loki
         # <string, required> Sets the data source type.
         type: loki
@@ -39,6 +40,8 @@ data:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
         editable: true
+      {{- end }}
+      {{- if .Values.local.metrics.enabled }}
       - name: Mimir
         # <string, required> Sets the data source type.
         type: prometheus
@@ -58,6 +61,8 @@ data:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
         editable: true
+      {{- end }}
+      {{- if .Values.local.traces.enabled }}
       - name: Tempo
         # <string, required> Sets the data source type.
         type: tempo
@@ -77,4 +82,5 @@ data:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
         editable: true
+      {{- end }}
 {{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.local.enabled }}
+{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -63,30 +63,34 @@ spec:
           volumeMounts:
             - mountPath: /var/lib/grafana
               name: grafana-pv
+            - mountPath: /etc/grafana/provisioning/datasources
+              name: loki-datasources-provisioning
+            {{- if .Values.local.logs.enabled }}
+            - mountPath: /etc/grafana/provisioning/dashboards
+              name: loki-dashboards-provisioning
             - mountPath: /var/lib/grafana/dashboards/loki-1
               name: loki-dashboards-1
             - mountPath: /var/lib/grafana/dashboards/loki-2
               name: loki-dashboards-2
-            - mountPath: /etc/grafana/provisioning/dashboards
-              name: loki-dashboards-provisioning
-            - mountPath: /etc/grafana/provisioning/datasources
-              name: loki-datasources-provisioning
+            {{- end }}
       volumes:
         - name: grafana-pv
           persistentVolumeClaim:
             claimName: grafana-pvc
-        - name: loki-dashboards-provisioning
-          configMap:
-            name: loki-dashboards-provisioning
         - name: loki-datasources-provisioning
           configMap:
             name: loki-datasources-provisioning
+        {{- if .Values.local.logs.enabled }}
+        - name: loki-dashboards-provisioning
+          configMap:
+            name: loki-dashboards-provisioning
         - name: loki-dashboards-1
           configMap:
             name: loki-dashboards-1
         - name: loki-dashboards-2
           configMap:
             name: loki-dashboards-2
+        {{- end }}
 
 ---
 apiVersion: v1

charts/meta-monitoring/templates/validate.yaml

@@ -0,0 +1,39 @@
+{{- if not (eq .Values.local.minio.enabled (or .Values.local.traces.enabled (or .Values.local.logs.enabled .Values.local.metrics.enabled))) -}}
+  {{- fail "minio.enabled should be set to true if any of the the local databases is enabled " -}}
+{{- end -}}
+
+{{- if eq .Values.cloud.logs.enabled true -}}
+  {{- if or (empty .Values.cloud.logs.endpoint) (or (empty .Values.cloud.logs.username) (empty .Values.cloud.logs.password)) -}}
+    {{- fail "if cloud.logs is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- end -}}
+{{- end -}}
+
+{{- if eq .Values.cloud.metrics.enabled true -}}
+  {{- if or (empty .Values.cloud.metrics.endpoint) (or (empty .Values.cloud.metrics.username) (empty .Values.cloud.metrics.password)) -}}
+    {{- fail "if cloud.metrics is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- end -}}
+{{- end -}}
+
+{{- if eq .Values.cloud.traces.enabled true -}}
+  {{- if or (empty .Values.cloud.traces.endpoint) (or (empty .Values.cloud.traces.username) (empty .Values.cloud.traces.password)) -}}
+    {{- fail "if cloud.traces is enabled then the endpoint, username and password have to be filled in" -}}
+  {{- end -}}
+{{- end -}}
+
+{{- if eq .Values.cloud.traces.enabled false -}}
+{{- if eq .Values.cloud.metrics.enabled false -}}
+{{- if eq .Values.cloud.logs.enabled false -}}
+{{- if eq .Values.local.traces.enabled false -}}
+{{- if eq .Values.local.metrics.enabled false -}}
+{{- if eq .Values.local.logs.enabled false -}}
+    {{- fail "nothing has been enabled" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- if empty .Values.namespacesToMonitor -}}
+  {{- fail "No namespaces have been specified in namespacesToMonitor" -}}
+{{- end -}}

charts/meta-monitoring/values.yaml

@@ -2,20 +2,32 @@ namespacesToMonitor:
 - loki
 - mimir
 - tempo
-clusterName: "observability"  # TODO check if this can be derived
+clusterName: "meta-monitoring"  # TODO check if this can be derived
+
 local:
-  enabled: true
-cloud:
-  enabled: false
   logs:
+    enabled: false
+  metrics:
+    enabled: false
+  traces:
+    enabled: false
+  minio:
+    enabled: false  # This should be set to true if any of the previous is enabled
+
+
+cloud:
+  logs:
+    enabled: true
     endpoint:
     username:
     password:
   metrics:
+    enabled: true
     endpoint:
     username:
     password:
   traces:
+    enabled: true
     endpoint:
     username:
     password:
@@ -25,6 +37,14 @@ global:
     rootUser: "rootuser"
     rootPassword: "rootpassword"
 
+kubeStateMetrics:
+  # Scrape https://github.com/kubernetes/kube-state-metrics by default
+  enabled: true
+  # This endpoint is created when the helm chart from
+  # https://artifacthub.io/packages/helm/prometheus-community/kube-state-metrics/
+  # is used. Change this if kube-state-metrics is installed somewhere else.
+  endpoint: kube-state-metrics.kube-state-metrics.svc.cluster.local:8080
+
 # The following are configuration for the dependencies.
 # These should not be changed.