Public/grafana-meta-monitoring-chart
2
0
Fork 0
forked from RemoteSync/grafana-meta-monitoring-chart
Code Pull Requests Activity

Removed cluster definitions

Browse Source
This commit is contained in:
Kim-Eirik Karlsen 2025-04-10 09:32:50 +02:00
parent a425aaae44
commit e0d9fea80f
Signed by: kimma
GPG Key ID: DDEB38C2C40B3B83
9 changed files with 151 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
charts/meta-monitoring/src/dashboards/alloy-cluster-node.json
View File

@ -16,7 +16,7 @@
{
"datasource": "$loki_datasource",
"enable": true,
"expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"expr": "{container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"iconColor": "rgba(0, 211, 255, 1)",
"instant": false,
"name": "Deployments",
@ -115,7 +115,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(cluster_node_lamport_time{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(cluster_node_lamport_time{instance=\"$instance\", namespace=\"$namespace\"})",
"format": "table",
"instant": true,
"legendFormat": "__auto",
@ -124,7 +124,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(cluster_node_update_observers{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(cluster_node_update_observers{instance=\"$instance\", namespace=\"$namespace\"})",
"format": "table",
"instant": true,
"legendFormat": "__auto",
@ -133,7 +133,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(cluster_node_gossip_health_score{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(cluster_node_gossip_health_score{instance=\"$instance\", namespace=\"$namespace\"})",
"format": "table",
"instant": true,
"legendFormat": "__auto",
@ -142,7 +142,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(cluster_node_gossip_proto_version{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(cluster_node_gossip_proto_version{instance=\"$instance\", namespace=\"$namespace\"})",
"format": "table",
"instant": true,
"legendFormat": "__auto",
@ -255,7 +255,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(cluster_node_gossip_received_events_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
"expr": "rate(cluster_node_gossip_received_events_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])",
"instant": false,
"legendFormat": "{{event}}",
"range": true,
@ -315,7 +315,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(cluster_node_peers{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(cluster_node_peers{instance=\"$instance\", namespace=\"$namespace\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -405,7 +405,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "cluster_node_peers{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}",
"expr": "cluster_node_peers{instance=\"$instance\", namespace=\"$namespace\"}",
"instant": false,
"legendFormat": "{{state}}",
"range": true,
@ -506,7 +506,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(cluster_transport_rx_bytes_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
"expr": "rate(cluster_transport_rx_bytes_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])",
"instant": false,
"legendFormat": "rx",
"range": true,
@ -514,7 +514,7 @@
},
{
"datasource": "${datasource}",
"expr": "-1 * rate(cluster_transport_tx_bytes_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
"expr": "-1 * rate(cluster_transport_tx_bytes_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])",
"instant": false,
"legendFormat": "tx",
"range": true,
@ -603,7 +603,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "1 - (\nrate(cluster_transport_tx_packets_failed_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) /\nrate(cluster_transport_tx_packets_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n)\n",
"expr": "1 - (\nrate(cluster_transport_tx_packets_failed_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval]) /\nrate(cluster_transport_tx_packets_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])\n)\n",
"instant": false,
"legendFormat": "Tx success %",
"range": true,
@ -611,7 +611,7 @@
},
{
"datasource": "${datasource}",
"expr": "1 - (\n rate(cluster_transport_rx_packets_failed_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_rx_packets_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"expr": "1 - (\n rate(cluster_transport_rx_packets_failed_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_rx_packets_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"instant": false,
"legendFormat": "Rx success %",
"range": true,
@ -701,7 +701,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "cluster_transport_tx_packet_queue_length{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}",
"expr": "cluster_transport_tx_packet_queue_length{instance=\"$instance\", namespace=\"$namespace\"}",
"instant": false,
"legendFormat": "tx queue",
"range": true,
@ -709,7 +709,7 @@
},
{
"datasource": "${datasource}",
"expr": "cluster_transport_rx_packet_queue_length{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}",
"expr": "cluster_transport_rx_packet_queue_length{instance=\"$instance\", namespace=\"$namespace\"}",
"instant": false,
"legendFormat": "rx queue",
"range": true,
@ -798,7 +798,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(cluster_transport_stream_rx_bytes_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
"expr": "rate(cluster_transport_stream_rx_bytes_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])",
"instant": false,
"legendFormat": "rx",
"range": true,
@ -806,7 +806,7 @@
},
{
"datasource": "${datasource}",
"expr": "-1 * rate(cluster_transport_stream_tx_bytes_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
"expr": "-1 * rate(cluster_transport_stream_tx_bytes_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])",
"instant": false,
"legendFormat": "tx",
"range": true,
@ -895,7 +895,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "1 - (\n rate(cluster_transport_stream_tx_packets_failed_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_stream_tx_packets_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"expr": "1 - (\n rate(cluster_transport_stream_tx_packets_failed_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_stream_tx_packets_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"instant": false,
"legendFormat": "Tx success %",
"range": true,
@ -903,7 +903,7 @@
},
{
"datasource": "${datasource}",
"expr": "1 - (\n rate(cluster_transport_stream_rx_packets_failed_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_stream_rx_packets_total{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"expr": "1 - (\n rate(cluster_transport_stream_rx_packets_failed_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval]) /\n rate(cluster_transport_stream_rx_packets_total{instance=\"$instance\", namespace=\"$namespace\"}[$__rate_interval])\n )\n",
"instant": false,
"legendFormat": "Rx success %",
"range": true,
@ -992,7 +992,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "cluster_transport_streams{instance=\"$instance\", cluster=\"$cluster\", namespace=\"$namespace\"}",
"expr": "cluster_transport_streams{instance=\"$instance\", namespace=\"$namespace\"}",
"instant": false,
"legendFormat": "Open streams",
"range": true,
@ -1122,7 +1122,7 @@
"name": "instance",
"options": [],
"query": {
"query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
"query": "label_values(alloy_component_controller_running_components{namespace=\"$namespace\"}, instance)\n",
"refId": "instance"
},
"refresh": 2,

10
charts/meta-monitoring/src/dashboards/alloy-cluster-overview.json
View File

@ -16,7 +16,7 @@
{
"datasource": "$loki_datasource",
"enable": true,
"expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"expr": "{container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"iconColor": "rgba(0, 211, 255, 1)",
"instant": false,
"name": "Deployments",
@ -99,7 +99,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "count(cluster_node_info{namespace=\"$namespace\"})",
"instant": true,
"legendFormat": "__auto",
"range": false,
@ -194,7 +194,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"}",
"expr": "cluster_node_info{namespace=\"$namespace\"}",
"format": "table",
"instant": true,
"legendFormat": "__auto",
@ -298,7 +298,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "clamp((\n sum(stddev by (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"}) != 0) or\n (sum(abs(sum without (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) - scalar(count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"})) != 0))\n ),\n 1, 1\n)\n",
"expr": "clamp((\n sum(stddev by (state) (cluster_node_peers{namespace=\"$namespace\"}) != 0) or\n (sum(abs(sum without (state) (cluster_node_peers{namespace=\"$namespace\"})) - scalar(count(cluster_node_info{namespace=\"$namespace\"})) != 0))\n ),\n 1, 1\n)\n",
"format": "time_series",
"instant": true,
"legendFormat": "__auto",
@ -390,7 +390,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "ceil(clamp((\n sum(stddev by (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) or\n (sum(abs(sum without (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) - scalar(count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"}))))\n ),\n 0, 1\n))\n",
"expr": "ceil(clamp((\n sum(stddev by (state) (cluster_node_peers{namespace=\"$namespace\"})) or\n (sum(abs(sum without (state) (cluster_node_peers{namespace=\"$namespace\"})) - scalar(count(cluster_node_info{namespace=\"$namespace\"}))))\n ),\n 0, 1\n))\n",
"instant": false,
"legendFormat": "Converged",
"range": true,

24
charts/meta-monitoring/src/dashboards/alloy-controller.json
View File

@ -16,7 +16,7 @@
{
"datasource": "$loki_datasource",
"enable": true,
"expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"expr": "{container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"iconColor": "rgba(0, 211, 255, 1)",
"instant": false,
"name": "Deployments",
@ -101,7 +101,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "count(alloy_component_controller_evaluating{cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "count(alloy_component_controller_evaluating{namespace=\"$namespace\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -232,7 +232,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"healthy\"}) or vector(0)",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\", health_type=\"healthy\"}) or vector(0)",
"instant": true,
"legendFormat": "Healthy",
"range": false,
@ -240,7 +240,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"unhealthy\"}) or vector(0)",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\", health_type=\"unhealthy\"}) or vector(0)",
"instant": true,
"legendFormat": "Unhealthy",
"range": false,
@ -248,7 +248,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"unknown\"}) or vector(0)",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\", health_type=\"unknown\"}) or vector(0)",
"instant": true,
"legendFormat": "Unknown",
"range": false,
@ -256,7 +256,7 @@
},
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"exited\"}) or vector(0)",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\", health_type=\"exited\"}) or vector(0)",
"instant": true,
"legendFormat": "Exited",
"range": false,
@ -316,7 +316,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"})",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -382,7 +382,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\",health_type=\"healthy\"}) /\nsum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"})\n",
"expr": "sum(alloy_component_controller_running_components{namespace=\"$namespace\",health_type=\"healthy\"}) /\nsum(alloy_component_controller_running_components{namespace=\"$namespace\"})\n",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -472,7 +472,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (instance) (rate(alloy_component_evaluation_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
"expr": "sum by (instance) (rate(alloy_component_evaluation_seconds_count{namespace=\"$namespace\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "__auto",
"range": true,
@ -668,7 +668,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (component_path, component_id) (rate(alloy_component_evaluation_slow_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/ scalar(sum(rate(alloy_component_evaluation_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))\n",
"expr": "sum by (component_path, component_id) (rate(alloy_component_evaluation_slow_seconds{namespace=\"$namespace\"}[$__rate_interval]))\n/ scalar(sum(rate(alloy_component_evaluation_seconds_sum{namespace=\"$namespace\"}[$__rate_interval])))\n",
"instant": false,
"legendFormat": "{{component path}} {{component_id}}",
"range": true,
@ -743,7 +743,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(increase(alloy_component_evaluation_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_evaluation_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n",
"expr": "sum(increase(alloy_component_evaluation_seconds{namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_evaluation_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval]))\n",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",
@ -819,7 +819,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(increase(alloy_component_dependencies_wait_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_dependencies_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n",
"expr": "sum(increase(alloy_component_dependencies_wait_seconds{namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_dependencies_wait_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval]))\n",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",

18
charts/meta-monitoring/src/dashboards/alloy-opentelemetry.json
View File

@ -125,7 +125,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(receiver_accepted_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"expr": "rate(receiver_accepted_spans_ratio_total{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"instant": false,
"legendFormat": "{{ pod }} / {{ transport }}",
"range": true,
@ -214,7 +214,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(receiver_refused_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"expr": "rate(receiver_refused_spans_ratio_total{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"instant": false,
"legendFormat": "{{ pod }} / {{ transport }}",
"range": true,
@ -289,7 +289,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (le) (increase(rpc_server_duration_milliseconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", rpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\"}[$__rate_interval]))",
"expr": "sum by (le) (increase(rpc_server_duration_milliseconds_bucket{namespace=\"$namespace\", instance=~\"$instance\", rpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\"}[$__rate_interval]))",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",
@ -377,7 +377,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (le) (increase(processor_batch_batch_send_size_ratio_bucket{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval]))",
"expr": "sum by (le) (increase(processor_batch_batch_send_size_ratio_bucket{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",
@ -467,7 +467,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "processor_batch_metadata_cardinality_ratio{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}\n",
"expr": "processor_batch_metadata_cardinality_ratio{namespace=\"$namespace\", instance=~\"$instance\"}\n",
"instant": false,
"legendFormat": "{{ pod }}",
"range": true,
@ -556,7 +556,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(processor_batch_timeout_trigger_send_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"expr": "rate(processor_batch_timeout_trigger_send_ratio_total{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"instant": false,
"legendFormat": "{{ pod }}",
"range": true,
@ -657,7 +657,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(exporter_sent_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"expr": "rate(exporter_sent_spans_ratio_total{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"instant": false,
"legendFormat": "{{ pod }}",
"range": true,
@ -746,7 +746,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "rate(exporter_send_failed_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"expr": "rate(exporter_send_failed_spans_ratio_total{namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
"instant": false,
"legendFormat": "{{ pod }}",
"range": true,
@ -874,7 +874,7 @@
"name": "instance",
"options": [],
"query": {
"query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
"query": "label_values(alloy_component_controller_running_components{namespace=\"$namespace\"}, instance)\n",
"refId": "instance"
},
"refresh": 2,

32
charts/meta-monitoring/src/dashboards/alloy-prometheus.json
View File

@ -16,7 +16,7 @@
{
"datasource": "$loki_datasource",
"enable": true,
"expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"expr": "{container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"iconColor": "rgba(0, 211, 255, 1)",
"instant": false,
"name": "Deployments",
@ -353,7 +353,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (instance, component_path, component_id) (\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\"}\n - ignoring(url, remote_name) group_right(instance)\n prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"expr": "sum by (instance, component_path, component_id) (\n prometheus_remote_storage_highest_timestamp_in_seconds{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\"}\n - ignoring(url, remote_name) group_right(instance)\n prometheus_remote_storage_queue_highest_sent_timestamp_seconds{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -443,7 +443,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum without (remote_name, url) (\n rate(prometheus_remote_storage_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval]) +\n rate(prometheus_remote_storage_metadata_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"expr": "sum without (remote_name, url) (\n rate(prometheus_remote_storage_bytes_total{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval]) +\n rate(prometheus_remote_storage_metadata_bytes_total{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -700,7 +700,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum without (remote_name, url) (\n prometheus_remote_storage_shards{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"expr": "sum without (remote_name, url) (\n prometheus_remote_storage_shards{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -708,7 +708,7 @@
},
{
"datasource": "${datasource}",
"expr": "min (\n prometheus_remote_storage_shards_min{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"expr": "min (\n prometheus_remote_storage_shards_min{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"instant": false,
"legendFormat": "Minimum",
"range": true,
@ -716,7 +716,7 @@
},
{
"datasource": "${datasource}",
"expr": "max (\n prometheus_remote_storage_shards_max{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"expr": "max (\n prometheus_remote_storage_shards_max{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
"instant": false,
"legendFormat": "Maximum",
"range": true,
@ -806,7 +806,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum without (url, remote_name) (\n rate(prometheus_remote_storage_samples_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"expr": "sum without (url, remote_name) (\n rate(prometheus_remote_storage_samples_total{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -896,7 +896,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum without (url,remote_name) (\n rate(prometheus_remote_storage_samples_failed_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"expr": "sum without (url,remote_name) (\n rate(prometheus_remote_storage_samples_failed_total{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -986,7 +986,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum without (url,remote_name) (\n rate(prometheus_remote_storage_samples_retried_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"expr": "sum without (url,remote_name) (\n rate(prometheus_remote_storage_samples_retried_total{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -1021,7 +1021,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum(prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
"expr": "sum(prometheus_remote_write_wal_storage_active_series{namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
"instant": false,
"legendFormat": "Series",
"range": true,
@ -1050,7 +1050,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n",
"expr": "prometheus_remote_write_wal_storage_active_series{namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n",
"instant": false,
"legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
"range": true,
@ -1079,7 +1079,7 @@
"targets": [
{
"datasource": "${datasource}",
"expr": "sum by (component_path, component_id) (prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
"expr": "sum by (component_path, component_id) (prometheus_remote_write_wal_storage_active_series{namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
"instant": false,
"legendFormat": "{{component_path}} {{component_id}}",
"range": true,
@ -1207,7 +1207,7 @@
"name": "instance",
"options": [],
"query": {
"query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
"query": "label_values(alloy_component_controller_running_components{namespace=\"$namespace\"}, instance)\n",
"refId": "instance"
},
"refresh": 2,
@ -1234,7 +1234,7 @@
"name": "component_path",
"options": [],
"query": {
"query": "label_values(prometheus_remote_write_wal_samples_appended_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\", component_path=~\".*\"}, component_path)\n",
"query": "label_values(prometheus_remote_write_wal_samples_appended_total{namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\", component_path=~\".*\"}, component_path)\n",
"refId": "component_path"
},
"refresh": 2,
@ -1261,7 +1261,7 @@
"name": "component",
"options": [],
"query": {
"query": "label_values(prometheus_remote_write_wal_samples_appended_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\"}, component_id)\n",
"query": "label_values(prometheus_remote_write_wal_samples_appended_total{namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\"}, component_id)\n",
"refId": "component"
},
"refresh": 2,
@ -1288,7 +1288,7 @@
"name": "url",
"options": [],
"query": {
"query": "label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"$component\"}, url)\n",
"query": "label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"$component\"}, url)\n",
"refId": "url"
},
"refresh": 2,

4
charts/meta-monitoring/src/dashboards/alloy-resources.json
View File

@ -16,7 +16,7 @@
{
"datasource": "$loki_datasource",
"enable": true,
"expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"expr": "{container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
"iconColor": "rgba(0, 211, 255, 1)",
"instant": false,
"name": "Deployments",
@ -791,7 +791,7 @@
"name": "instance",
"options": [],
"query": {
"query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
"query": "label_values(alloy_component_controller_running_components{namespace=\"$namespace\"}, instance)\n",
"refId": "instance"
},
"refresh": 2,

48
charts/meta-monitoring/src/dashboards/loki-chunks.json
View File

@ -65,7 +65,7 @@
"span": 6,
"targets": [
{
"expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"})",
"expr": "sum(loki_ingester_memory_chunks{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"})",
"format": "time_series",
"legendFormat": "series",
"legendLink": null
@ -113,7 +113,7 @@
"span": 6,
"targets": [
{
"expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"})",
"expr": "sum(loki_ingester_memory_chunks{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}) / sum(loki_ingester_memory_streams{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"})",
"format": "time_series",
"legendFormat": "chunks",
"legendLink": null
@ -174,19 +174,19 @@
"span": 6,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"legendFormat": "99th Percentile",
"refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"legendFormat": "50th Percentile",
"refId": "B"
},
{
"expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_ingester_chunk_utilization_sum{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "Average",
"refId": "C"
@ -253,19 +253,19 @@
"span": 6,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"legendFormat": "99th Percentile",
"refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"legendFormat": "50th Percentile",
"refId": "B"
},
{
"expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "Average",
"refId": "C"
@ -344,19 +344,19 @@
"span": 6,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"legendFormat": "99th Percentile",
"refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"legendFormat": "50th Percentile",
"refId": "B"
},
{
"expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_ingester_chunk_entries_sum{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "Average",
"refId": "C"
@ -422,7 +422,7 @@
"span": 6,
"targets": [
{
"expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "Index Entries",
"legendLink": null
@ -482,7 +482,7 @@
"span": 6,
"targets": [
{
"expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}",
"expr": "loki_ingester_flush_queue_length{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} or cortex_ingester_flush_queue_length{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}",
"format": "time_series",
"legendFormat": "{{pod}}",
"legendLink": null
@ -681,7 +681,7 @@
"stack": true,
"targets": [
{
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"legendFormat": "{{status}}",
"refId": "A"
@ -741,7 +741,7 @@
"span": 6,
"targets": [
{
"expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_ingester_chunks_flushed_total{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "{{pod}}",
"legendLink": null
@ -790,7 +790,7 @@
"stack": true,
"targets": [
{
"expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "{{reason}}",
"legendLink": null
@ -854,7 +854,7 @@
"span": 12,
"targets": [
{
"expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "heatmap",
"intervalFactor": 2,
"legendFormat": "{{le}}",
@ -917,7 +917,7 @@
"span": 12,
"targets": [
{
"expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)",
"expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le)",
"format": "heatmap",
"intervalFactor": 2,
"legendFormat": "{{le}}",
@ -994,19 +994,19 @@
"span": 12,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"format": "time_series",
"legendFormat": "p99",
"legendLink": null
},
{
"expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"format": "time_series",
"legendFormat": "p90",
"legendLink": null
},
{
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"format": "time_series",
"legendFormat": "p50",
"legendLink": null
@ -1066,19 +1066,19 @@
"span": 12,
"targets": [
{
"expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"format": "time_series",
"legendFormat": "p50",
"legendLink": null
},
{
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (le))",
"format": "time_series",
"legendFormat": "p99",
"legendLink": null
},
{
"expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{job=~\"$namespace/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "avg",
"legendLink": null

26
charts/meta-monitoring/src/dashboards/loki-logs.json
View File

@ -64,7 +64,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})",
"expr": "sum(go_goroutines{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})",
"refId": "A"
}
],
@ -156,7 +156,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)",
"expr": "sum(go_gc_duration_seconds{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)",
"legendFormat": "{{quantile}}",
"refId": "A"
}
@ -244,7 +244,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[$__rate_interval]))",
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[$__rate_interval]))",
"refId": "A"
}
],
@ -336,7 +336,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})",
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})",
"refId": "A"
}
],
@ -428,7 +428,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
"refId": "A"
}
],
@ -520,7 +520,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
"refId": "A"
}
],
@ -607,7 +607,7 @@
"steppedLine": false,
"targets": [
{
"expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0",
"expr": "increase(kube_pod_container_status_last_terminated_reason{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0",
"legendFormat": "{{reason}}",
"refId": "A"
}
@ -700,7 +700,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)",
"expr": "sum(rate(promtail_custom_bad_words_total{exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)",
"legendFormat": "{{level}}",
"refId": "A"
}
@ -810,7 +810,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" | __error__=\"\" [$__interval])) by (level)",
"expr": "sum(rate({namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" | __error__=\"\" [$__interval])) by (level)",
"intervalFactor": 3,
"legendFormat": "{{level}}",
"refId": "A"
@ -876,7 +876,7 @@
},
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"",
"expr": "{namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"",
"refId": "A"
}
],
@ -975,7 +975,7 @@
"multi": false,
"name": "deployment",
"options": [ ],
"query": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)",
"query": "label_values(kube_deployment_created{namespace=\"$namespace\"}, deployment)",
"refresh": 0,
"regex": "",
"sort": 1,
@ -995,7 +995,7 @@
"multi": false,
"name": "pod",
"options": [ ],
"query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)",
"query": "label_values(kube_pod_container_info{namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)",
"refresh": 0,
"regex": "",
"sort": 1,
@ -1015,7 +1015,7 @@
"multi": false,
"name": "container",
"options": [ ],
"query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)",
"query": "label_values(kube_pod_container_info{namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)",
"refresh": 0,
"regex": "",
"sort": 1,

100
charts/meta-monitoring/src/dashboards/loki-operational.json
View File

@ -89,7 +89,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(.*query-frontend|(loki|enterprise-logs)-read|loki-single-binary)\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)",
"expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{job=~\"($namespace)/(.*query-frontend|(loki|enterprise-logs)-read|loki-single-binary)\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)",
"legendFormat": "{{status}}",
"refId": "A"
}
@ -186,7 +186,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\", route=~\"api_prom_push|loki_api_v1_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\", route=~\"api_prom_push|loki_api_v1_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"legendFormat": "{{status}}",
"refId": "A"
}
@ -281,7 +281,7 @@
"steppedLine": false,
"targets": [
{
"expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))",
"expr": "topk(10, sum(rate(loki_distributor_lines_received_total{namespace=\"$namespace\"}[$__rate_interval])) by (tenant))",
"legendFormat": "{{tenant}}",
"refId": "A"
}
@ -379,7 +379,7 @@
"steppedLine": false,
"targets": [
{
"expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024",
"expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024",
"legendFormat": "{{tenant}}",
"refId": "A"
}
@ -474,7 +474,7 @@
"steppedLine": false,
"targets": [
{
"expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0",
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[10m]) > 0",
"hide": false,
"interval": "",
"legendFormat": "{{container}}-{{pod}}",
@ -789,7 +789,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*distributor|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
@ -998,7 +998,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
@ -1315,7 +1315,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*querier|(loki|enterprise-logs)-read|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*querier|(loki|enterprise-logs)-read|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*querier|(loki|enterprise-logs)-read|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*querier|(loki|enterprise-logs)-read|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
@ -1523,7 +1523,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
@ -1921,7 +1921,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"(.*/.*distributor|$namespace/(loki|enterprise-logs)-write|.*/loki|$namespace/loki-single-binary)\"}",
"expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\", job=~\"(.*/.*distributor|$namespace/(loki|enterprise-logs)-write|.*/loki|$namespace/loki-single-binary)\"}",
"instant": false,
"intervalFactor": 3,
"legendFormat": "{{pod}}",
@ -2017,7 +2017,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"expr": "sum(rate({namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"refId": "A"
}
],
@ -2081,7 +2081,7 @@
"panels": [ ],
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"} |= \"level=error\"",
"expr": "{namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"} |= \"level=error\"",
"refId": "A"
}
],
@ -2133,7 +2133,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*distributor.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{route}}",
@ -2229,7 +2229,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"expr": "sum(rate(loki_distributor_ingester_append_failures_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"intervalFactor": 1,
"legendFormat": "{{pod}}",
"refId": "A"
@ -2324,7 +2324,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"intervalFactor": 1,
"legendFormat": "{{pod}}",
"refId": "A"
@ -2419,7 +2419,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)",
"intervalFactor": 1,
"legendFormat": "{{pod}}",
"refId": "A"
@ -2622,7 +2622,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary|$namespace-[0-9]+)\"}",
"expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary|$namespace-[0-9]+)\"}",
"instant": false,
"intervalFactor": 3,
"legendFormat": "{{pod}}",
@ -2718,7 +2718,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"expr": "sum(rate({namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"refId": "A"
}
],
@ -2782,7 +2782,7 @@
"panels": [ ],
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} |= \"level=error\"",
"expr": "{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"} |= \"level=error\"",
"refId": "A"
}
],
@ -2834,7 +2834,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{route}}",
@ -3152,7 +3152,7 @@
"refId": "A"
},
{
"expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) < 1",
"expr": "sum(increase(loki_chunk_store_deduped_chunks_total{job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval])) < 1",
"interval": "",
"legendFormat": "De-Dupe Ratio",
"refId": "B"
@ -3388,7 +3388,7 @@
"reverseYBuckets": false,
"targets": [
{
"expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{job=~\"($namespace)/(.*ingester|(loki|enterprise-logs)-write|loki-single-binary)\"}[$__rate_interval]))",
"format": "heatmap",
"instant": false,
"interval": "",
@ -3578,7 +3578,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(.*querier.*|(loki|enterprise-logs)-read.*|.*loki-single-binary|$namespace-[0-9]+)\"}",
"expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\", pod=~\"(.*querier.*|(loki|enterprise-logs)-read.*|.*loki-single-binary|$namespace-[0-9]+)\"}",
"instant": false,
"intervalFactor": 3,
"legendFormat": "{{pod}}",
@ -3674,7 +3674,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"expr": "sum(rate({namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\"} | logfmt | level=\"error\"[$__interval]))",
"refId": "A"
}
],
@ -3737,7 +3737,7 @@
"panels": [ ],
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(querier|(loki|enterprise-logs)-read|loki-single-binary)\"} |= \"level=error\"",
"expr": "{namespace=\"$namespace\", job=~\"($namespace)/(querier|(loki|enterprise-logs)-read|loki-single-binary)\"} |= \"level=error\"",
"refId": "A"
}
],
@ -3789,7 +3789,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", job=~\"($namespace)/(.*querier.*|(loki|enterprise-logs)-read|loki-single-binary)\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{route}}",
@ -3903,19 +3903,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"intervalFactor": 1,
"legendFormat": "{{container}}: .99-{{method}}-{{name}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"hide": false,
"legendFormat": "{{container}}: .9-{{method}}-{{name}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))",
"hide": false,
"legendFormat": "{{container}}: .5-{{method}}-{{name}}",
"refId": "C"
@ -4007,7 +4007,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)",
"expr": "sum(rate(loki_memcache_request_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)",
"intervalFactor": 1,
"legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}",
"refId": "A"
@ -4120,19 +4120,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"intervalFactor": 1,
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".5-{{operation}}",
"refId": "C"
@ -4229,7 +4229,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)",
"expr": "sum(rate(loki_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)",
"intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"
@ -4342,19 +4342,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"intervalFactor": 1,
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".5-{{operation}}",
"refId": "C"
@ -4446,7 +4446,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"expr": "sum(rate(loki_gcs_request_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"
@ -4558,19 +4558,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"intervalFactor": 1,
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".5-{{operation}}",
"refId": "C"
@ -4661,7 +4661,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"expr": "sum(rate(loki_s3_request_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"
@ -4773,19 +4773,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"intervalFactor": 1,
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".5-{{operation}}",
"refId": "C"
@ -4876,7 +4876,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"
@ -4988,19 +4988,19 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"intervalFactor": 1,
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))",
"hide": false,
"legendFormat": ".5-{{operation}}",
"refId": "C"
@ -5091,7 +5091,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)",
"intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"