forked from RemoteSync/grafana-meta-monitoring-chart
Compare commits
10 Commits
add_tempo_
...
fix_tempo_
| Author | SHA1 | Date | |
|---|---|---|---|
|
092423c2b3 | ||
|
|
dcbe85a37a | ||
|
|
db8558982c | ||
|
|
49034b9f6b | ||
|
6fb22ae671 | ||
|
|
d3878e1516 | ||
|
|
8ae136e0c4 | ||
|
|
ac3e4462f9 | ||
|
|
c95c0e2ca9 | ||
|
|
c288a80bd4 |
@@ -161,7 +161,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])",
|
||||
"expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])",
|
||||
"interval": "",
|
||||
"legendFormat": "{{pod}}",
|
||||
"refId": "A"
|
||||
@@ -256,7 +256,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
|
||||
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{pod}}",
|
||||
"refId": "A"
|
||||
@@ -351,7 +351,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
|
||||
"expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}",
|
||||
"legendFormat": "{{pod}}",
|
||||
"refId": "A"
|
||||
}
|
||||
@@ -441,7 +441,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])",
|
||||
"expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\".*$component.*\", container!=\"POD\"}[$__rate_interval])",
|
||||
"interval": "",
|
||||
"intervalFactor": 5,
|
||||
"legendFormat": "{{pod}}",
|
||||
@@ -537,7 +537,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}",
|
||||
"expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\".*$component.*\", container!=\"POD\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{pod}}",
|
||||
"refId": "A"
|
||||
@@ -632,14 +632,14 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
|
||||
"expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\".*$component.*\"}[$__rate_interval])",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "rx-{{pod}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
|
||||
"expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\".*$component.*\"}[$__rate_interval])",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "tx-{{pod}}",
|
||||
@@ -735,7 +735,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}",
|
||||
"expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\".*$component.*\"}",
|
||||
"legendFormat": "{{persistentvolumeclaim}}",
|
||||
"refId": "A"
|
||||
}
|
||||
@@ -829,7 +829,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])",
|
||||
"expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\".*$component.*\"}[$__rate_interval])",
|
||||
"interval": "",
|
||||
"legendFormat": "{{exported_pod}}",
|
||||
"refId": "A"
|
||||
@@ -934,7 +934,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
|
||||
"expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
@@ -1024,17 +1024,17 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
|
||||
"expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (job)",
|
||||
"legendFormat": "compaction_err",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
|
||||
"expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (job)",
|
||||
"legendFormat": "retention_err",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
|
||||
"expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (job)",
|
||||
"legendFormat": "blocklist_err",
|
||||
"refId": "D"
|
||||
}
|
||||
@@ -1124,18 +1124,18 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -1227,7 +1227,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)",
|
||||
"expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}) by (tenant)",
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{tenant}}",
|
||||
@@ -1319,19 +1319,19 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*compactor\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*compactor\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*compactor\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -1422,13 +1422,13 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))",
|
||||
"expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval]))",
|
||||
"interval": "",
|
||||
"legendFormat": "deleted",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))",
|
||||
"expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval]))",
|
||||
"interval": "",
|
||||
"legendFormat": "marked_for_deletion",
|
||||
"refId": "B"
|
||||
@@ -2049,7 +2049,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/.*query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -2145,19 +2145,19 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -2253,7 +2253,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/.*query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -2351,7 +2351,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A",
|
||||
@@ -2359,7 +2359,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B",
|
||||
@@ -2367,7 +2367,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C",
|
||||
@@ -2463,7 +2463,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -2559,19 +2559,19 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -2667,7 +2667,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -2765,7 +2765,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A",
|
||||
@@ -2773,7 +2773,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B",
|
||||
@@ -2781,7 +2781,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C",
|
||||
@@ -2877,7 +2877,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -2973,19 +2973,19 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -3081,7 +3081,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -3179,7 +3179,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A",
|
||||
@@ -3187,7 +3187,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B",
|
||||
@@ -3195,7 +3195,7 @@
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C",
|
||||
@@ -3527,7 +3527,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"$namespace/.*query-frontend\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -3632,7 +3632,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"range": true,
|
||||
@@ -3646,7 +3646,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"range": true,
|
||||
@@ -3660,7 +3660,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"range": true,
|
||||
@@ -3763,7 +3763,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}} {{route}}",
|
||||
@@ -3866,7 +3866,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"range": true,
|
||||
@@ -3878,7 +3878,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"range": true,
|
||||
@@ -3890,7 +3890,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"range": true,
|
||||
@@ -3992,7 +3992,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\"}[$__rate_interval])) by (status_code)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\"}[$__rate_interval])) by (status_code)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -4095,7 +4095,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"range": true,
|
||||
@@ -4107,7 +4107,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"range": true,
|
||||
@@ -4119,7 +4119,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"range": true,
|
||||
@@ -4437,7 +4437,7 @@
|
||||
"pluginVersion": "9.0.0-d452322apre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))",
|
||||
"expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\"}[1h]))",
|
||||
"interval": "",
|
||||
"legendFormat": "{{pod}}",
|
||||
"refId": "A"
|
||||
@@ -5132,19 +5132,19 @@
|
||||
"pluginVersion": "9.0.0-d452322apre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"refId": "C"
|
||||
@@ -5359,7 +5359,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99",
|
||||
"range": true,
|
||||
@@ -5371,7 +5371,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9",
|
||||
"range": true,
|
||||
@@ -5383,7 +5383,7 @@
|
||||
"uid": "cortex-ops-01"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5",
|
||||
"range": true,
|
||||
@@ -5496,7 +5496,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)",
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (status_code, method)",
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}-{{method}}",
|
||||
"refId": "A"
|
||||
@@ -5590,19 +5590,19 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (method, le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".99-{{method}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (method, le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".9-{{method}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (method, le))",
|
||||
"interval": "",
|
||||
"legendFormat": ".5-{{method}}",
|
||||
"refId": "C"
|
||||
@@ -5714,7 +5714,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, operation)",
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (status_code, operation)",
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}-{{operation}}",
|
||||
"refId": "A"
|
||||
@@ -5808,17 +5808,17 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"legendFormat": ".99-{{operation}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"legendFormat": ".9-{{operation}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}[$__rate_interval])) by (operation, le))",
|
||||
"legendFormat": ".5-{{operation}}",
|
||||
"refId": "C"
|
||||
}
|
||||
@@ -5934,7 +5934,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "P666011C0B63BDCA4"
|
||||
},
|
||||
"expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
|
||||
"expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
@@ -6028,7 +6028,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
|
||||
"expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
@@ -6122,13 +6122,13 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
|
||||
"expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "min",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
|
||||
"expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "max",
|
||||
"refId": "B"
|
||||
@@ -6227,7 +6227,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "P666011C0B63BDCA4"
|
||||
},
|
||||
"expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
|
||||
"expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "min",
|
||||
"refId": "A"
|
||||
@@ -6237,7 +6237,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "P666011C0B63BDCA4"
|
||||
},
|
||||
"expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
|
||||
"expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*$component\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "max",
|
||||
"refId": "B"
|
||||
@@ -6516,7 +6516,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)",
|
||||
"expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*compactor\"}[$__rate_interval])) by (level)",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
@@ -6608,7 +6608,7 @@
|
||||
"pluginVersion": "9.0.0-d373beebpre",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)",
|
||||
"expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*compactor\"}[$__rate_interval])) by (level)",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
@@ -6701,7 +6701,7 @@
|
||||
"uid": "P666011C0B63BDCA4"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)",
|
||||
"expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*compactor\"}[$__rate_interval])) by (level)",
|
||||
"interval": "",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
@@ -6795,7 +6795,7 @@
|
||||
"uid": "P666011C0B63BDCA4"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)",
|
||||
"expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/.*compactor\"}[5m])) by (level)",
|
||||
"interval": "",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
|
||||
@@ -282,7 +282,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -369,7 +369,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -378,7 +378,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -387,7 +387,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (route)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (route)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -492,7 +492,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -579,7 +579,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -588,7 +588,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -597,7 +597,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (route)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (route)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -702,7 +702,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -789,7 +789,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (le,endpoint)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (le,endpoint)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -798,7 +798,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (le,endpoint)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (le,endpoint)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -807,7 +807,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (endpoint)",
|
||||
"expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (endpoint)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -912,7 +912,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -999,7 +999,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1008,7 +1008,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1017,7 +1017,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route)",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1122,7 +1122,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1209,7 +1209,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1218,7 +1218,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1227,7 +1227,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1332,7 +1332,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1419,7 +1419,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1428,7 +1428,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1437,7 +1437,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
|
||||
@@ -621,7 +621,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -948,7 +948,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1275,7 +1275,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*metrics-generator\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1602,7 +1602,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1929,7 +1929,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -2256,7 +2256,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})",
|
||||
"expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n) by (limit_name)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\"})\n) by (limit_name)\n",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"intervalFactor": 2,
|
||||
@@ -198,7 +198,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_distributor_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"expr": "sum(rate(tempo_distributor_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -207,7 +207,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -216,7 +216,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -303,7 +303,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_distributor_spans_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"expr": "sum(rate(tempo_distributor_spans_received_total{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -312,7 +312,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_discarded_spans_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)",
|
||||
"expr": "sum(rate(tempo_discarded_spans_total{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -408,7 +408,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(tempo_ingester_live_traces{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",tenant=\"$tenant\"})",
|
||||
"expr": "max(tempo_ingester_live_traces{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",tenant=\"$tenant\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -417,7 +417,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -426,7 +426,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -525,7 +525,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"traces\"}[$__rate_interval])) by (status)",
|
||||
"expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",tenant=\"$tenant\",op=\"traces\"}[$__rate_interval])) by (status)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -612,7 +612,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)",
|
||||
"expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -705,7 +705,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})",
|
||||
"expr": "avg(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",tenant=\"$tenant\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -786,7 +786,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(tempodb_compaction_outstanding_blocks{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})\n/\ncount(tempo_build_info{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n",
|
||||
"expr": "sum(tempodb_compaction_outstanding_blocks{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",tenant=\"$tenant\"})\n/\ncount(tempo_build_info{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\"})\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -879,7 +879,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"expr": "sum(rate(tempo_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/.*metrics-generator\",tenant=\"$tenant\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -970,7 +970,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(tempo_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"})",
|
||||
"expr": "sum(tempo_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=~\"($namespace)/.*metrics-generator\",tenant=\"$tenant\"})",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -979,7 +979,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n",
|
||||
"expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1131,7 +1131,7 @@
|
||||
"options": [
|
||||
|
||||
],
|
||||
"query": "label_values(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"}, tenant)",
|
||||
"query": "label_values(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\"}, tenant)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"sort": 2,
|
||||
|
||||
@@ -399,7 +399,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(tempo_receiver_accepted_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval]))",
|
||||
"expr": "sum(rate(tempo_receiver_accepted_spans{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -408,7 +408,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_receiver_refused_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval]))",
|
||||
"expr": "sum(rate(tempo_receiver_refused_spans{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -495,7 +495,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -504,7 +504,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -513,7 +513,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*distributor\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -618,7 +618,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -705,7 +705,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -714,7 +714,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -723,7 +723,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -828,7 +828,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -915,7 +915,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -924,7 +924,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -933,7 +933,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",method=\"Memcache.Put\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1038,7 +1038,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1125,7 +1125,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1134,7 +1134,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1143,7 +1143,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1248,7 +1248,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1335,7 +1335,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1344,7 +1344,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1353,7 +1353,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",method=\"Memcache.Put\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1458,7 +1458,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1545,7 +1545,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1554,7 +1554,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
@@ -1563,7 +1563,7 @@
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by ()",
|
||||
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by ()",
|
||||
"format": "time_series",
|
||||
"interval": "1m",
|
||||
"intervalFactor": 2,
|
||||
|
||||
53
charts/meta-monitoring/src/rules/loki-rules.yaml
Normal file
53
charts/meta-monitoring/src/rules/loki-rules.yaml
Normal file
@@ -0,0 +1,53 @@
|
||||
groups:
|
||||
- name: loki_rules
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:loki_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:loki_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[1m]))
|
||||
by (cluster, job)
|
||||
record: cluster_job:loki_request_duration_seconds:avg
|
||||
- expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job)
|
||||
record: cluster_job:loki_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:loki_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job)
|
||||
record: cluster_job:loki_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:loki_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:loki_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
||||
/ sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:loki_request_duration_seconds:avg
|
||||
- expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job,
|
||||
route)
|
||||
record: cluster_job_route:loki_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:loki_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:loki_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
||||
job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster,
|
||||
namespace, job, route)
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds:avg
|
||||
- expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate
|
||||
571
charts/meta-monitoring/src/rules/mimir-rules.yaml
Normal file
571
charts/meta-monitoring/src/rules/mimir-rules.yaml
Normal file
@@ -0,0 +1,571 @@
|
||||
groups:
|
||||
- name: mimir_api_1
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
|
||||
by (cluster, job)
|
||||
record: cluster_job:cortex_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
|
||||
record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_api_2
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:cortex_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:cortex_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
||||
/ sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:cortex_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
|
||||
route)
|
||||
record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
||||
record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_api_3
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
||||
job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
|
||||
namespace, job, route)
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
|
||||
job, route)
|
||||
record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_querier_api
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_querier_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_querier_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_querier_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_querier_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, route))
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
|
||||
(cluster, job, route)
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job, route)
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job, route)
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
||||
job, route)
|
||||
record: cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
|
||||
by (cluster, namespace, job, route)
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
namespace, job, route)
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
||||
namespace, job, route)
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
||||
namespace, job, route)
|
||||
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_cache
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, method))
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, method))
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
|
||||
by (cluster, job, method)
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_cache_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_cache_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
/ sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_cache_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_cache_request_duration_seconds_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, method))
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job, method))
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
||||
method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
||||
method)
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
||||
job, method)
|
||||
record: cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_storage
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_kv_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_kv_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
/ sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_kv_request_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
|
||||
- name: mimir_queries
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_query_frontend_retries:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_query_frontend_retries:50quantile
|
||||
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
|
||||
by (cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_retries:avg
|
||||
- expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
|
||||
- expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_retries_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
|
||||
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
||||
job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
|
||||
(cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
|
||||
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
|
||||
cluster, job)
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
|
||||
- name: mimir_ingester_queries
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_series:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_series:50quantile
|
||||
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
|
||||
by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_series:avg
|
||||
- expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_series_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_samples:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_samples:50quantile
|
||||
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
|
||||
by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_samples:avg
|
||||
- expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
|
||||
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_exemplars:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
|
||||
by (le, cluster, job))
|
||||
record: cluster_job:cortex_ingester_queried_exemplars:50quantile
|
||||
- expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
|
||||
sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_exemplars:avg
|
||||
- expr: sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
|
||||
job)
|
||||
record: cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate
|
||||
- expr: sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
|
||||
record: cluster_job:cortex_ingester_queried_exemplars_count:sum_rate
|
||||
- name: mimir_received_samples
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
|
||||
record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
||||
- name: mimir_exemplars_in
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))
|
||||
record: cluster_namespace_job:cortex_distributor_exemplars_in:rate5m
|
||||
- name: mimir_received_exemplars
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))
|
||||
record: cluster_namespace_job:cortex_distributor_received_exemplars:rate5m
|
||||
- name: mimir_exemplars_ingested
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))
|
||||
record: cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m
|
||||
- name: mimir_exemplars_appended
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))
|
||||
record: cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m
|
||||
- name: mimir_scaling_rules
|
||||
rules:
|
||||
- expr: |
|
||||
# Convenience rule to get the number of replicas for both a deployment and a statefulset.
|
||||
# Multi-zone deployments are grouped together removing the "zone-X" suffix.
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
kube_deployment_spec_replicas,
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
or
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
|
||||
)
|
||||
record: cluster_namespace_deployment:actual_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
quantile_over_time(0.99,
|
||||
sum by (cluster, namespace) (
|
||||
cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
||||
)[24h:]
|
||||
)
|
||||
/ 240000
|
||||
)
|
||||
labels:
|
||||
deployment: distributor
|
||||
reason: sample_rate
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
|
||||
* 0.59999999999999998 / 240000
|
||||
)
|
||||
labels:
|
||||
deployment: distributor
|
||||
reason: sample_rate_limits
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
quantile_over_time(0.99,
|
||||
sum by (cluster, namespace) (
|
||||
cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
||||
)[24h:]
|
||||
)
|
||||
* 3 / 80000
|
||||
)
|
||||
labels:
|
||||
deployment: ingester
|
||||
reason: sample_rate
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
quantile_over_time(0.99,
|
||||
sum by(cluster, namespace) (
|
||||
cortex_ingester_memory_series
|
||||
)[24h:]
|
||||
)
|
||||
/ 1500000
|
||||
)
|
||||
labels:
|
||||
deployment: ingester
|
||||
reason: active_series
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
|
||||
* 3 * 0.59999999999999998 / 1500000
|
||||
)
|
||||
labels:
|
||||
deployment: ingester
|
||||
reason: active_series_limits
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
|
||||
* 0.59999999999999998 / 80000
|
||||
)
|
||||
labels:
|
||||
deployment: ingester
|
||||
reason: sample_rate_limits
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
ceil(
|
||||
(sum by (cluster, namespace) (
|
||||
cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
|
||||
) / 4)
|
||||
/
|
||||
avg by (cluster, namespace) (
|
||||
memcached_limit_bytes{job=~".+/memcached"}
|
||||
)
|
||||
)
|
||||
labels:
|
||||
deployment: memcached
|
||||
reason: active_series
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
record: cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate
|
||||
- expr: |
|
||||
# Convenience rule to get the CPU request for both a deployment and a statefulset.
|
||||
# Multi-zone deployments are grouped together removing the "zone-X" suffix.
|
||||
# This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
|
||||
# that remove resource metrics, ref:
|
||||
# - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
|
||||
# - https://github.com/kubernetes/kube-state-metrics/pull/1004
|
||||
#
|
||||
# This is the old expression, compatible with kube-state-metrics < v2.0.0,
|
||||
# where kube_pod_container_resource_requests_cpu_cores was removed:
|
||||
(
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
kube_pod_container_resource_requests_cpu_cores,
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
)
|
||||
or
|
||||
# This expression is compatible with kube-state-metrics >= v1.4.0,
|
||||
# where kube_pod_container_resource_requests was introduced.
|
||||
(
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
kube_pod_container_resource_requests{resource="cpu"},
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
)
|
||||
record: cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
|
||||
- expr: |
|
||||
# Jobs should be sized to their CPU usage.
|
||||
# We do this by comparing 99th percentile usage over the last 24hrs to
|
||||
# their current provisioned #replicas and resource requests.
|
||||
ceil(
|
||||
cluster_namespace_deployment:actual_replicas:count
|
||||
*
|
||||
quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
|
||||
/
|
||||
cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
|
||||
)
|
||||
labels:
|
||||
reason: cpu_usage
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- expr: |
|
||||
# Convenience rule to get the Memory utilization for both a deployment and a statefulset.
|
||||
# Multi-zone deployments are grouped together removing the "zone-X" suffix.
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
container_memory_usage_bytes{image!=""},
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
record: cluster_namespace_deployment:container_memory_usage_bytes:sum
|
||||
- expr: |
|
||||
# Convenience rule to get the Memory request for both a deployment and a statefulset.
|
||||
# Multi-zone deployments are grouped together removing the "zone-X" suffix.
|
||||
# This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
|
||||
# that remove resource metrics, ref:
|
||||
# - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
|
||||
# - https://github.com/kubernetes/kube-state-metrics/pull/1004
|
||||
#
|
||||
# This is the old expression, compatible with kube-state-metrics < v2.0.0,
|
||||
# where kube_pod_container_resource_requests_memory_bytes was removed:
|
||||
(
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
kube_pod_container_resource_requests_memory_bytes,
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
)
|
||||
or
|
||||
# This expression is compatible with kube-state-metrics >= v1.4.0,
|
||||
# where kube_pod_container_resource_requests was introduced.
|
||||
(
|
||||
sum by (cluster, namespace, deployment) (
|
||||
label_replace(
|
||||
label_replace(
|
||||
kube_pod_container_resource_requests{resource="memory"},
|
||||
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
||||
),
|
||||
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
||||
# always matches everything and the (optional) zone is not removed.
|
||||
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
||||
)
|
||||
)
|
||||
)
|
||||
record: cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
|
||||
- expr: |
|
||||
# Jobs should be sized to their Memory usage.
|
||||
# We do this by comparing 99th percentile usage over the last 24hrs to
|
||||
# their current provisioned #replicas and resource requests.
|
||||
ceil(
|
||||
cluster_namespace_deployment:actual_replicas:count
|
||||
*
|
||||
quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
|
||||
/
|
||||
cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
|
||||
)
|
||||
labels:
|
||||
reason: memory_usage
|
||||
record: cluster_namespace_deployment_reason:required_replicas:count
|
||||
- name: mimir_alertmanager_rules
|
||||
rules:
|
||||
- expr: |
|
||||
sum by (cluster, job, pod) (cortex_alertmanager_alerts)
|
||||
record: cluster_job_pod:cortex_alertmanager_alerts:sum
|
||||
- expr: |
|
||||
sum by (cluster, job, pod) (cortex_alertmanager_silences)
|
||||
record: cluster_job_pod:cortex_alertmanager_silences:sum
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_alerts_received_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))
|
||||
record: cluster_job_integration:cortex_alertmanager_notifications_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))
|
||||
record: cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_state_replication_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m
|
||||
- expr: |
|
||||
sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))
|
||||
record: cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m
|
||||
- name: mimir_ingester_rules
|
||||
rules:
|
||||
- expr: |
|
||||
sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))
|
||||
record: cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m
|
||||
15
charts/meta-monitoring/src/rules/tempo-rules.yaml
Normal file
15
charts/meta-monitoring/src/rules/tempo-rules.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
groups:
|
||||
- name: tempo_rules
|
||||
rules:
|
||||
- expr: histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds:99quantile
|
||||
- expr: histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds:50quantile
|
||||
- expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds:avg
|
||||
- expr: sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route)
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds_bucket:sum_rate
|
||||
- expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route)
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds_sum:sum_rate
|
||||
- expr: sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
|
||||
record: cluster_namespace_job_route:tempo_request_duration_seconds_count:sum_rate
|
||||
@@ -37,6 +37,8 @@ data:
|
||||
}
|
||||
}
|
||||
|
||||
// Logs
|
||||
|
||||
{{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.rename_meta_labels.output
|
||||
@@ -58,6 +60,8 @@ data:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
// Metrics
|
||||
|
||||
{{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
|
||||
prometheus.scrape "pods" {
|
||||
targets = discovery.relabel.rename_meta_labels.output
|
||||
@@ -70,8 +74,117 @@ data:
|
||||
forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
// cAdvisor and Kubelete metrics
|
||||
// Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
|
||||
discovery.kubernetes "all_nodes" {
|
||||
role = "node"
|
||||
}
|
||||
|
||||
discovery.relabel "all_nodes" {
|
||||
targets = discovery.kubernetes.all_nodes.targets
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
||||
separator = "/"
|
||||
regex = "(.*)/(.*)/(.*)"
|
||||
replacement = "${1}/${2}-${3}"
|
||||
target_label = "job"
|
||||
}
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = "{{- .Values.clusterName -}}"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "cadvisor" {
|
||||
targets = discovery.relabel.all_nodes.output
|
||||
forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
|
||||
|
||||
scrape_interval = "15s"
|
||||
metrics_path = "/metrics/cadvisor"
|
||||
scheme = "https"
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
tls_config {
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "kubelet" {
|
||||
targets = discovery.relabel.all_nodes.output
|
||||
forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
|
||||
|
||||
scrape_interval = "15s"
|
||||
metrics_path = "/metrics"
|
||||
scheme = "https"
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
tls_config {
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.exporter.unix {}
|
||||
|
||||
prometheus.scrape "node_exporter" {
|
||||
targets = prometheus.exporter.unix.targets
|
||||
forward_to = [prometheus.relabel.node_exporter.receiver]
|
||||
|
||||
job_name = "node-exporter"
|
||||
scrape_interval = "15s"
|
||||
}
|
||||
|
||||
prometheus.relabel "node_exporter" {
|
||||
forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
|
||||
|
||||
rule {
|
||||
replacement = env("HOSTNAME")
|
||||
target_label = "nodename"
|
||||
}
|
||||
rule {
|
||||
replacement = "node-exporter"
|
||||
target_label = "job"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
||||
separator = "/"
|
||||
regex = "(.*)/(.*)/(.*)"
|
||||
replacement = "${1}/${2}-${3}"
|
||||
target_label = "job"
|
||||
}
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = "{{- .Values.clusterName -}}"
|
||||
}
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
// Traces
|
||||
|
||||
{{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
|
||||
// Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
|
||||
otelcol.receiver.otlp "otlp_receiver" {
|
||||
|
||||
126
charts/meta-monitoring/templates/ruler/ruler.yaml
Normal file
126
charts/meta-monitoring/templates/ruler/ruler.yaml
Normal file
@@ -0,0 +1,126 @@
|
||||
{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: meta-mimir-ruler-for-dashboards
|
||||
namespace: meta
|
||||
spec:
|
||||
progressDeadlineSeconds: 600
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: ruler-for-dashboards
|
||||
app.kubernetes.io/instance: meta
|
||||
app.kubernetes.io/name: mimir
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 50%
|
||||
maxUnavailable: 0
|
||||
type: RollingUpdate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: ruler-for-dashboards
|
||||
app.kubernetes.io/instance: meta
|
||||
app.kubernetes.io/name: mimir
|
||||
namespace: meta
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- -target=ruler
|
||||
- -log.level=debug
|
||||
- -ruler-storage.backend=local
|
||||
- -ruler-storage.local.directory=/etc/rules
|
||||
- -ruler.ring.prefix=dashboards/
|
||||
- -config.expand-env=true
|
||||
- -config.file=/etc/mimir/mimir.yaml
|
||||
image: grafana/mimir:2.8.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: ruler
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http-metrics
|
||||
protocol: TCP
|
||||
- containerPort: 9095
|
||||
name: grpc
|
||||
protocol: TCP
|
||||
- containerPort: 7946
|
||||
name: memberlist
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http-metrics
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 45
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /etc/mimir
|
||||
name: config
|
||||
- mountPath: /var/mimir
|
||||
name: runtime-config
|
||||
- mountPath: /data
|
||||
name: storage
|
||||
- mountPath: /active-query-tracker
|
||||
name: active-queries
|
||||
- mountPath: /etc/rules/anonymous
|
||||
name: rules
|
||||
dnsPolicy: ClusterFirst
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext:
|
||||
fsGroup: 10001
|
||||
runAsGroup: 10001
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
serviceAccount: meta-mimir
|
||||
serviceAccountName: meta-mimir
|
||||
terminationGracePeriodSeconds: 180
|
||||
topologySpreadConstraints:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: ruler
|
||||
app.kubernetes.io/instance: meta
|
||||
app.kubernetes.io/name: mimir
|
||||
maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
volumes:
|
||||
- configMap:
|
||||
defaultMode: 420
|
||||
items:
|
||||
- key: mimir.yaml
|
||||
path: mimir.yaml
|
||||
name: meta-mimir-config
|
||||
name: config
|
||||
- configMap:
|
||||
defaultMode: 420
|
||||
name: meta-mimir-runtime
|
||||
name: runtime-config
|
||||
- emptyDir: {}
|
||||
name: storage
|
||||
- emptyDir: {}
|
||||
name: active-queries
|
||||
- configMap:
|
||||
defaultMode: 420
|
||||
name: rules
|
||||
name: rules
|
||||
{{- end }}
|
||||
18
charts/meta-monitoring/templates/ruler/rules-configmap.yaml
Normal file
18
charts/meta-monitoring/templates/ruler/rules-configmap.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: rules
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
data:
|
||||
{{- if .Values.dashboards.logs.enabled }}
|
||||
{{ ($.Files.Glob "src/rules/loki-rules.yaml").AsConfig | indent 2 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dashboards.metrics.enabled }}
|
||||
{{ ($.Files.Glob "src/rules/mimir-rules.yaml").AsConfig | indent 2 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dashboards.traces.enabled }}
|
||||
{{ ($.Files.Glob "src/rules/tempo-rules.yaml").AsConfig | indent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user