Public/grafana-meta-monitoring-chart
2
0
Fork 0
forked from RemoteSync/grafana-meta-monitoring-chart
Code Pull Requests Activity

Merge pull request #36 from grafana/add_more_metrics

Browse Source 
Use new Loki dashboards
This commit is contained in:
Michel Hollands 2024-04-11 10:09:50 +01:00 committed by GitHub
commit f275b2d1b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 3589 additions and 3667 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1249
charts/meta-monitoring/src/dashboards/loki-chunks.json
View File

File diff suppressed because it is too large Load Diff

764
charts/meta-monitoring/src/dashboards/loki-deletion.json
View File

@ -62,7 +62,6 @@
"expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})", "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
"format": "time_series", "format": "time_series",
"instant": true, "instant": true,
"intervalFactor": 2,
"refId": "A" "refId": "A"
} }
], ],
@ -138,7 +137,6 @@
"expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})", "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
"format": "time_series", "format": "time_series",
"instant": true, "instant": true,
"intervalFactor": 2,
"refId": "A" "refId": "A"
} }
], ],
@ -191,232 +189,145 @@
"height": "250px", "height": "250px",
"panels": [ "panels": [
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 3, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 3,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))", "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "in progress", "legendFormat": "in progress",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "# of Delete Requests (received - processed) ", "title": "# of Delete Requests (received - processed) ",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 4, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 4,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "received", "legendFormat": "received",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Delete Requests Received / Day", "title": "Delete Requests Received / Day",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 5, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 5,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "processed", "legendFormat": "processed",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Delete Requests Processed / Day", "title": "Delete Requests Processed / Day",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
} }
], ],
"repeat": null, "repeat": null,
@ -431,232 +342,145 @@
"height": "250px", "height": "250px",
"panels": [ "panels": [
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 6, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 6,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}", "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Compactor CPU usage", "title": "Compactor CPU usage",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 7, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 7,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": " {{pod}} ", "legendFormat": " {{pod}} ",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Compactor memory usage (MiB)", "title": "Compactor memory usage (MiB)",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 8, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 8,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 4, "span": 4,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Compaction run duration (seconds)", "title": "Compaction run duration (seconds)",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
} }
], ],
"repeat": null, "repeat": null,
@ -671,156 +495,98 @@
"height": "250px", "height": "250px",
"panels": [ "panels": [
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 9, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 9,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 6, "span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "failures", "legendFormat": "failures",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Failures in Loading Delete Requests / Hour", "title": "Failures in Loading Delete Requests / Hour",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 10, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 10,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 6, "span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{user}}", "legendFormat": "{{user}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Lines Deleted / Sec", "title": "Lines Deleted / Sec",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
} }
], ],
"repeat": null, "repeat": null,
@ -882,7 +648,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",

28
charts/meta-monitoring/src/dashboards/loki-logs.json
View File

@ -77,7 +77,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -164,7 +164,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -250,7 +250,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -336,7 +336,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -422,7 +422,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -508,7 +508,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -595,7 +595,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -682,7 +682,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -718,7 +718,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": "$logs", "datasource": "$loki_datasource",
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
@ -787,7 +787,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -819,7 +819,7 @@
} }
}, },
{ {
"datasource": "$logs", "datasource": "$loki_datasource",
"gridPos": { "gridPos": {
"h": 19, "h": 19,
"w": 24, "w": 24,
@ -861,7 +861,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",
@ -918,7 +918,7 @@
{ {
"hide": 0, "hide": 0,
"label": null, "label": null,
"name": "logs", "name": "loki_datasource",
"options": [ ], "options": [ ],
"query": "loki", "query": "loki",
"refresh": 1, "refresh": 1,
@ -1069,4 +1069,4 @@
"title": "Loki / Logs", "title": "Loki / Logs",
"uid": "logs", "uid": "logs",
"version": 0 "version": 0
} }

2
charts/meta-monitoring/src/dashboards/loki-mixin-recording-rules.json
View File

@ -600,7 +600,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",

160
charts/meta-monitoring/src/dashboards/loki-operational.json
View File

@ -102,7 +102,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -198,7 +198,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -293,7 +293,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -389,7 +389,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -485,7 +485,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -591,7 +591,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -697,7 +697,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -794,7 +794,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -903,7 +903,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1000,7 +1000,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1109,7 +1109,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1215,7 +1215,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1312,7 +1312,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1420,7 +1420,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1517,7 +1517,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1618,7 +1618,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1799,9 +1799,9 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))",
"intervalFactor": 3, "intervalFactor": 3,
"legendFormat": "{{pod}}-{{container}}", "legendFormat": "{{pod}}",
"refId": "A" "refId": "A"
} }
], ],
@ -1815,7 +1815,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1907,7 +1907,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -1943,7 +1943,7 @@
"bars": true, "bars": true,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": "$logs", "datasource": "$loki_datasource",
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
@ -1999,7 +1999,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2031,7 +2031,7 @@
} }
}, },
{ {
"datasource": "$logs", "datasource": "$loki_datasource",
"gridPos": { "gridPos": {
"h": 18, "h": 18,
"w": 12, "w": 12,
@ -2116,7 +2116,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2205,7 +2205,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2294,7 +2294,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2383,7 +2383,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2490,7 +2490,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2581,7 +2581,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2697,7 +2697,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -2850,7 +2850,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3006,9 +3006,9 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))",
"intervalFactor": 3, "intervalFactor": 3,
"legendFormat": "{{pod}}-{{container}}", "legendFormat": "{{pod}}",
"refId": "A" "refId": "A"
} }
], ],
@ -3022,7 +3022,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3114,7 +3114,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3150,7 +3150,7 @@
"bars": true, "bars": true,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": "$logs", "datasource": "$loki_datasource",
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
@ -3206,7 +3206,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3238,7 +3238,7 @@
} }
}, },
{ {
"datasource": "$logs", "datasource": "$loki_datasource",
"gridPos": { "gridPos": {
"h": 18, "h": 18,
"w": 18, "w": 18,
@ -3323,7 +3323,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3443,7 +3443,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3535,7 +3535,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3655,7 +3655,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3747,7 +3747,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3861,7 +3861,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -3964,7 +3964,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4067,7 +4067,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4165,7 +4165,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4255,7 +4255,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4345,7 +4345,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4435,7 +4435,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4525,7 +4525,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4645,7 +4645,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4737,7 +4737,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4825,7 +4825,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
"refId": "A" "refId": "A"
} }
], ],
@ -4839,7 +4839,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4911,7 +4911,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
"refId": "A" "refId": "A"
} }
], ],
@ -4925,7 +4925,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -4997,7 +4997,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
"refId": "A" "refId": "A"
} }
], ],
@ -5011,7 +5011,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5083,7 +5083,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
"refId": "A" "refId": "A"
} }
], ],
@ -5097,7 +5097,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5169,17 +5169,17 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
"legendFormat": ".99", "legendFormat": ".99",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
"legendFormat": ".9", "legendFormat": ".9",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
"legendFormat": ".5", "legendFormat": ".5",
"refId": "C" "refId": "C"
} }
@ -5194,7 +5194,7 @@
"sort": 0, "sort": 0,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5269,19 +5269,19 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
"intervalFactor": 1, "intervalFactor": 1,
"legendFormat": ".99-{{operation}}", "legendFormat": ".99-{{operation}}",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
"hide": false, "hide": false,
"legendFormat": ".9-{{operation}}", "legendFormat": ".9-{{operation}}",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
"hide": false, "hide": false,
"legendFormat": ".5-{{operation}}", "legendFormat": ".5-{{operation}}",
"refId": "C" "refId": "C"
@ -5297,7 +5297,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5372,7 +5372,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
"intervalFactor": 1, "intervalFactor": 1,
"legendFormat": "{{status_code}}-{{operation}}", "legendFormat": "{{status_code}}-{{operation}}",
"refId": "A" "refId": "A"
@ -5388,7 +5388,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5507,7 +5507,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5598,7 +5598,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5717,7 +5717,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5808,7 +5808,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -5927,7 +5927,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -6018,7 +6018,7 @@
"sort": 2, "sort": 2,
"value_type": "individual" "value_type": "individual"
}, },
"type": "graph", "type": "timeseries",
"xaxis": { "xaxis": {
"buckets": null, "buckets": null,
"mode": "time", "mode": "time",
@ -6070,7 +6070,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",
@ -6081,7 +6081,7 @@
{ {
"hide": 0, "hide": 0,
"label": null, "label": null,
"name": "logs", "name": "loki_datasource",
"options": [ ], "options": [ ],
"query": "loki", "query": "loki",
"refresh": 1, "refresh": 1,
@ -6169,4 +6169,4 @@
"title": "Loki / Operational", "title": "Loki / Operational",
"uid": "operational", "uid": "operational",
"version": 0 "version": 0
} }

1057
charts/meta-monitoring/src/dashboards/loki-reads-resources.json
View File

File diff suppressed because it is too large Load Diff

734
charts/meta-monitoring/src/dashboards/loki-reads.json
View File

@ -33,113 +33,234 @@
"3xx": "#6ED0E0", "3xx": "#6ED0E0",
"4xx": "#EF843C", "4xx": "#EF843C",
"5xx": "#E24D42", "5xx": "#E24D42",
"OK": "#7EB26D",
"cancel": "#A9A9A9",
"error": "#E24D42", "error": "#E24D42",
"success": "#7EB26D" "success": "#7EB26D"
}, },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 100,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
}
},
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "1xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "2xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6ED0E0",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EF843C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "OK"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "cancel"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#A9A9A9",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
}
]
},
"fill": 10, "fill": 10,
"id": 1, "id": 1,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0, "linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
},
"span": 4,
"stack": true, "stack": true,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{status}}", "legendFormat": "{{status}}",
"refId": "A", "refId": "A"
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "QPS", "title": "QPS",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 2, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
}, },
"lines": true, "id": 2,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"percentage": false, "options": {
"pointradius": 5, "legend": {
"points": false, "showLegend": true
"renderer": "flot", },
"seriesOverrides": [ ], "tooltip": {
"spaceLength": 10, "mode": "single",
"span": 6, "sort": "none"
"stack": false, }
"steppedLine": false, },
"span": 4,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3",
@ -166,23 +287,8 @@
"step": 10 "step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Latency", "title": "Latency",
"tooltip": { "type": "timeseries",
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [ "yaxes": [
{ {
"format": "ms", "format": "ms",
@ -201,6 +307,57 @@
"show": false "show": false
} }
] ]
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
},
"id": 3,
"links": [ ],
"nullPointMode": "null as zero",
"options": {
"legend": {
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"span": 4,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])) by (le,pod)) * 1e3",
"format": "time_series",
"interval": "1m",
"intervalFactor": 2,
"legendFormat": "__auto",
"refId": "A",
"step": 10
}
],
"title": "Per Pod Latency (p99)",
"type": "timeseries"
} }
], ],
"repeat": null, "repeat": null,
@ -221,156 +378,256 @@
"3xx": "#6ED0E0", "3xx": "#6ED0E0",
"4xx": "#EF843C", "4xx": "#EF843C",
"5xx": "#E24D42", "5xx": "#E24D42",
"OK": "#7EB26D",
"cancel": "#A9A9A9",
"error": "#E24D42", "error": "#E24D42",
"success": "#7EB26D" "success": "#7EB26D"
}, },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 10, "fieldConfig": {
"id": 3, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 100,
"max": false, "lineWidth": 0,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "normal"
}
},
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "1xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "2xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6ED0E0",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EF843C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "OK"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "cancel"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#A9A9A9",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
}
]
}, },
"lines": true, "fill": 10,
"id": 4,
"linewidth": 0, "linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
},
"span": 4,
"stack": true, "stack": true,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{status}}", "legendFormat": "{{status}}",
"refId": "A", "refId": "A"
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "QPS", "title": "QPS",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 4, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
}, },
"lines": true, "id": 5,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"percentage": false, "options": {
"pointradius": 5, "legend": {
"points": false, "showLegend": true
"renderer": "flot", },
"seriesOverrides": [ ], "tooltip": {
"spaceLength": 10, "mode": "single",
"span": 6, "sort": "none"
"stack": false, }
"steppedLine": false, },
"span": 4,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "99th Percentile", "legendFormat": "99th Percentile",
"refId": "A", "refId": "A"
"step": 10
}, },
{ {
"expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "50th Percentile", "legendFormat": "50th Percentile",
"refId": "B", "refId": "B"
"step": 10
}, },
{ {
"expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "Average", "legendFormat": "Average",
"refId": "C", "refId": "C"
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Latency", "title": "Latency",
"tooltip": { "type": "timeseries",
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [ "yaxes": [
{ {
"format": "ms", "format": "ms",
@ -389,6 +646,57 @@
"show": false "show": false
} }
] ]
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
},
"id": 6,
"links": [ ],
"nullPointMode": "null as zero",
"options": {
"legend": {
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"span": 4,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3",
"format": "time_series",
"interval": "1m",
"intervalFactor": 2,
"legendFormat": "__auto",
"refId": "A",
"step": 10
}
],
"title": "Per Pod Latency (p99)",
"type": "timeseries"
} }
], ],
"repeat": null, "repeat": null,
@ -412,7 +720,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",

1741
charts/meta-monitoring/src/dashboards/loki-retention.json
View File

File diff suppressed because it is too large Load Diff

762
charts/meta-monitoring/src/dashboards/loki-writes-resources.json
View File

@ -27,589 +27,447 @@
"collapsed": false, "collapsed": false,
"panels": [ "panels": [
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": { }, "gridPos": { },
"id": 1, "id": 1,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
"stack": false, },
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "In-memory streams", "title": "In-memory streams",
"tooltip": { "tooltip": {
"sort": 2 "sort": 2
}, },
"type": "graph", "type": "timeseries"
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "request"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#FFC000",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "limit"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E02F44",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
}
]
}
]
},
"gridPos": { }, "gridPos": { },
"id": 2, "id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "request",
"color": "#FFC000",
"fill": 0
}, },
{ "tooltip": {
"alias": "limit", "mode": "single",
"color": "#E02F44", "sort": "none"
"fill": 0
} }
], },
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
}, },
{ {
"expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "request", "legendFormat": "request",
"legendLink": null, "legendLink": null
"step": 10
}, },
{ {
"expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "limit", "legendFormat": "limit",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "CPU", "title": "CPU",
"tooltip": { "tooltip": {
"sort": 2 "sort": 2
}, },
"type": "graph", "type": "timeseries"
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "request"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#FFC000",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "limit"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E02F44",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
}
]
}
]
},
"gridPos": { }, "gridPos": { },
"id": 3, "id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "request",
"color": "#FFC000",
"fill": 0
}, },
{ "tooltip": {
"alias": "limit", "mode": "single",
"color": "#E02F44", "sort": "none"
"fill": 0
} }
], },
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
}, },
{ {
"expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "request", "legendFormat": "request",
"legendLink": null, "legendLink": null
"step": 10
}, },
{ {
"expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "limit", "legendFormat": "limit",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Memory (workingset)", "title": "Memory (workingset)",
"tooltip": { "tooltip": {
"sort": 2 "sort": 2
}, },
"type": "graph", "type": "timeseries"
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "bytes"
},
"overrides": [ ]
},
"gridPos": { }, "gridPos": { },
"id": 4, "id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
"stack": false, },
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Memory (go heap inuse)", "title": "Memory (go heap inuse)",
"tooltip": { "tooltip": {
"sort": 2 "sort": 2
}, },
"type": "graph", "type": "timeseries"
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 10, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 100,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "Bps"
},
"overrides": [ ]
},
"gridPos": { }, "gridPos": { },
"id": 5, "id": 5,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
"stack": true, },
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}", "legendFormat": "{{pod}} - {{device}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Disk Writes", "title": "Disk Writes",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "Bps",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 10, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 100,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "Bps"
},
"overrides": [ ]
},
"gridPos": { }, "gridPos": { },
"id": 6, "id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
"stack": true, },
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}", "legendFormat": "{{pod}} - {{device}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Disk Reads", "title": "Disk Reads",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "Bps",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "percentunit"
},
"overrides": [ ]
},
"gridPos": { }, "gridPos": { },
"id": 7, "id": 7,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
"span": 6, }
"stack": false, },
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{persistentvolumeclaim}}", "legendFormat": "{{persistentvolumeclaim}}",
"legendLink": null, "legendLink": null
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Disk Space Utilization", "title": "Disk Space Utilization",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
} }
], ],
"repeat": null, "repeat": null,
@ -634,7 +492,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",

745
charts/meta-monitoring/src/dashboards/loki-writes.json
View File

@ -33,116 +33,237 @@
"3xx": "#6ED0E0", "3xx": "#6ED0E0",
"4xx": "#EF843C", "4xx": "#EF843C",
"5xx": "#E24D42", "5xx": "#E24D42",
"OK": "#7EB26D",
"cancel": "#A9A9A9",
"error": "#E24D42", "error": "#E24D42",
"success": "#7EB26D" "success": "#7EB26D"
}, },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 100,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
}
},
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "1xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "2xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6ED0E0",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EF843C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "OK"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "cancel"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#A9A9A9",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
}
]
},
"fill": 10, "fill": 10,
"id": 1, "id": 1,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0, "linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 6, "span": 6,
"stack": true, "stack": true,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{status}}", "legendFormat": "{{status}}",
"refId": "A", "refId": "A"
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "QPS", "title": "QPS",
"tooltip": { "type": "timeseries"
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 2, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
}, },
"lines": true, "id": 2,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"percentage": false, "options": {
"pointradius": 5, "legend": {
"points": false, "showLegend": true
"renderer": "flot", },
"seriesOverrides": [ ], "tooltip": {
"spaceLength": 10, "mode": "single",
"sort": "none"
}
},
"span": 6, "span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})) * 1e3", "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2, "intervalFactor": 2,
"legendFormat": "99th Percentile", "legendFormat": "99th Percentile",
@ -150,7 +271,7 @@
"step": 10 "step": 10
}, },
{ {
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})) * 1e3", "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2, "intervalFactor": 2,
"legendFormat": "50th Percentile", "legendFormat": "50th Percentile",
@ -158,7 +279,7 @@
"step": 10 "step": 10
}, },
{ {
"expr": "1e3 * sum(cluster_job:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}) / sum(cluster_job:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})",
"format": "time_series", "format": "time_series",
"intervalFactor": 2, "intervalFactor": 2,
"legendFormat": "Average", "legendFormat": "Average",
@ -166,23 +287,8 @@
"step": 10 "step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Latency", "title": "Latency",
"tooltip": { "type": "timeseries",
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [ "yaxes": [
{ {
"format": "ms", "format": "ms",
@ -215,76 +321,105 @@
"height": "250px", "height": "250px",
"panels": [ "panels": [
{ {
"aliasColors": {
"1xx": "#EAB839",
"2xx": "#7EB26D",
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
"error": "#E24D42",
"success": "#7EB26D"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 10, "fieldConfig": {
"id": 3, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"lines": true, "id": 3,
"linewidth": 0,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "options": {
"percentage": false, "legend": {
"pointradius": 5, "showLegend": true
"points": false, },
"renderer": "flot", "tooltip": {
"seriesOverrides": [ ], "mode": "single",
"spaceLength": 10, "sort": "none"
}
},
"span": 6, "span": 6,
"stack": true,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2, "legendFormat": "bytes",
"legendFormat": "{{status}}", "legendLink": null
"refId": "A",
"step": 10
} }
], ],
"thresholds": [ ], "title": "Per Total Received Bytes",
"timeFrom": null, "type": "timeseries"
"timeShift": null, },
"title": "QPS", {
"tooltip": { "datasource": "$datasource",
"shared": true, "fieldConfig": {
"sort": 2, "defaults": {
"value_type": "individual" "custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [ ]
}, },
"type": "graph", "id": 4,
"xaxis": { "links": [ ],
"buckets": null, "options": {
"mode": "time", "legend": {
"name": null, "showLegend": true
"show": true, },
"values": [ ] "tooltip": {
"mode": "single",
"sort": "none"
}
}, },
"span": 6,
"stack": true,
"targets": [
{
"expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))",
"format": "time_series",
"legendFormat": "{{tenant}}",
"legendLink": null
}
],
"title": "Per Tenant",
"type": "timeseries",
"yaxes": [ "yaxes": [
{ {
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": 1,
"min": 0, "min": 0,
"show": true "show": true
}, },
@ -292,85 +427,281 @@
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": 1,
"min": null, "min": null,
"show": false "show": false
} }
] ]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Write Path",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {
"1xx": "#EAB839",
"2xx": "#7EB26D",
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
"OK": "#7EB26D",
"cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 100,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
}
},
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "1xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "2xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6ED0E0",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EF843C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "OK"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "cancel"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#A9A9A9",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E24D42",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
}
]
},
"fill": 10,
"id": 5,
"linewidth": 0,
"links": [ ],
"options": {
"legend": {
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"span": 6,
"stack": true,
"targets": [
{
"expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"legendFormat": "{{status}}",
"refId": "A"
}
],
"title": "QPS",
"type": "timeseries"
}, },
{ {
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource", "datasource": "$datasource",
"fill": 1, "fieldConfig": {
"id": 4, "defaults": {
"legend": { "custom": {
"avg": false, "drawStyle": "line",
"current": false, "fillOpacity": 10,
"max": false, "lineWidth": 1,
"min": false, "pointSize": 5,
"show": true, "showPoints": "never",
"total": false, "spanNulls": false,
"values": false "stacking": {
"group": "A",
"mode": "none"
}
},
"thresholds": {
"mode": "absolute",
"steps": [ ]
},
"unit": "ms"
},
"overrides": [ ]
}, },
"lines": true, "id": 6,
"linewidth": 1,
"links": [ ], "links": [ ],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"percentage": false, "options": {
"pointradius": 5, "legend": {
"points": false, "showLegend": true
"renderer": "flot", },
"seriesOverrides": [ ], "tooltip": {
"spaceLength": 10, "mode": "single",
"sort": "none"
}
},
"span": 6, "span": 6,
"stack": false,
"steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "99th Percentile", "legendFormat": "99th Percentile",
"refId": "A", "refId": "A"
"step": 10
}, },
{ {
"expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "50th Percentile", "legendFormat": "50th Percentile",
"refId": "B", "refId": "B"
"step": 10
}, },
{ {
"expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"intervalFactor": 2,
"legendFormat": "Average", "legendFormat": "Average",
"refId": "C", "refId": "C"
"step": 10
} }
], ],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Latency", "title": "Latency",
"tooltip": { "type": "timeseries",
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [ "yaxes": [
{ {
"format": "ms", "format": "ms",
@ -412,7 +743,7 @@
"value": "default" "value": "default"
}, },
"hide": 0, "hide": 0,
"label": "Data Source", "label": "Data source",
"name": "datasource", "name": "datasource",
"options": [ ], "options": [ ],
"query": "prometheus", "query": "prometheus",

14
charts/meta-monitoring/values.yaml
View File

@ -93,6 +93,12 @@ metrics:
- container_spec_cpu_quota - container_spec_cpu_quota
- container_spec_memory_limit_bytes - container_spec_memory_limit_bytes
- cortex_ingester_flush_queue_length - cortex_ingester_flush_queue_length
- cortex_prometheus_rule_group_iterations_total
- cortex_prometheus_rule_evaluation_failures_total
- cortex_prometheus_rule_group_rules
- cortex_prometheus_rule_group_last_duration_seconds
- cortex_prometheus_rule_group_last_evaluation_timestamp_seconds
- cortex_prometheus_rule_group_iterations_missed_total
- go_gc_duration_seconds - go_gc_duration_seconds
- go_goroutines - go_goroutines
- go_memstats_heap_inuse_bytes - go_memstats_heap_inuse_bytes
@ -111,11 +117,14 @@ metrics:
- loki_boltdb_shipper_retention_marker_table_processed_total - loki_boltdb_shipper_retention_marker_table_processed_total
- loki_boltdb_shipper_request_duration_seconds_bucket - loki_boltdb_shipper_request_duration_seconds_bucket
- loki_boltdb_shipper_request_duration_seconds_count - loki_boltdb_shipper_request_duration_seconds_count
- loki_boltdb_shipper_request_duration_seconds_sum
- loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket
- loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count
- loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum
- loki_boltdb_shipper_retention_sweeper_marker_files_current - loki_boltdb_shipper_retention_sweeper_marker_files_current
- loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time - loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time
- loki_build_info - loki_build_info
- loki_chunk_store_index_entries_per_chunk_bucket
- loki_chunk_store_index_entries_per_chunk_count - loki_chunk_store_index_entries_per_chunk_count
- loki_chunk_store_index_entries_per_chunk_sum - loki_chunk_store_index_entries_per_chunk_sum
- loki_compactor_delete_requests_processed_total - loki_compactor_delete_requests_processed_total
@ -123,7 +132,9 @@ metrics:
- loki_compactor_deleted_lines - loki_compactor_deleted_lines
- loki_compactor_oldest_pending_delete_request_age_seconds - loki_compactor_oldest_pending_delete_request_age_seconds
- loki_compactor_pending_delete_requests_count - loki_compactor_pending_delete_requests_count
- loki_distributor_bytes_received_total
- loki_distributor_lines_received_total - loki_distributor_lines_received_total
- loki_distributor_structured_metadata_bytes_received_total
- loki_ingester_chunk_age_seconds_bucket - loki_ingester_chunk_age_seconds_bucket
- loki_ingester_chunk_age_seconds_count - loki_ingester_chunk_age_seconds_count
- loki_ingester_chunk_age_seconds_sum - loki_ingester_chunk_age_seconds_sum
@ -137,9 +148,12 @@ metrics:
- loki_ingester_chunk_utilization_bucket - loki_ingester_chunk_utilization_bucket
- loki_ingester_chunk_utilization_sum - loki_ingester_chunk_utilization_sum
- loki_ingester_chunks_flushed_total - loki_ingester_chunks_flushed_total
- loki_ingester_flush_queue_length
- loki_ingester_memory_chunks - loki_ingester_memory_chunks
- loki_ingester_memory_streams - loki_ingester_memory_streams
- loki_request_duration_seconds_bucket
- loki_request_duration_seconds_count - loki_request_duration_seconds_count
- loki_request_duration_seconds_sum
- loki_ruler_wal_appender_ready - loki_ruler_wal_appender_ready
- loki_ruler_wal_disk_size - loki_ruler_wal_disk_size
- loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds - loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds