{ "__inputs": [ ], "__requires": [ ], "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "id": null, "links": [ ], "refresh": "30s", "rows": [ { "collapse": false, "collapsed": false, "height": 500, "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { }, "id": 2, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (rate(promtail_dropped_bytes_total{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Dropped bytes rate [B/s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { }, "id": 3, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (rate(promtail_request_duration_seconds_bucket{status_code=~\"2..\", cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n/\nsum by($groupBy) (rate(promtail_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n* 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Write requests success rate [%]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "%", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "%", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Errors", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "height": 500, "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { }, "id": 4, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(\n 0.990000, \n sum by (le, $groupBy)\n (rate(promtail_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Write latencies p99 [s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { }, "id": 5, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(\n 0.900000, \n sum by (le, $groupBy)\n (rate(promtail_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Write latencies p90 [s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { }, "id": 6, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(\n 0.500000, \n sum by (le, $groupBy)\n (rate(promtail_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Write latencies p50 [s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { }, "id": 7, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "(sum by (le, $groupBy) (rate(promtail_request_duration_seconds_sum{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval])))\n/\n(sum by (le, $groupBy) (rate(promtail_request_duration_seconds_count{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval])))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Write latencies average [s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Latencies", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "height": 500, "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { }, "id": 8, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (rate(promtail_read_bytes_total{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Bytes read rate [B/s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { }, "id": 9, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (rate(promtail_read_lines_total{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Lines read rate [lines/s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { }, "id": 10, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (promtail_files_active_total{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"})\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Active files count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { }, "id": 11, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by($groupBy) (rate(promtail_sent_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}[$__rate_interval]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{$groupBy}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Entries sent rate [entries/s]", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Logs volume", "titleSize": "h6", "type": "row" } ], "schemaVersion": 14, "style": "dark", "tags": [ "grafana-agent-mixin" ], "templating": { "list": [ { "hide": 0, "label": null, "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": null, "current": { "text": { "selected": true, "text": "All", "value": "$__all" }, "value": { "selected": true, "text": "All", "value": "$__all" } }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "cluster", "options": [ ], "query": "label_values(agent_build_info, cluster)", "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": { "selected": true, "text": "All", "value": "$__all" }, "value": { "selected": true, "text": "All", "value": "$__all" } }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [ ], "query": "label_values(agent_build_info, namespace)", "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": { "selected": true, "text": "All", "value": "$__all" }, "value": { "selected": true, "text": "All", "value": "$__all" } }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "container", "options": [ ], "query": "label_values(agent_build_info, container)", "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": { "selected": true, "text": "All", "value": "$__all" }, "value": { "selected": true, "text": "All", "value": "$__all" } }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "pod", "options": [ ], "query": "label_values(agent_build_info{container=~\"$container\"}, pod)", "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": "pod", "value": "pod" }, "hide": 0, "includeAll": false, "label": "", "multi": false, "name": "groupBy", "options": [ { "text": "pod", "value": "pod" }, { "text": "cluster", "value": "cluster" }, { "text": "namespace", "value": "namespace" } ], "query": "pod,cluster,namespace", "refresh": 0, "type": "custom" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Agent Logs Pipeline", "version": 0 }