{ "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "loki" ], "targetBlank": false, "title": "Loki Dashboards", "type": "dashboards" } ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 1, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 2, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 3, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Query Frontend", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 4, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 5, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 6, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*query-scheduler\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Query Scheduler", "titleSize": "h6" }, { "collapse": false, "collapsed": false, "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 7, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 8, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "gridPos": { }, "id": 9, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 10, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Writes", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 11, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Reads", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "percentunit" }, "overrides": [ ] }, "gridPos": { }, "id": 12, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"querier.*\"})", "format": "time_series", "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null } ], "title": "Disk Space Utilization", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Querier", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 13, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 14, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "gridPos": { }, "id": 15, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 16, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Writes", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 17, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|index-gateway\", pod=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary)\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Reads", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "percentunit" }, "overrides": [ ] }, "gridPos": { }, "id": 18, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(.*index-gateway.*|(loki|enterprise-logs)-read.*|loki-single-binary).*\"})", "format": "time_series", "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null } ], "title": "Disk Space Utilization", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Index Gateway", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 19, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 20, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "gridPos": { }, "id": 21, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/.*bloom-gateway\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 22, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Writes", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "gridPos": { }, "id": 23, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk Reads", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "percentunit" }, "overrides": [ ] }, "gridPos": { }, "id": 24, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-gateway.*\"})", "format": "time_series", "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null } ], "title": "Disk Space Utilization", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Bloom Gateway", "titleSize": "h6", "type": "row" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 25, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "id": 26, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"loki|ingester\", pod=~\"(.*ingester.*|(loki|enterprise-logs)-write.*|loki-single-binary)\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 27, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(.*ingester.*|(loki|enterprise-logs)-write|loki-single-binary)\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ingester", "titleSize": "h6" }, { "collapse": false, "collapsed": false, "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "gridPos": { }, "id": 28, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (loki_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}) or sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Rules", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 29, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"cpu\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "CPU", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "request" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFC000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { }, "id": 30, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"memory\"} > 0)", "format": "time_series", "legendFormat": "request", "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} > 0)", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Memory (workingset)", "tooltip": { "sort": 2 }, "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "gridPos": { }, "id": 31, "interval": "1m", "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ruler", "titleSize": "h6", "type": "row" } ], "schemaVersion": 14, "style": "dark", "tags": [ "loki" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": null, "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "cluster", "multi": false, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", "refresh": 1, "regex": "", "sort": 2, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "namespace", "multi": false, "name": "namespace", "options": [ ], "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "utc", "title": "Loki / Reads Resources", "uid": "reads-resources", "version": 0 }