{
 "annotations": {
  "list": [

  ]
 },
 "editable": true,
 "gnetId": null,
 "graphTooltip": 0,
 "hideControls": false,
 "links": [
  {
   "asDropdown": true,
   "icon": "external link",
   "includeVars": true,
   "keepTime": true,
   "tags": [
    "tempo"
   ],
   "targetBlank": false,
   "title": "Tempo Dashboards",
   "type": "dashboards"
  }
 ],
 "refresh": "",
 "rows": [
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 1,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"api_.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 2,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (route)",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Gateway",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 3,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 4,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*query-frontend\",route=~\"api_.*\"}[$__interval])) by (route)",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Query Frontend",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 5,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 6,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",route=~\"querier_api_.*\"}[$__interval])) by (route)",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Querier",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 7,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 8,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (le,endpoint)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (le,endpoint)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\"}[$__interval])) by (endpoint)",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Querier External Endpoint",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 9,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 10,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (le,route)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*ingester\",route=~\"/tempopb.Querier/.*\"}[$__interval])) by (route)",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Ingester",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 11,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 12,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by ()",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Memcached",
   "titleSize": "h6"
  },
  {
   "collapse": false,
   "height": "250px",
   "panels": [
    {
     "aliasColors": {
      "1xx": "#EAB839",
      "2xx": "#7EB26D",
      "3xx": "#6ED0E0",
      "4xx": "#EF843C",
      "5xx": "#E24D42",
      "error": "#E24D42",
      "success": "#7EB26D"
     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 10,
     "id": 13,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 0,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": true,
     "steppedLine": false,
     "targets": [
      {
       "expr": "sum by (status) (\n  label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{status}}",
       "refId": "A",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "QPS",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    },
    {
     "aliasColors": {

     },
     "bars": false,
     "dashLength": 10,
     "dashes": false,
     "datasource": "$datasource",
     "fill": 1,
     "id": 14,
     "legend": {
      "avg": false,
      "current": false,
      "max": false,
      "min": false,
      "show": true,
      "total": false,
      "values": false
     },
     "lines": true,
     "linewidth": 1,
     "links": [

     ],
     "nullPointMode": "null as zero",
     "percentage": false,
     "pointradius": 5,
     "points": false,
     "renderer": "flot",
     "seriesOverrides": [

     ],
     "spaceLength": 10,
     "span": 6,
     "stack": false,
     "steppedLine": false,
     "targets": [
      {
       "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 99th",
       "refId": "A",
       "step": 10
      },
      {
       "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} 50th",
       "refId": "B",
       "step": 10
      },
      {
       "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*querier\",operation=\"GET\"}[$__interval])) by ()",
       "format": "time_series",
       "interval": "1m",
       "intervalFactor": 2,
       "legendFormat": "{{route}} Average",
       "refId": "C",
       "step": 10
      }
     ],
     "thresholds": [

     ],
     "timeFrom": null,
     "timeShift": null,
     "title": "Latency",
     "tooltip": {
      "shared": true,
      "sort": 2,
      "value_type": "individual"
     },
     "type": "graph",
     "xaxis": {
      "buckets": null,
      "mode": "time",
      "name": null,
      "show": true,
      "values": [

      ]
     },
     "yaxes": [
      {
       "format": "ms",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": 0,
       "show": true
      },
      {
       "format": "short",
       "label": null,
       "logBase": 1,
       "max": null,
       "min": null,
       "show": false
      }
     ]
    }
   ],
   "repeat": null,
   "repeatIteration": null,
   "repeatRowId": null,
   "showTitle": true,
   "title": "Backend",
   "titleSize": "h6"
  }
 ],
 "schemaVersion": 14,
 "style": "dark",
 "tags": [
  "tempo"
 ],
 "templating": {
  "list": [
   {
    "current": {
     "text": "default",
     "value": "default"
    },
    "hide": 0,
    "label": "Data Source",
    "name": "datasource",
    "options": [

    ],
    "query": "prometheus",
    "refresh": 1,
    "regex": "",
    "type": "datasource"
   },
   {
    "allValue": null,
    "current": {
     "selected": true,
     "text": "All",
     "value": "$__all"
    },
    "datasource": "$datasource",
    "hide": 0,
    "includeAll": true,
    "label": "cluster",
    "multi": true,
    "name": "cluster",
    "options": [

    ],
    "query": "label_values(tempo_build_info, cluster)",
    "refresh": 1,
    "regex": "",
    "sort": 2,
    "tagValuesQuery": "",
    "tags": [

    ],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   },
   {
    "allValue": null,
    "current": {
     "selected": true,
     "text": "All",
     "value": "$__all"
    },
    "datasource": "$datasource",
    "hide": 0,
    "includeAll": true,
    "label": "namespace",
    "multi": true,
    "name": "namespace",
    "options": [

    ],
    "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
    "refresh": 1,
    "regex": "",
    "sort": 2,
    "tagValuesQuery": "",
    "tags": [

    ],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   }
  ]
 },
 "time": {
  "from": "now-1h",
  "to": "now"
 },
 "timepicker": {
  "refresh_intervals": [
   "5s",
   "10s",
   "30s",
   "1m",
   "5m",
   "15m",
   "30m",
   "1h",
   "2h",
   "1d"
  ],
  "time_options": [
   "5m",
   "15m",
   "1h",
   "6h",
   "12h",
   "24h",
   "2d",
   "7d",
   "30d"
  ]
 },
 "timezone": "utc",
 "title": "Tempo / Reads",
 "uid": "",
 "version": 0
}