{
    "annotations": {
      "list": [
        {
          "builtIn": 1,
          "datasource": {
            "type": "grafana",
            "uid": "-- Grafana --"
          },
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "type": "dashboard"
        },
        {
          "datasource": "$loki_datasource",
          "enable": true,
          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
          "iconColor": "rgba(0, 211, 255, 1)",
          "instant": false,
          "name": "Deployments",
          "titleFormat": "{{cluster}}/{{namespace}}"
        }
      ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 1,
    "id": 29,
    "links": [
      {
        "icon": "doc",
        "targetBlank": true,
        "title": "Documentation",
        "tooltip": "Component documentation",
        "type": "link",
        "url": "https://grafana.com/docs/alloy/latest/reference/components/prometheus.remote_write/"
      },
      {
        "asDropdown": true,
        "icon": "external link",
        "includeVars": true,
        "keepTime": true,
        "tags": [
          "alloy-mixin"
        ],
        "targetBlank": false,
        "title": "Dashboards",
        "type": "dashboards"
      }
    ],
    "panels": [
      {
        "collapsed": false,
        "datasource": "${datasource}",
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 0
        },
        "id": 1,
        "title": "prometheus.scrape",
        "type": "row"
      },
      {
        "datasource": "${datasource}",
        "description": "Percentage of targets successfully scraped by prometheus.scrape\ncomponents.\n\nThis metric is calculated by dividing the number of targets\nsuccessfully scraped by the total number of targets scraped,\nacross all the namespaces in the selected cluster.\n\nLow success rates can indicate a problem with scrape targets,\nstale service discovery, or Alloy misconfiguration.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "percentunit"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 12,
          "x": 0,
          "y": 1
        },
        "id": 2,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum(up{cluster=\"$cluster\"})\n/\ncount (up{cluster=\"$cluster\"})\n",
            "instant": false,
            "legendFormat": "% of targets successfully scraped",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Scrape success rate in $cluster",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Duration of successful scrapes by prometheus.scrape components,\nacross all the namespaces in the selected cluster.\n\nThis metric should be below your configured scrape interval.\nHigh durations can indicate a problem with a scrape target or\na performance issue with Alloy.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 12,
          "x": 12,
          "y": 1
        },
        "id": 3,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "quantile(0.99, scrape_duration_seconds{cluster=\"$cluster\"})\n",
            "instant": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "A"
          },
          {
            "datasource": "${datasource}",
            "expr": "quantile(0.95, scrape_duration_seconds{cluster=\"$cluster\"})\n",
            "instant": false,
            "legendFormat": "p95",
            "range": true,
            "refId": "B"
          },
          {
            "datasource": "${datasource}",
            "expr": "quantile(0.50, scrape_duration_seconds{cluster=\"$cluster\"})\n",
            "instant": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "C"
          }
        ],
        "title": "Scrape duration in $cluster",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "datasource": "${datasource}",
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 11
        },
        "id": 4,
        "title": "prometheus.remote_write",
        "type": "row"
      },
      {
        "datasource": "${datasource}",
        "description": "How far behind prometheus.remote_write from samples recently written\nto the WAL.\n\nEach endpoint prometheus.remote_write is configured to send metrics\nhas its own delay. The time shown here is the sum across all\nendpoints for the given component.\n\nIt is normal for the WAL delay to be within 1-3 scrape intervals. If\nthe WAL delay continues to increase beyond that amount, try\nincreasing the number of maximum shards.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 6,
          "x": 0,
          "y": 12
        },
        "id": 5,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum by (instance, component_path, component_id) (\n  prometheus_remote_storage_highest_timestamp_in_seconds{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\"}\n  - ignoring(url, remote_name) group_right(instance)\n  prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "WAL delay",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Rate of data containing samples and metadata sent by\nprometheus.remote_write.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 20,
              "gradientMode": "hue",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "Bps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 6,
          "x": 6,
          "y": 12
        },
        "id": 6,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum without (remote_name, url) (\n    rate(prometheus_remote_storage_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval]) +\n    rate(prometheus_remote_storage_metadata_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Data write throughput",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Latency of writes to the remote system made by\nprometheus.remote_write.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 6,
          "x": 12,
          "y": 12
        },
        "id": 7,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "histogram_quantile(0.99, sum by (le) (\n  rate(prometheus_remote_storage_sent_batch_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n))\n",
            "instant": false,
            "legendFormat": "99th percentile",
            "range": true,
            "refId": "A"
          },
          {
            "datasource": "${datasource}",
            "expr": "histogram_quantile(0.50, sum by (le) (\n  rate(prometheus_remote_storage_sent_batch_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n))\n",
            "instant": false,
            "legendFormat": "50th percentile",
            "range": true,
            "refId": "B"
          },
          {
            "datasource": "${datasource}",
            "expr": "sum(rate(prometheus_remote_storage_sent_batch_duration_seconds_sum{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\"}[$__rate_interval])) /\nsum(rate(prometheus_remote_storage_sent_batch_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\"}[$__rate_interval]))\n",
            "instant": false,
            "legendFormat": "Average",
            "range": true,
            "refId": "C"
          }
        ],
        "title": "Write latency",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Total number of shards which are concurrently sending samples read\nfrom the Write-Ahead Log.\n\nShards are bound to a minimum and maximum, displayed on the graph.\nThe lowest minimum and the highest maximum across all clients is\nshown.\n\nEach client has its own set of shards, minimum shards, and maximum\nshards; filter to a specific URL to display more granular\ninformation.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "none"
          },
          "overrides": [
            {
              "matcher": {
                "id": "byName",
                "options": "Minimum"
              },
              "properties": [
                {
                  "id": "custom.lineStyle",
                  "value": {
                    "dash": [
                      10,
                      15
                    ],
                    "fill": "dash"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "never"
                },
                {
                  "id": "custom.hideFrom",
                  "value": {
                    "legend": true,
                    "tooltip": false,
                    "viz": false
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "Maximum"
              },
              "properties": [
                {
                  "id": "custom.lineStyle",
                  "value": {
                    "dash": [
                      10,
                      15
                    ],
                    "fill": "dash"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "never"
                },
                {
                  "id": "custom.hideFrom",
                  "value": {
                    "legend": true,
                    "tooltip": false,
                    "viz": false
                  }
                }
              ]
            }
          ]
        },
        "gridPos": {
          "h": 10,
          "w": 6,
          "x": 18,
          "y": 12
        },
        "id": 8,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum without (remote_name, url) (\n    prometheus_remote_storage_shards{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          },
          {
            "datasource": "${datasource}",
            "expr": "min (\n    prometheus_remote_storage_shards_min{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
            "instant": false,
            "legendFormat": "Minimum",
            "range": true,
            "refId": "B"
          },
          {
            "datasource": "${datasource}",
            "expr": "max (\n    prometheus_remote_storage_shards_max{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n)\n",
            "instant": false,
            "legendFormat": "Maximum",
            "range": true,
            "refId": "C"
          }
        ],
        "title": "Shards",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Total outgoing samples sent by prometheus.remote_write.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 20,
              "gradientMode": "hue",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "cps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 0,
          "y": 22
        },
        "id": 9,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum without (url, remote_name) (\n  rate(prometheus_remote_storage_samples_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Sent samples / second",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Rate of samples which prometheus.remote_write could not send due to\nnon-recoverable errors.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 20,
              "gradientMode": "hue",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "cps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 8,
          "y": 22
        },
        "id": 10,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum without (url,remote_name) (\n  rate(prometheus_remote_storage_samples_failed_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Failed samples / second",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Rate of samples which prometheus.remote_write attempted to resend\nafter receiving a recoverable error.\n",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 20,
              "gradientMode": "hue",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "cps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 16,
          "y": 22
        },
        "id": 11,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum without (url,remote_name) (\n  rate(prometheus_remote_storage_samples_retried_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}[$__rate_interval])\n)\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Retried samples / second",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Total number of active series across all components.\n\nAn \"active series\" is a series that prometheus.remote_write recently\nreceived a sample for. Active series are garbage collected whenever a\ntruncation of the WAL occurs.\n",
        "fieldConfig": {
          "defaults": {
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 0,
          "y": 32
        },
        "id": 12,
        "options": {
          "legend": {
            "displayMode": "list",
            "showLegend": false
          }
        },
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum(prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
            "instant": false,
            "legendFormat": "Series",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Active series (total)",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Total number of active series which are currently being tracked by\nprometheus.remote_write components, with separate lines for each Alloy instance.\n\nAn \"active series\" is a series that prometheus.remote_write recently\nreceived a sample for. Active series are garbage collected whenever a\ntruncation of the WAL occurs.\n",
        "fieldConfig": {
          "defaults": {
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 8,
          "y": 32
        },
        "id": 13,
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"}\n",
            "instant": false,
            "legendFormat": "{{instance}} / {{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Active series (by instance/component)",
        "type": "timeseries"
      },
      {
        "datasource": "${datasource}",
        "description": "Total number of active series which are currently being tracked by\nprometheus.remote_write components, aggregated across all instances.\n\nAn \"active series\" is a series that prometheus.remote_write recently\nreceived a sample for. Active series are garbage collected whenever a\ntruncation of the WAL occurs.\n",
        "fieldConfig": {
          "defaults": {
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 10,
          "w": 8,
          "x": 16,
          "y": 32
        },
        "id": 14,
        "targets": [
          {
            "datasource": "${datasource}",
            "expr": "sum by (component_path, component_id) (prometheus_remote_write_wal_storage_active_series{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id!=\"\", component_path=~\"$component_path\", component_id=~\"$component\", url=~\"$url\"})\n",
            "instant": false,
            "legendFormat": "{{component_path}} {{component_id}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Active series (by component)",
        "type": "timeseries"
      }
    ],
    "refresh": "10s",
    "schemaVersion": 39,
    "tags": [
      "alloy-mixin"
    ],
    "templating": {
      "list": [
        {
          "current": {
            "selected": false,
            "text": "Mimir",
            "value": "mimir_ds"
          },
          "hide": 0,
          "includeAll": false,
          "label": "Data Source",
          "multi": false,
          "name": "datasource",
          "options": [],
          "query": "prometheus",
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "datasource"
        },
        {
          "current": {
            "selected": false,
            "text": "Loki",
            "value": "loki_ds"
          },
          "hide": 0,
          "includeAll": false,
          "label": "Loki Data Source",
          "multi": false,
          "name": "loki_datasource",
          "options": [],
          "query": "loki",
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "datasource"
        },
        {
          "current": {
            "isNone": true,
            "selected": false,
            "text": "None",
            "value": ""
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": false,
          "label": "cluster",
          "multi": false,
          "name": "cluster",
          "options": [],
          "query": {
            "query": "label_values(alloy_component_controller_running_components, cluster)\n",
            "refId": "cluster"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        },
        {
          "current": {
            "isNone": true,
            "selected": false,
            "text": "None",
            "value": ""
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": false,
          "label": "namespace",
          "multi": false,
          "name": "namespace",
          "options": [],
          "query": {
            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\"}, namespace)\n",
            "refId": "namespace"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        },
        {
          "allValue": ".*",
          "current": {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": true,
          "label": "instance",
          "multi": true,
          "name": "instance",
          "options": [],
          "query": {
            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
            "refId": "instance"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        },
        {
          "allValue": ".*",
          "current": {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": true,
          "label": "component_path",
          "multi": true,
          "name": "component_path",
          "options": [],
          "query": {
            "query": "label_values(prometheus_remote_write_wal_samples_appended_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\", component_path=~\".*\"}, component_path)\n",
            "refId": "component_path"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        },
        {
          "allValue": ".*",
          "current": {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": true,
          "label": "component",
          "multi": true,
          "name": "component",
          "options": [],
          "query": {
            "query": "label_values(prometheus_remote_write_wal_samples_appended_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"prometheus\\\\.remote_write\\\\..*\"}, component_id)\n",
            "refId": "component"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        },
        {
          "allValue": ".*",
          "current": {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          "datasource": {
            "uid": "${datasource}"
          },
          "definition": "",
          "hide": 0,
          "includeAll": true,
          "label": "url",
          "multi": true,
          "name": "url",
          "options": [],
          "query": {
            "query": "label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", component_id=~\"$component\"}, url)\n",
            "refId": "url"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 2,
          "type": "query"
        }
      ]
    },
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {
      "refresh_intervals": [
        "5s",
        "10s",
        "30s",
        "1m",
        "5m",
        "15m",
        "30m",
        "1h",
        "2h",
        "1d"
      ],
      "time_options": [
        "5m",
        "15m",
        "1h",
        "6h",
        "12h",
        "24h",
        "2d",
        "7d",
        "30d",
        "90d"
      ]
    },
    "timezone": "",
    "title": "Alloy / Prometheus Components",
    "uid": "e324cc55567d7f3a8e32860ff8e6d0d9",
    "version": 0,
    "weekStart": ""
  }