Merge pull request #186 from grafana/chore/update-dependencies

[dependency] Update the subcharts

.github/configs/cluster-config.yaml

@@ -0,0 +1,19 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: ClusterConfiguration
+        controllerManager:
+          extraArgs:
+            bind-address: 0.0.0.0
+            secure-port: "10257"
+        scheduler:
+          extraArgs:
+            bind-address: 0.0.0.0
+            secure-port: "10259"
+      - |
+        kind: KubeProxyConfiguration
+        metricsBindAddress: 0.0.0.0:10249
+  - role: worker

.github/configs/cr.yaml

@@ -0,0 +1,3 @@
+owner: grafana
+git-repo: helm-charts
+skip-existing: true

.github/configs/ct.yaml

@@ -0,0 +1,15 @@
+## Reference: https://github.com/helm/chart-testing/blob/master/doc/ct_lint-and-install.md
+remote: origin
+target-branch: main
+chart-dirs:
+- charts
+chart-repos:
+  - grafana=https://grafana.github.io/helm-charts
+  - minio=https://charts.min.io
+validate-chart-schema: true
+validate-maintainers: true
+validate-yaml: true
+exclude-deprecated: true
+excluded-charts: []
+namespace: meta-monitoring # Need to set the namespace because we create the secret there
+release-label: app.kubernetes.io/instance

.github/configs/updatecli.d/alloy.yaml

@@ -0,0 +1,30 @@
+name: Bump dependency "alloy" for Helm chart "meta-monitoring"
+sources:
+    alloy:
+        name: Get latest "alloy" Helm chart version
+        kind: helmchart
+        spec:
+            name: alloy
+            url: https://grafana.github.io/helm-charts
+            versionfilter:
+                kind: semver
+                pattern: '*'
+conditions:
+    alloy:
+        name: Ensure Helm chart dependency "alloy" is specified
+        kind: yaml
+        spec:
+            file: charts/meta-monitoring/Chart.yaml
+            key: $.dependencies[1].name
+            value: alloy
+        disablesourceinput: true
+targets:
+    alloy:
+        name: Bump Helm chart dependency "alloy" for Helm chart "meta-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[1].version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: alloy

.github/configs/updatecli.d/grafana.yaml

@@ -0,0 +1,30 @@
+name: Bump grafana version specified in the values.yaml
+sources:
+    latestGrafanaRelease:
+        name: Get latest grafana release on Github
+        kind: githubrelease
+        spec:
+            owner: grafana
+            repository: grafana
+            token: '{{ requiredEnv "UPDATECLI_GITHUB_TOKEN" }}'
+            versionfilter:
+                kind: latest
+        transformers:
+          - trimprefix: "v"
+conditions:
+    grafanaImagePublished:
+        name: Ensure the latest Grafana is published on DockerHub
+        kind: dockerimage
+        source-id: latestGrafanaRelease
+        spec:
+            image: "grafana/grafana"
+targets:
+    grafana:
+        name: Update Grafana version in values.yaml
+        kind: helmchart
+        spec:
+            file: values.yaml
+            key: $.grafana.version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: latestGrafanaRelease

.github/configs/updatecli.d/loki.yaml

@@ -0,0 +1,30 @@
+name: Bump dependency "loki" for Helm chart "meta-monitoring"
+sources:
+    loki:
+        name: Get latest "loki" Helm chart version
+        kind: helmchart
+        spec:
+            name: loki
+            url: https://grafana.github.io/helm-charts
+            versionfilter:
+                kind: semver
+                pattern: '*'
+conditions:
+    loki:
+        name: Ensure Helm chart dependency "loki" is specified
+        kind: yaml
+        spec:
+            file: charts/meta-monitoring/Chart.yaml
+            key: $.dependencies[0].name
+            value: loki
+        disablesourceinput: true
+targets:
+    loki:
+        name: Bump Helm chart dependency "loki" for Helm chart "meta-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[0].version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: loki

.github/configs/updatecli.d/mimir-distributed.yaml

@@ -0,0 +1,30 @@
+name: Bump dependency "mimir-distributed" for Helm chart "meta-monitoring"
+sources:
+    mimir-distributed:
+        name: Get latest "mimir-distributed" Helm chart version
+        kind: helmchart
+        spec:
+            name: mimir-distributed
+            url: https://grafana.github.io/helm-charts
+            versionfilter:
+                kind: semver
+                pattern: '*'
+conditions:
+    mimir-distributed:
+        name: Ensure Helm chart dependency "mimir-distributed" is specified
+        kind: yaml
+        spec:
+            file: charts/meta-monitoring/Chart.yaml
+            key: $.dependencies[2].name
+            value: mimir-distributed
+        disablesourceinput: true
+targets:
+    mimir-distributed:
+        name: Bump Helm chart dependency "mimir-distributed" for Helm chart "meta-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[2].version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: mimir-distributed

.github/configs/updatecli.d/minio.yaml

@@ -0,0 +1,30 @@
+name: Bump dependency "minio" for Helm chart "meta-monitoring"
+sources:
+    minio:
+        name: Get latest "minio" Helm chart version
+        kind: helmchart
+        spec:
+            name: minio
+            url: https://charts.min.io
+            versionfilter:
+                kind: semver
+                pattern: '*'
+conditions:
+    minio:
+        name: Ensure Helm chart dependency "minio" is specified
+        kind: yaml
+        spec:
+            file: charts/meta-monitoring/Chart.yaml
+            key: $.dependencies[4].name
+            value: minio
+        disablesourceinput: true
+targets:
+    minio:
+        name: Bump Helm chart dependency "minio" for Helm chart "meta-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[4].version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: minio

.github/configs/updatecli.d/tempo-distributed.yaml

@@ -0,0 +1,30 @@
+name: Bump dependency "tempo-distributed" for Helm chart "meta-monitoring"
+sources:
+    tempo-distributed:
+        name: Get latest "tempo-distributed" Helm chart version
+        kind: helmchart
+        spec:
+            name: tempo-distributed
+            url: https://grafana.github.io/helm-charts
+            versionfilter:
+                kind: semver
+                pattern: '*'
+conditions:
+    tempo-distributed:
+        name: Ensure Helm chart dependency "tempo-distributed" is specified
+        kind: yaml
+        spec:
+            file: charts/meta-monitoring/Chart.yaml
+            key: $.dependencies[3].name
+            value: tempo-distributed
+        disablesourceinput: true
+targets:
+    tempo-distributed:
+        name: Bump Helm chart dependency "tempo-distributed" for Helm chart "meta-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[3].version
+            name: charts/meta-monitoring
+            versionincrement: none
+        sourceid: tempo-distributed

.github/workflows/check-for-dependency-updates.yaml

@@ -0,0 +1,151 @@
+---
+name: Check for dependency updates
+
+on:
+    workflow_dispatch:
+    schedule:
+      # Run once a day
+      - cron: '0 7 * * *'
+
+permissions:
+    contents: "write"
+    pull-requests: "write"
+
+env:
+    UPDATECLI_CONFIG_DIR: "${{ github.workspace }}/.github/configs/updatecli.d"
+    UPDATECLI_GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+jobs:
+    updateVersions:
+        name: Update the subcharts
+        runs-on: "ubuntu-latest"
+        permissions:
+          contents: write
+          id-token: write
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+
+            - name: Install Updatecli
+              uses: updatecli/updatecli-action@v2
+
+            - name: Run Updatecli for Loki
+              id: update-loki
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/loki.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - name: Run Updatecli for Alloy
+              id: update-grafana-alloy
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/alloy.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - name: Run Updatecli for Mimir
+              id: update-mimir-distributed
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/mimir-distributed.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - name: Run Updatecli for Tempo
+              id: update-tempo-distributed
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/tempo-distributed.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - name: Run Updatecli for Minio
+              id: update-minio
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/minio.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - id: get-secrets
+              uses: grafana/shared-workflows/actions/get-vault-secrets@main
+              with:
+                # Secrets placed in the ci/repo/grafana/<repo>/<path> path in Vault
+                repo_secrets: |
+                  APP_ID=github-app:app-id
+                  PRIVATE_KEY=github-app:private-key
+
+            - uses: actions/create-github-app-token@v1
+              id: app-token
+              with:
+                app-id: ${{ env.APP_ID }}
+                private-key: ${{ env.PRIVATE_KEY }}
+
+            - name: Create pull request
+              if: steps.update-loki.outputs.changed == 'true' || steps.update-grafana-alloy.outputs.changed == 'true' || steps.update-mimir-distributed.outputs.changed == 'true' || steps.update-tempo-distributed.outputs.changed == 'true' || steps.update-minio.outputs.changed == 'true'
+              uses: peter-evans/create-pull-request@v5
+              with:
+                title: "[dependency] Update the subcharts"
+                body: "Updates the subcharts"
+                base: main
+                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
+                committer: "GitHub <noreply@github.com>"
+                commit-message: Update dependencies
+                labels: dependencies
+                branch: chore/update-dependencies
+                delete-branch: true
+                team-reviewers: "@grafana/loki-squad"
+                token: ${{ steps.app-token.outputs.token }}
+
+    updateGrafana:
+        name: Update the Grafana version
+        runs-on: "ubuntu-latest"
+        permissions:
+          contents: write
+          id-token: write
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+
+            - name: Install Updatecli
+              uses: updatecli/updatecli-action@v2
+
+            - name: Run Updatecli
+              id: update-grafana
+              run: |
+                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/grafana.yaml
+                if ! git diff --exit-code > /dev/null; then
+                  echo "changed=true" >> "${GITHUB_OUTPUT}"
+                fi
+
+            - id: get-secrets
+              uses: grafana/shared-workflows/actions/get-vault-secrets@main
+              with:
+                # Secrets placed in the ci/repo/grafana/<repo>/<path> path in Vault
+                repo_secrets: |
+                  APP_ID=github-app:app-id
+                  PRIVATE_KEY=github-app:private-key
+
+            - uses: actions/create-github-app-token@v1
+              id: app-token
+              with:
+                app-id: ${{ env.APP_ID }}
+                private-key: ${{ env.PRIVATE_KEY }}
+
+            - name: Create pull request
+              if: steps.update-grafana.outputs.changed == 'true'
+              uses: peter-evans/create-pull-request@v5
+              with:
+                title: "[dependency] Update the Grafana version"
+                body: "Updates the Grafana version"
+                base: main
+                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
+                committer: "GitHub <noreply@github.com>"
+                commit-message: Update Grafana version
+                labels: dependencies
+                branch: chore/update-minio
+                delete-branch: true
+                team-reviewers: "@grafana/loki-squad"
+                token: ${{ steps.app-token.outputs.token }}

.github/workflows/helm-ci.yml

@@ -0,0 +1,67 @@
+---
+name: helm-ci
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "charts/meta-monitoring/**"
+
+env:
+  CT_CONFIGFILE: charts/meta-monitoring/ct.yaml
+
+jobs:
+  call-lint:
+    name: Lint Helm Chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Lint Yaml
+        run: make helm-lint
+
+  # call-test:
+  #   name: Test Helm Chart
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+
+  #     - name: Set up Helm
+  #       uses: azure/setup-helm@v3
+  #       with:
+  #         version: v3.8.2
+
+  #     # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+  #     # yamllint (https://github.com/adrienverge/yamllint) which require Python
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.9
+
+  #     - name: Set up chart-testing
+  #       uses: helm/chart-testing-action@v2
+
+  #     - name: Run chart-testing (list-changed)
+  #       id: list-changed
+  #       run: |
+  #         changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+  #         if [[ -n "$changed" ]]; then
+  #           echo "changed=true" >> $GITHUB_OUTPUT
+  #         fi
+
+  #     - name: Run chart-testing (lint)
+  #       run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false
+
+  #     - name: Create kind cluster
+  #       uses: helm/kind-action@v1
+  #       if: steps.list-changed.outputs.changed == 'true'
+  #       with:
+  #         config: "${{ github.workspace }}/.github/configs/cluster-config.yaml"
+
+  #     - name: Run chart-testing (install)
+  #       run: |
+  #         changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+  #         ct install --config "${CT_CONFIGFILE}"

.github/workflows/helm-release.yml

@@ -0,0 +1,175 @@
+name: Release Helm chart
+on:
+  workflow_dispatch:
+env:
+  CR_CONFIGFILE: "${{ github.workspace }}/source/.github/configs/cr.yaml"
+  CT_CONFIGFILE: "${{ github.workspace }}/source/.github/configs/ct.yaml"
+  CR_INDEX_PATH: "${{ github.workspace }}/.cr-index"
+  CR_PACKAGE_PATH: "${{ github.workspace }}/.cr-release-packages"
+  CR_TOOL_PATH: "${{ github.workspace }}/.cr-tool"
+  CR_VERSION: "1.5.0"
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.list-changed.outputs.changed }}
+      chartpath: ${{ steps.list-changed.outputs.chartpath }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        path: source
+
+    - name: Install chart-testing
+      uses: helm/chart-testing-action@v2
+
+    - name: List changed charts
+      id: list-changed
+      run: |
+        cd source
+
+        latest_tag=$( if ! git describe --tags --abbrev=0 --match='helm-chart/*' 2> /dev/null ; then git rev-list --max-parents=0 --first-parent HEAD; fi )
+
+        echo "Running: ct list-changed --config ${CT_CONFIGFILE} --since ${latest_tag} --target-branch ${{ github.ref_name }}"
+        changed=$(ct list-changed --config "${CT_CONFIGFILE}" --since "${latest_tag}" --target-branch "${{ github.ref_name }}")
+        echo "${changed}"
+
+        num_changed=$(wc -l <<< ${changed})
+        if [[ "${num_changed}" -gt "1" ]] ; then
+          echo "More than one chart changed, exiting"
+          exit 1
+        fi
+        if [[ -n "${changed}" ]]; then
+          name=$(yq ".name" < ${changed}/Chart.yaml)
+          version=$(yq ".version" < ${changed}/Chart.yaml)
+          tagname="v${version}"
+
+          if [ $(git tag -l "${tagname}") ]; then
+            echo "Tag ${tagname} already exists, skipping release"
+            echo "changed=false" >> $GITHUB_OUTPUT
+          else
+            echo "Releasing ${changed}"
+            echo "changed=true" >> $GITHUB_OUTPUT
+            echo "chartpath=${changed}" >> $GITHUB_OUTPUT
+          fi
+        else
+          echo "No charts have changed, skipping release"
+          echo "changed=false" >> $GITHUB_OUTPUT
+        fi
+
+  release:
+    needs: [setup]
+    runs-on: ubuntu-latest
+    if: needs.setup.outputs.changed == 'true'
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+    - id: get-secrets
+      uses: grafana/shared-workflows/actions/get-vault-secrets@main
+      with:
+        # Secrets placed in the ci/repo/grafana/<repo>/<path> path in Vault
+        repo_secrets: |
+          APP_ID=github-app:app-id
+          PRIVATE_KEY=github-app:private-key
+    - uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ env.APP_ID }}
+        private-key: ${{ env.PRIVATE_KEY }}
+        owner: ${{ github.repository_owner }}
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        path: source
+
+    - name: Configure Git
+      run: |
+        cd source
+        git config user.name "$GITHUB_ACTOR"
+        git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+    - name: Checkout helm-charts
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        repository: grafana/helm-charts
+        path: helm-charts
+        token: "${{ steps.app-token.outputs.token }}"
+
+    - name: Configure Git for helm-charts
+      run: |
+        cd helm-charts
+        git config user.name "$GITHUB_ACTOR"
+        git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+    - name: Set up Helm
+      uses: azure/setup-helm@v4
+
+    - name: Parse Chart.yaml
+      id: parse-chart
+      run: |
+        cd source
+        changed="${{ needs.setup.outputs.chartpath }}"
+        description=$(yq ".description" < ${changed}/Chart.yaml)
+        name=$(yq ".name" < ${changed}/Chart.yaml)
+        version=$(yq ".version" < ${changed}/Chart.yaml)
+
+        echo "chartpath=${changed}" >> $GITHUB_OUTPUT
+        echo "desc=${description}" >> $GITHUB_OUTPUT
+        echo "tagname=v${version}" >> $GITHUB_OUTPUT
+        echo "packagename=${name}-${version}" >> $GITHUB_OUTPUT
+
+    - name: Install CR tool
+      run: |
+        mkdir "${CR_TOOL_PATH}"
+        mkdir "${CR_PACKAGE_PATH}"
+        mkdir "${CR_INDEX_PATH}"
+        curl -sSLo cr.tar.gz "https://github.com/helm/chart-releaser/releases/download/v${CR_VERSION}/chart-releaser_${CR_VERSION}_linux_amd64.tar.gz"
+        tar -xzf cr.tar.gz -C "${CR_TOOL_PATH}"
+        rm -f cr.tar.gz
+
+    - name: Create Helm package
+      run: |
+        cd source
+        helm repo add grafana https://grafana.github.io/helm-charts
+        helm repo add minio https://charts.min.io
+
+        "${CR_TOOL_PATH}/cr" package "${{ steps.parse-chart.outputs.chartpath }}" --config "${CR_CONFIGFILE}" --package-path "${CR_PACKAGE_PATH}"
+
+    - name: Make a release on this repo
+      uses: softprops/action-gh-release@v2
+      with:
+        name: ${{ steps.parse-chart.outputs.tagname }}
+        repository: grafana/meta-monitoring-chart
+        tag_name: ${{ steps.parse-chart.outputs.tagname }}
+        token: ${{ steps.app-token.outputs.token }}
+        generate_release_notes: true
+        files: |
+          ${{ env.CR_PACKAGE_PATH }}/${{ steps.parse-chart.outputs.packagename }}.tgz
+
+    # Note that this creates a release in grafana/helm-charts with a new tag.
+    # The tag name in grafana/helm-charts is <package>-<version>, while the
+    # tag name for grafana/meta-monitoring-chart is <version>.
+    - name: Make release on Helm Charts
+      uses: softprops/action-gh-release@v2
+      with:
+        name: ${{ steps.parse-chart.outputs.packagename }}
+        repository: grafana/helm-charts
+        tag_name: ${{ steps.parse-chart.outputs.packagename }}
+        token: ${{ steps.app-token.outputs.token }}
+        body: |
+          ${{ steps.parse-chart.outputs.desc }}
+
+          Source commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }}
+
+          Tag on source: https://github.com/${{ github.repository }}/releases/tag/${{ steps.parse-chart.outputs.tagname }}
+        files: |
+          ${{ env.CR_PACKAGE_PATH }}/${{ steps.parse-chart.outputs.packagename }}.tgz
+
+    - name: Update helm-charts index.yaml
+      run: |
+        cd helm-charts
+        "${CR_TOOL_PATH}/cr" index --config "${CR_CONFIGFILE}" --token "${{ steps.app-token.outputs.token }}" --index-path "${CR_INDEX_PATH}" --package-path "${CR_PACKAGE_PATH}" --push

.gitignore

@@ -0,0 +1 @@
+.DS_Store

Makefile

@@ -0,0 +1,10 @@
+# Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/
+.PHONY: help
+help: ## Display this help and any documented user-facing targets. Other undocumented targets may be present in the Makefile.
+help:
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
+.PHONY: helm-lint
+
+helm-lint: ## Run helm linter
+	$(MAKE) -BC charts/meta-monitoring lint

README.md

@@ -1,42 +1,41 @@
 # meta-monitoring-chart
 
-This is a meta-monitoring chart for GEL, GEM and GET. It should be installed in a
-separate namespace next to GEM, GEL or GET installations.
-
-## Preparation
-
-Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml).
-
-1. Add or remove the namespaces to monitor in the `namespacesToMonitor` setting
-
-1. Set the cluster name in the `clusterName` setting. This will be added as a label to all logs, metrics and traces.
-
-1. Create a `meta` namespace.
+This is a meta-monitoring chart for Loki, specifically Loki installed via the Loki helm chart.
 
 ## Local and cloud modes
 
-The chart has 2 modes: local and cloud. In the local mode logs, metrics and traces are sent
+The chart has 2 modes: local and cloud. In the local mode logs, metrics and/or traces are sent
 to small Loki, Mimir and Tempo installations running in the meta-monitoring namespace.
 
 ![local mode](docs/images/Meta%20monitoring%20local.png)
 
-To enable local mode set `local.enabled` to true.
-
-In the cloud mode the logs, metrics and traces are sent to
+In the cloud mode the logs, metrics and/or traces are sent to Grafana Cloud.
 
 ![cloud mode](docs/images/Meta%20monitoring%20cloud.png)
 
-To enable cloud mode set `cloud.enabled` to true. The `endpoint`, `username` and `password` settings for your Grafana Cloud logs, metrics and traces instances have to be filled in as well.
-
-Both modes can be enabled at the same time.
+Both modes can be enabled at the same time. Cloud mode is preferred.
 
 ## Installation
 
-```
-helm install -n meta -f values.yaml meta ./charts/meta-monitoring
-```
+For more instructions including how to install the chart go to the [installation](docs/installation.md) page.
 
-For more instructions including how to update the chart go to the [installation](docs/installation.md) page.
+## Supported features
+
+- Specify which namespaces are monitored
+- Specify if logs, metrics or traces should be enabled for cloud or local
+- Specify the cluster name used for the logs, metrics and traces
+- Specify PII regexes that are applied to logs before they are sent to Loki (cloud or local). The capture group in the regex is replaced with *****.
+- a Grafana instance is installed (when local mode is used) with the relevant datasources installed. The following dashboards are installed:
+  - logs dashboards
+  - Alloy dashboards
+
+Most of these features are enabled by default. See the values.yaml file for how to enable/disable them.
+
+## Caveats
+
+- This has not been tested on Openshift yet.
+- The underlying Loki, Mimir and Tempo are at the default size installed by the Helm chart. This might need changing when monitoring bigger Loki, Mimir or Tempo installations.
+- MinIO is used as storage for the local mode at the moment with a limited retention. At the moment this chart cannot be used for monitoring over longer periods.
 
 ## Developer help topics
 

charts/meta-monitoring/Chart.lock

@@ -1,18 +1,18 @@
 dependencies:
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: 5.8.0
-- name: grafana-agent
+  version: 6.29.0
+- name: alloy
   repository: https://grafana.github.io/helm-charts
-  version: 0.15.0
+  version: 0.12.5
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 4.4.1
+  version: 5.6.0
 - name: tempo-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 1.4.7
+  version: 1.33.0
 - name: minio
   repository: https://charts.min.io
-  version: 5.0.11
-digest: sha256:4b04084e6fe821c4d481017b2430f7c8cd782a5d60830dd3a24eb8f10a9ece09
-generated: "2023-06-29T14:25:07.247853+01:00"
+  version: 5.4.0
+digest: sha256:5225a03d9003384639f5d43b1971126371269347f16f221b7aed377ab85d71be
+generated: "2025-03-27T07:03:11.17404081Z"

charts/meta-monitoring/Chart.yaml

@@ -1,7 +1,6 @@
 apiVersion: v2
 name: meta-monitoring
 description: A Helm chart for meta monitoring Grafana Loki, Mimir and Tempo
-
 # A chart can be either an 'application' or a 'library' chart.
 #
 # Application charts are a collection of templates that can be packaged into versioned archives
@@ -11,35 +10,32 @@ description: A Helm chart for meta monitoring Grafana Loki, Mimir and Tempo
 # a dependency of application charts to inject those utilities and functions into the rendering
 # pipeline. Library charts do not define any templates and therefore cannot be deployed.
 type: application
-
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.1
-
+version: 1.3.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
 appVersion: "0.0.1"
-
 dependencies:
 - name: loki
-  repository:  https://grafana.github.io/helm-charts
-  version: "5.8.0"
+  repository: https://grafana.github.io/helm-charts
+  version: 6.29.0
   condition: local.logs.enabled
-- name: grafana-agent
-  repository:  https://grafana.github.io/helm-charts
-  version: "0.15.0"
+- name: alloy
+  repository: https://grafana.github.io/helm-charts
+  version: 0.12.5
 - name: mimir-distributed
-  repository:  https://grafana.github.io/helm-charts
-  version: "4.4.1"
+  repository: https://grafana.github.io/helm-charts
+  version: 5.6.0
   condition: local.metrics.enabled
 - name: tempo-distributed
-  repository:  https://grafana.github.io/helm-charts
-  version: "1.4.7"
+  repository: https://grafana.github.io/helm-charts
+  version: 1.33.0
   condition: local.traces.enabled
 - name: minio
   repository: https://charts.min.io
-  version: "5.0.11"
+  version: 5.4.0
   condition: local.minio.enabled

charts/meta-monitoring/Makefile

@@ -0,0 +1,7 @@
+.DEFAULT_GOAL := lint
+.PHONY: lint lint-yaml
+
+lint: lint-yaml
+
+lint-yaml:
+	yamllint -c $(CURDIR)/src/.yamllint.yaml $(CURDIR)/src

charts/meta-monitoring/ct.yaml

@@ -0,0 +1,11 @@
+---
+remote: origin
+target-branch: main
+chart-dirs:
+  - charts
+chart-repos:
+  - grafana=https://grafana.github.io/helm-charts
+  - minio=https://charts.min.io
+helm-extra-args: --timeout 1200s
+check-version-increment: false
+validate-maintainers: false

charts/meta-monitoring/src/.yamllint.yaml

@@ -0,0 +1,4 @@
+---
+rules:
+  quoted-strings:
+    required: true

charts/meta-monitoring/src/dashboards/alloy-cluster-overview.json

@@ -0,0 +1,540 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        },
+        {
+          "datasource": "$loki_datasource",
+          "enable": true,
+          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "instant": false,
+          "name": "Deployments",
+          "titleFormat": "{{cluster}}/{{namespace}}"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": 27,
+    "links": [
+      {
+        "icon": "doc",
+        "targetBlank": true,
+        "title": "Documentation",
+        "tooltip": "Clustering documentation",
+        "type": "link",
+        "url": "https://grafana.com/docs/alloy/latest/reference/cli/run/#clustered-mode"
+      },
+      {
+        "asDropdown": true,
+        "icon": "external link",
+        "includeVars": true,
+        "keepTime": true,
+        "tags": [
+          "alloy-mixin"
+        ],
+        "targetBlank": false,
+        "title": "Dashboards",
+        "type": "dashboards"
+      }
+    ],
+    "panels": [
+      {
+        "datasource": "${datasource}",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 8,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"})",
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A"
+          }
+        ],
+        "title": "Nodes",
+        "type": "stat"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Nodes info.\n",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "align": "auto",
+              "cellOptions": {
+                "type": "auto"
+              },
+              "inspect": false
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": [
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "Dashboard"
+              },
+              "properties": [
+                {
+                  "id": "mappings",
+                  "value": [
+                    {
+                      "options": {
+                        "1": {
+                          "index": 0,
+                          "text": "Link"
+                        }
+                      },
+                      "type": "value"
+                    }
+                  ]
+                },
+                {
+                  "id": "links",
+                  "value": [
+                    {
+                      "targetBlank": false,
+                      "title": "Detail dashboard for node",
+                      "url": "/d/4047e755d822da63c8158cde32ae4dce/alloy-cluster-node?var-instance=${__data.fields.instance}&var-datasource=${datasource}&var-loki_datasource=${loki_datasource}&var-cluster=${cluster}&var-namespace=${namespace}"
+                    }
+                  ]
+                }
+              ]
+            }
+          ]
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 16,
+          "x": 8,
+          "y": 0
+        },
+        "id": 2,
+        "options": {
+          "cellHeight": "sm",
+          "footer": {
+            "countRows": false,
+            "fields": "",
+            "reducer": [
+              "sum"
+            ],
+            "show": false
+          },
+          "showHeader": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"}",
+            "format": "table",
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A"
+          }
+        ],
+        "title": "Node table",
+        "transformations": [
+          {
+            "id": "organize",
+            "options": {
+              "excludeByName": {
+                "Time": true,
+                "Value": false,
+                "__name__": true,
+                "cluster": true,
+                "namespace": true,
+                "state": false
+              },
+              "indexByName": {},
+              "renameByName": {
+                "Value": "Dashboard",
+                "instance": "",
+                "state": ""
+              }
+            }
+          }
+        ],
+        "type": "table"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Whether the cluster state has converged.\n\nIt is normal for the cluster state to be diverged briefly as gossip events propagate. It is not normal for the cluster state to be diverged for a long period of time.\n\nThis will show one of the following:\n\n* Converged: Nodes are aware of all other nodes, with the correct states.\n* Not converged: A subset of nodes aren't aware of their peers, or don't have an updated view of peer states.\n",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [
+              {
+                "options": {
+                  "1": {
+                    "color": "red",
+                    "index": 1,
+                    "text": "Not converged"
+                  }
+                },
+                "type": "value"
+              },
+              {
+                "options": {
+                  "match": "null",
+                  "result": {
+                    "color": "green",
+                    "index": 0,
+                    "text": "Converged"
+                  }
+                },
+                "type": "special"
+              }
+            ],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "suffix:nodes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 8,
+          "x": 0,
+          "y": 9
+        },
+        "id": 3,
+        "options": {
+          "colorMode": "background",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "clamp((\n  sum(stddev by (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"}) != 0) or\n  (sum(abs(sum without (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) - scalar(count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"})) != 0))\n  ),\n  1, 1\n)\n",
+            "format": "time_series",
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A"
+          }
+        ],
+        "title": "Convergance state",
+        "type": "stat"
+      },
+      {
+        "datasource": "${datasource}",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "continuous-GrYlRd"
+            },
+            "custom": {
+              "fillOpacity": 80,
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineWidth": 0,
+              "spanNulls": true
+            },
+            "mappings": [
+              {
+                "options": {
+                  "0": {
+                    "color": "green",
+                    "text": "Yes"
+                  }
+                },
+                "type": "value"
+              },
+              {
+                "options": {
+                  "1": {
+                    "color": "red",
+                    "text": "No"
+                  }
+                },
+                "type": "value"
+              }
+            ],
+            "max": 1,
+            "noValue": 0,
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 16,
+          "x": 8,
+          "y": 9
+        },
+        "id": 4,
+        "options": {
+          "alignValue": "left",
+          "legend": {
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "mergeValues": true,
+          "rowHeight": 0.9,
+          "showValue": "auto",
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "ceil(clamp((\n  sum(stddev by (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) or\n  (sum(abs(sum without (state) (cluster_node_peers{cluster=\"$cluster\", namespace=\"$namespace\"})) - scalar(count(cluster_node_info{cluster=\"$cluster\", namespace=\"$namespace\"}))))\n  ),\n  0, 1\n))\n",
+            "instant": false,
+            "legendFormat": "Converged",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Convergance state timeline",
+        "type": "state-timeline"
+      }
+    ],
+    "refresh": "10s",
+    "schemaVersion": 39,
+    "tags": [
+      "alloy-mixin"
+    ],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "Mimir",
+            "value": "mimir_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Data Source",
+          "multi": false,
+          "name": "datasource",
+          "options": [],
+          "query": "prometheus",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Loki",
+            "value": "loki_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Loki Data Source",
+          "multi": false,
+          "name": "loki_datasource",
+          "options": [],
+          "query": "loki",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "cluster",
+          "multi": false,
+          "name": "cluster",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components, cluster)\n",
+            "refId": "cluster"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "namespace",
+          "multi": false,
+          "name": "namespace",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\"}, namespace)\n",
+            "refId": "namespace"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d",
+        "90d"
+      ]
+    },
+    "timezone": "",
+    "title": "Alloy / Cluster Overview",
+    "uid": "",
+    "version": 0,
+    "weekStart": ""
+  }

charts/meta-monitoring/src/dashboards/alloy-controller.json

@@ -0,0 +1,970 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        },
+        {
+          "datasource": "$loki_datasource",
+          "enable": true,
+          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "instant": false,
+          "name": "Deployments",
+          "titleFormat": "{{cluster}}/{{namespace}}"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": 28,
+    "links": [
+      {
+        "icon": "doc",
+        "targetBlank": true,
+        "title": "Documentation",
+        "tooltip": "Component controller documentation",
+        "type": "link",
+        "url": "https://grafana.com/docs/alloy/latest/concepts/component_controller/"
+      },
+      {
+        "asDropdown": true,
+        "icon": "external link",
+        "includeVars": true,
+        "keepTime": true,
+        "tags": [
+          "alloy-mixin"
+        ],
+        "targetBlank": false,
+        "title": "Dashboards",
+        "type": "dashboards"
+      }
+    ],
+    "panels": [
+      {
+        "datasource": "${datasource}",
+        "description": "The number of Alloy instances whose metrics are being sent and reported.\n",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "instances"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 4,
+          "w": 10,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "colorMode": "none",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "count(alloy_component_controller_evaluating{cluster=\"$cluster\", namespace=\"$namespace\"})",
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Running instances",
+        "type": "stat"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Breakdown of components by health across all running instances.\n\n* Healthy: components have been evaluated completely and are reporting themselves as healthy.\n* Unhealthy: Components either could not be evaluated or are reporting themselves as unhealthy.\n* Unknown: A component has been created but has not yet been started.\n* Exited: A component has exited. It will not return to the running state.\n\nMore information on a component's health state can be retrieved using\nthe Alloy UI.\n\nNote that components may be in a degraded state even if they report\nthemselves as healthy. Use component-specific dashboards and alerts\nto observe detailed information about the behavior of a component.\n",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [],
+            "min": 0,
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                }
+              ]
+            }
+          },
+          "overrides": [
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "Unhealthy"
+              },
+              "properties": [
+                {
+                  "id": "thresholds",
+                  "value": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green",
+                        "value": null
+                      },
+                      {
+                        "color": "red",
+                        "value": 1
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "Unknown"
+              },
+              "properties": [
+                {
+                  "id": "thresholds",
+                  "value": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green",
+                        "value": null
+                      },
+                      {
+                        "color": "blue",
+                        "value": 1
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "Exited"
+              },
+              "properties": [
+                {
+                  "id": "thresholds",
+                  "value": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green",
+                        "value": null
+                      },
+                      {
+                        "color": "orange",
+                        "value": 1
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+          ]
+        },
+        "gridPos": {
+          "h": 12,
+          "w": 14,
+          "x": 10,
+          "y": 0
+        },
+        "id": 4,
+        "options": {
+          "displayMode": "gradient",
+          "maxVizHeight": 300,
+          "minVizHeight": 16,
+          "minVizWidth": 8,
+          "namePlacement": "auto",
+          "orientation": "vertical",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showUnfilled": true,
+          "sizing": "auto",
+          "valueMode": "color"
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"healthy\"}) or vector(0)",
+            "instant": true,
+            "legendFormat": "Healthy",
+            "range": false,
+            "refId": "A"
+          },
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"unhealthy\"}) or vector(0)",
+            "instant": true,
+            "legendFormat": "Unhealthy",
+            "range": false,
+            "refId": "B"
+          },
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"unknown\"}) or vector(0)",
+            "instant": true,
+            "legendFormat": "Unknown",
+            "range": false,
+            "refId": "C"
+          },
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\", health_type=\"exited\"}) or vector(0)",
+            "instant": true,
+            "legendFormat": "Exited",
+            "range": false,
+            "refId": "D"
+          }
+        ],
+        "title": "Components by health",
+        "type": "bargauge"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The number of running components across all running instances.\n",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "components"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 4,
+          "w": 10,
+          "x": 0,
+          "y": 4
+        },
+        "id": 2,
+        "options": {
+          "colorMode": "none",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"})",
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Running components",
+        "type": "stat"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The percentage of components which are in a healthy state.\n",
+        "fieldConfig": {
+          "defaults": {
+            "mappings": [],
+            "max": 1,
+            "min": 0,
+            "noValue": "No components",
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "percentunit"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 4,
+          "w": 10,
+          "x": 0,
+          "y": 8
+        },
+        "id": 3,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showPercentChange": false,
+          "text": {
+            "valueSize": 80
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\",health_type=\"healthy\"}) /\nsum(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"})\n",
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Overall component health",
+        "type": "stat"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The frequency at which components get updated.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "points",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 3,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "ops"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 0,
+          "y": 12
+        },
+        "id": 5,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum by (instance) (rate(alloy_component_evaluation_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Component evaluation rate",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The percentiles for how long it takes to complete component evaluations.\n\nComponent evaluations must complete for components to have the latest\narguments. The longer the evaluations take, the slower it will be to\nreconcile the state of components.\n\nIf evaluation is taking too long, consider sharding your components to\ndeal with smaller amounts of data and reuse data as much as possible.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "s"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 8,
+          "y": 12
+        },
+        "id": 6,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "histogram_quantile(0.99, sum(rate(alloy_component_evaluation_seconds{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))\nor\nhistogram_quantile(0.99, sum by (le) (rate(alloy_component_evaluation_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))\n",
+            "instant": false,
+            "legendFormat": "99th percentile",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": "${datasource}",
+            "expr": "histogram_quantile(0.50, sum(rate(alloy_component_evaluation_seconds{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))\nor\nhistogram_quantile(0.50, sum by (le) (rate(alloy_component_evaluation_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))\n",
+            "instant": false,
+            "legendFormat": "50th percentile",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": "${datasource}",
+            "expr": "(\n  histogram_sum(sum(rate(alloy_component_evaluation_seconds{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))) /\n  histogram_count(sum(rate(alloy_component_evaluation_seconds{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))\n)\nor\n(\n  sum(rate(alloy_component_evaluation_seconds_sum{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])) /\n  sum(rate(alloy_component_evaluation_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))\n)\n",
+            "instant": false,
+            "legendFormat": "Average",
+            "range": true,
+            "refId": "C"
+          }
+        ],
+        "title": "Component evaluation time",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The percentage of time spent evaluating 'slow' components - components that took longer than 1 minute to evaluate.\n\nIdeally, no component should take more than 1 minute to evaluate. The components displayed in this chart\nmay be a sign of a problem with the pipeline.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "percentunit"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 16,
+          "y": 12
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum by (component_path, component_id) (rate(alloy_component_evaluation_slow_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/ scalar(sum(rate(alloy_component_evaluation_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))\n",
+            "instant": false,
+            "legendFormat": "{{component path}} {{component_id}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Slow components evaluation times",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Detailed histogram view of how long component evaluations take.\n\nThe goal is to design your config so that evaluations take as little\ntime as possible; under 100ms is a good goal.\n",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 0,
+          "y": 22
+        },
+        "id": 8,
+        "maxDataPoints": 30,
+        "options": {
+          "calculate": false,
+          "cellGap": 0,
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Spectral",
+            "steps": 64
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 0.1
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "mode": "single",
+            "showColorScale": false,
+            "yHistogram": true
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "reverse": false,
+            "unit": "s"
+          }
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(increase(alloy_component_evaluation_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_evaluation_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n",
+            "format": "heatmap",
+            "instant": false,
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Component evaluation histogram",
+        "type": "heatmap"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Detailed histogram of how long components wait to be evaluated after their dependency is updated.\n\nThe goal is to design your config so that most of the time components do not\nqueue for long; under 10ms is a good goal.\n",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 8,
+          "y": 22
+        },
+        "id": 9,
+        "maxDataPoints": 30,
+        "options": {
+          "calculate": false,
+          "cellGap": 0,
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Spectral",
+            "steps": 64
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 0.1
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "mode": "single",
+            "showColorScale": false,
+            "yHistogram": true
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "reverse": false,
+            "unit": "s"
+          }
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum(increase(alloy_component_dependencies_wait_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\nor ignoring (le)\nsum by (le) (increase(alloy_component_dependencies_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n",
+            "format": "heatmap",
+            "instant": false,
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Component dependency wait histogram",
+        "type": "heatmap"
+      }
+    ],
+    "refresh": "10s",
+    "schemaVersion": 39,
+    "tags": [
+      "alloy-mixin"
+    ],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "Mimir",
+            "value": "mimir_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Data Source",
+          "multi": false,
+          "name": "datasource",
+          "options": [],
+          "query": "prometheus",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Loki",
+            "value": "loki_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Loki Data Source",
+          "multi": false,
+          "name": "loki_datasource",
+          "options": [],
+          "query": "loki",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "cluster",
+          "multi": false,
+          "name": "cluster",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components, cluster)\n",
+            "refId": "cluster"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "namespace",
+          "multi": false,
+          "name": "namespace",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\"}, namespace)\n",
+            "refId": "namespace"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d",
+        "90d"
+      ]
+    },
+    "timezone": "",
+    "title": "Alloy / Controller",
+    "uid": "bf9f456aad7108b2c808dbd9973e386f",
+    "version": 0,
+    "weekStart": ""
+  }

charts/meta-monitoring/src/dashboards/alloy-opentelemetry.json

@@ -0,0 +1,923 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": 25,
+    "links": [
+      {
+        "asDropdown": true,
+        "icon": "external link",
+        "includeVars": true,
+        "keepTime": true,
+        "tags": [
+          "alloy-mixin"
+        ],
+        "targetBlank": false,
+        "title": "Dashboards",
+        "type": "dashboards"
+      }
+    ],
+    "panels": [
+      {
+        "datasource": "${datasource}",
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "title": "Receivers for traces [otelcol.receiver]",
+        "type": "row"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of spans successfully pushed into the pipeline.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 20,
+              "gradientMode": "hue",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 0,
+          "y": 1
+        },
+        "id": 2,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(receiver_accepted_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{ pod }} / {{ transport }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Accepted spans",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of spans that could not be pushed into the pipeline.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 20,
+              "gradientMode": "hue",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 8,
+          "y": 1
+        },
+        "id": 3,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(receiver_refused_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{ pod }} / {{ transport }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Refused spans",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "The duration of inbound RPCs.\n",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 16,
+          "y": 1
+        },
+        "id": 4,
+        "maxDataPoints": 30,
+        "options": {
+          "calculate": false,
+          "cellGap": 1,
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Oranges",
+            "steps": 65
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 1e-9
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "mode": "single",
+            "showColorScale": false,
+            "yHistogram": true
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "reverse": false,
+            "unit": "ms"
+          }
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum by (le) (increase(rpc_server_duration_milliseconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\", rpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\"}[$__rate_interval]))",
+            "format": "heatmap",
+            "instant": false,
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "RPC server duration",
+        "type": "heatmap"
+      },
+      {
+        "datasource": "${datasource}",
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 11
+        },
+        "id": 5,
+        "title": "Batching of logs, metrics, and traces [otelcol.processor.batch]",
+        "type": "row"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of spans, metric datapoints, or log lines in a batch\n",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 0,
+          "y": 12
+        },
+        "id": 6,
+        "maxDataPoints": 30,
+        "options": {
+          "calculate": false,
+          "cellGap": 1,
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Oranges",
+            "steps": 65
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 1e-9
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "mode": "single",
+            "showColorScale": false,
+            "yHistogram": true
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "reverse": false,
+            "unit": "short"
+          }
+        },
+        "pluginVersion": "10.4.2",
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "sum by (le) (increase(processor_batch_batch_send_size_ratio_bucket{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval]))",
+            "format": "heatmap",
+            "instant": false,
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Number of units in the batch",
+        "type": "heatmap"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of distinct metadata value combinations being processed\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 8,
+          "y": 12
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "processor_batch_metadata_cardinality_ratio{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}\n",
+            "instant": false,
+            "legendFormat": "{{ pod }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Distinct metadata values",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of times the batch was sent due to a timeout trigger\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 16,
+          "y": 12
+        },
+        "id": 8,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(processor_batch_timeout_trigger_send_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{ pod }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Timeout trigger",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 22
+        },
+        "id": 9,
+        "title": "Exporters for traces [otelcol.exporter]",
+        "type": "row"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of spans successfully sent to destination.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 20,
+              "gradientMode": "hue",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 0,
+          "y": 23
+        },
+        "id": 10,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(exporter_sent_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{ pod }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Exported sent spans",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of spans in failed attempts to send to destination.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 20,
+              "gradientMode": "hue",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 8,
+          "x": 8,
+          "y": 23
+        },
+        "id": 11,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(exporter_send_failed_spans_ratio_total{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{ pod }}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Exported failed spans",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "10s",
+    "schemaVersion": 39,
+    "tags": [
+      "alloy-mixin"
+    ],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "Mimir",
+            "value": "mimir_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Data Source",
+          "multi": false,
+          "name": "datasource",
+          "options": [],
+          "query": "prometheus",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Loki",
+            "value": "loki_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Loki Data Source",
+          "multi": false,
+          "name": "loki_datasource",
+          "options": [],
+          "query": "loki",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "cluster",
+          "multi": false,
+          "name": "cluster",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components, cluster)\n",
+            "refId": "cluster"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "namespace",
+          "multi": false,
+          "name": "namespace",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\"}, namespace)\n",
+            "refId": "namespace"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "allValue": ".*",
+          "current": {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": true,
+          "label": "instance",
+          "multi": true,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
+            "refId": "instance"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d",
+        "90d"
+      ]
+    },
+    "timezone": "",
+    "title": "Alloy / OpenTelemetry",
+    "uid": "9b6d37c8603e19e8922133984faad93d",
+    "version": 0,
+    "weekStart": ""
+  }

charts/meta-monitoring/src/dashboards/alloy-resources.json

@@ -0,0 +1,840 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        },
+        {
+          "datasource": "$loki_datasource",
+          "enable": true,
+          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"alloy\" | name_extracted=~\"alloy.*\"",
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "instant": false,
+          "name": "Deployments",
+          "titleFormat": "{{cluster}}/{{namespace}}"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": 26,
+    "links": [
+      {
+        "asDropdown": true,
+        "icon": "external link",
+        "includeVars": true,
+        "keepTime": true,
+        "tags": [
+          "alloy-mixin"
+        ],
+        "targetBlank": false,
+        "title": "Dashboards",
+        "type": "dashboards"
+      }
+    ],
+    "panels": [
+      {
+        "datasource": "${datasource}",
+        "description": "CPU usage of the Alloy process relative to 1 CPU core.\n\nFor example, 100% means using one entire CPU core.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "percentunit"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(alloy_resources_process_cpu_seconds_total{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}[$__rate_interval])",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "CPU usage",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Resident memory size of the Alloy process.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "decbytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 0
+        },
+        "id": 2,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "alloy_resources_process_resident_memory_bytes{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Memory (RSS)",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Rate at which the Alloy process performs garbage collections.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "points",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 3,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "ops"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 8,
+          "x": 0,
+          "y": 8
+        },
+        "id": 3,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}[5m])\nand on(instance)\nalloy_build_info{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}\n",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Garbage collections",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Number of goroutines which are running in parallel. An infinitely\ngrowing number of these indicates a goroutine leak.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "none"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 8,
+          "x": 8,
+          "y": 8
+        },
+        "id": 4,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "go_goroutines{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}\nand on(instance)\nalloy_build_info{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}\n",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Goroutines",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Heap memory currently in use by the Alloy process.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "decbytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 8,
+          "x": 16,
+          "y": 8
+        },
+        "id": 5,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}\nand on(instance)\nalloy_build_info{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}\n",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Memory (heap inuse)",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Rate of data received across all network interfaces for the machine\nAlloy is running on.\n\nData shown here is across all running processes and not exclusive to\nthe running Alloy process.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 30,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "Bps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 16
+        },
+        "id": 6,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(alloy_resources_machine_rx_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Network receive bandwidth",
+        "type": "timeseries"
+      },
+      {
+        "datasource": "${datasource}",
+        "description": "Rate of data sent across all network interfaces for the machine\nAlloy is running on.\n\nData shown here is across all running processes and not exclusive to\nthe running Alloy process.\n",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 30,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "Bps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 16
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "${datasource}",
+            "expr": "rate(alloy_resources_machine_tx_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\",instance=~\"$instance\"}[$__rate_interval])\n",
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Network send bandwidth",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "10s",
+    "schemaVersion": 39,
+    "tags": [
+      "alloy-mixin"
+    ],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "Mimir",
+            "value": "mimir_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Data Source",
+          "multi": false,
+          "name": "datasource",
+          "options": [],
+          "query": "prometheus",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Loki",
+            "value": "loki_ds"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "Loki Data Source",
+          "multi": false,
+          "name": "loki_datasource",
+          "options": [],
+          "query": "loki",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "cluster",
+          "multi": false,
+          "name": "cluster",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components, cluster)\n",
+            "refId": "cluster"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "current": {
+            "isNone": true,
+            "selected": false,
+            "text": "None",
+            "value": ""
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": false,
+          "label": "namespace",
+          "multi": false,
+          "name": "namespace",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\"}, namespace)\n",
+            "refId": "namespace"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        },
+        {
+          "allValue": ".*",
+          "current": {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          "datasource": {
+            "uid": "${datasource}"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": true,
+          "label": "instance",
+          "multi": true,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(alloy_component_controller_running_components{cluster=\"$cluster\", namespace=\"$namespace\"}, instance)\n",
+            "refId": "instance"
+          },
+          "refresh": 2,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 2,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d",
+        "90d"
+      ]
+    },
+    "timezone": "",
+    "title": "Alloy / Resources",
+    "uid": "d6a8574c31f3d7cb8f1345ec84d15a67",
+    "version": 0,
+    "weekStart": ""
+  }

charts/meta-monitoring/src/dashboards/loki-deletion.json

@@ -35,6 +35,7 @@
                   "fill": 1,
                   "format": "none",
                   "id": 1,
+                  "interval": "1m",
                   "legend": {
                      "avg": false,
                      "current": false,
@@ -62,7 +63,6 @@
                         "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                         "format": "time_series",
                         "instant": true,
-                        "intervalFactor": 2,
                         "refId": "A"
                      }
                   ],
@@ -111,6 +111,7 @@
                   "fill": 1,
                   "format": "dtdurations",
                   "id": 2,
+                  "interval": "1m",
                   "legend": {
                      "avg": false,
                      "current": false,
@@ -138,7 +139,6 @@
                         "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                         "format": "time_series",
                         "instant": true,
-                        "intervalFactor": 2,
                         "refId": "A"
                      }
                   ],
@@ -191,232 +191,148 @@
             "height": "250px",
             "panels": [
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 3,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 3,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "in progress",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "# of Delete Requests (received - processed) ",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                },
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 4,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 4,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "received",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Delete Requests Received / Day",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                },
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 5,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 5,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "processed",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Delete Requests Processed / Day",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                }
             ],
             "repeat": null,
@@ -431,232 +347,148 @@
             "height": "250px",
             "panels": [
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 6,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 6,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Compactor CPU usage",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                },
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 7,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 7,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": " {{pod}} ",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Compactor memory usage (MiB)",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                },
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 8,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 8,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Compaction run duration (seconds)",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                }
             ],
             "repeat": null,
@@ -671,156 +503,100 @@
             "height": "250px",
             "panels": [
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 9,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 9,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "failures",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Failures in Loading Delete Requests / Hour",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                },
                {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
                   "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 10,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
                   },
-                  "lines": true,
-                  "linewidth": 1,
+                  "id": 10,
+                  "interval": "1m",
                   "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
                   "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)",
+                        "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*/compactor|(loki|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
                         "format": "time_series",
-                        "intervalFactor": 2,
                         "legendFormat": "{{user}}",
-                        "legendLink": null,
-                        "step": 10
+                        "legendLink": null
                      }
                   ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
                   "title": "Lines Deleted / Sec",
-                  "tooltip": {
-                     "shared": true,
-                     "sort": 2,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
+                  "type": "timeseries"
                }
             ],
             "repeat": null,
@@ -837,10 +613,11 @@
                {
                   "datasource": "$loki_datasource",
                   "id": 11,
+                  "interval": "1m",
                   "span": 6,
                   "targets": [
                      {
-                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
+                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*/compactor|(loki|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
                         "refId": "A"
                      }
                   ],
@@ -850,10 +627,11 @@
                {
                   "datasource": "$loki_datasource",
                   "id": 12,
+                  "interval": "1m",
                   "span": 6,
                   "targets": [
                      {
-                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
+                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*/compactor|(loki|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
                         "refId": "A"
                      }
                   ],
@@ -882,7 +660,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data Source",
+               "label": "Data source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",
@@ -935,6 +713,16 @@
                "tagsQuery": "",
                "type": "query",
                "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
             }
          ]
       },

charts/meta-monitoring/src/dashboards/loki-logs.json

@@ -38,6 +38,7 @@
             },
             "hiddenSeries": false,
             "id": 35,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -77,7 +78,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -114,6 +115,11 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "s"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -124,6 +130,7 @@
             },
             "hiddenSeries": false,
             "id": 41,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -164,7 +171,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -211,6 +218,7 @@
             },
             "hiddenSeries": false,
             "id": 36,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -236,7 +244,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))",
+                  "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[$__rate_interval]))",
                   "refId": "A"
                }
             ],
@@ -250,7 +258,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -287,6 +295,11 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "bytes"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -297,6 +310,7 @@
             },
             "hiddenSeries": false,
             "id": 40,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -336,7 +350,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -373,6 +387,11 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "binBps"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -383,6 +402,7 @@
             },
             "hiddenSeries": false,
             "id": 38,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -408,7 +428,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
+                  "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
                   "refId": "A"
                }
             ],
@@ -422,7 +442,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -459,6 +479,11 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "binBps"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -469,6 +494,7 @@
             },
             "hiddenSeries": false,
             "id": 39,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -494,7 +520,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
+                  "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))",
                   "refId": "A"
                }
             ],
@@ -508,7 +534,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -555,6 +581,7 @@
             },
             "hiddenSeries": false,
             "id": 37,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -595,7 +622,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -632,6 +659,11 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "ops"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -642,6 +674,7 @@
             },
             "hiddenSeries": false,
             "id": 42,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -667,7 +700,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)",
+                  "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)",
                   "legendFormat": "{{level}}",
                   "refId": "A"
                }
@@ -682,7 +715,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -718,7 +751,12 @@
             "bars": false,
             "dashLength": 10,
             "dashes": false,
-            "datasource": "$logs",
+            "datasource": "$loki_datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "ops"
+               }
+            },
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -729,6 +767,7 @@
             },
             "hiddenSeries": false,
             "id": 31,
+            "interval": "1m",
             "legend": {
                "avg": false,
                "current": false,
@@ -771,7 +810,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)",
+                  "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" | __error__=\"\" [$__interval])) by (level)",
                   "intervalFactor": 3,
                   "legendFormat": "{{level}}",
                   "refId": "A"
@@ -787,7 +826,7 @@
                "sort": 2,
                "value_type": "individual"
             },
-            "type": "graph",
+            "type": "timeseries",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -819,7 +858,7 @@
             }
          },
          {
-            "datasource": "$logs",
+            "datasource": "$loki_datasource",
             "gridPos": {
                "h": 19,
                "w": 24,
@@ -827,6 +866,7 @@
                "y": 6
             },
             "id": 29,
+            "interval": "1m",
             "maxDataPoints": "",
             "options": {
                "showLabels": false,
@@ -861,7 +901,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data Source",
+               "label": "Data source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",
@@ -918,7 +958,7 @@
             {
                "hide": 0,
                "label": null,
-               "name": "logs",
+               "name": "loki_datasource",
                "options": [ ],
                "query": "loki",
                "refresh": 1,

charts/meta-monitoring/src/dashboards/loki-mixin-recording-rules.json

@@ -1,723 +0,0 @@
-{
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "fiscalYearStartMonth": 0,
-      "gnetId": null,
-      "graphTooltip": 0,
-      "hideControls": false,
-      "iteration": 1635347545534,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "loki"
-            ],
-            "targetBlank": false,
-            "title": "Loki Dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "liveNow": false,
-      "panels": [
-         {
-            "datasource": "${datasource}",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "thresholds"
-                  },
-                  "mappings": [ ],
-                  "noValue": "0",
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 1
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 2,
-               "x": 0,
-               "y": 0
-            },
-            "id": 2,
-            "options": {
-               "colorMode": "value",
-               "graphMode": "area",
-               "justifyMode": "auto",
-               "orientation": "auto",
-               "reduceOptions": {
-                  "calcs": [
-                     "lastNotNull"
-                  ],
-                  "fields": "",
-                  "values": false
-               },
-               "textMode": "auto"
-            },
-            "pluginVersion": "8.3.0-38205pre",
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": false,
-                  "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0",
-                  "instant": true,
-                  "interval": "",
-                  "legendFormat": "",
-                  "refId": "A"
-               }
-            ],
-            "title": "Appenders Not Ready",
-            "type": "stat"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 11,
-               "x": 2,
-               "y": 0
-            },
-            "id": 4,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "Samples Appended to WAL per Second",
-            "type": "timeseries"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "Series are unique combinations of labels",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 11,
-               "x": 13,
-               "y": 0
-            },
-            "id": 5,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "Series Created per Second",
-            "type": "timeseries"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 12,
-               "x": 0,
-               "y": 10
-            },
-            "id": 6,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n  (\n    loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n    or vector(0)\n  )",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "Write Behind",
-            "type": "timeseries"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 12,
-               "x": 12,
-               "y": 10
-            },
-            "id": 7,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "Samples Sent per Second",
-            "type": "timeseries"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "\n",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  },
-                  "unit": "bytes"
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 12,
-               "x": 0,
-               "y": 20
-            },
-            "id": 8,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "WAL Disk Size",
-            "type": "timeseries"
-         },
-         {
-            "datasource": "${datasource}",
-            "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high",
-            "fieldConfig": {
-               "defaults": {
-                  "color": {
-                     "mode": "palette-classic"
-                  },
-                  "custom": {
-                     "axisLabel": "",
-                     "axisPlacement": "auto",
-                     "barAlignment": 0,
-                     "drawStyle": "line",
-                     "fillOpacity": 0,
-                     "gradientMode": "none",
-                     "hideFrom": {
-                        "legend": false,
-                        "tooltip": false,
-                        "viz": false
-                     },
-                     "lineInterpolation": "linear",
-                     "lineWidth": 1,
-                     "pointSize": 5,
-                     "scaleDistribution": {
-                        "type": "linear"
-                     },
-                     "showPoints": "auto",
-                     "spanNulls": false,
-                     "stacking": {
-                        "group": "A",
-                        "mode": "none"
-                     },
-                     "thresholdsStyle": {
-                        "mode": "off"
-                     }
-                  },
-                  "mappings": [ ],
-                  "thresholds": {
-                     "mode": "absolute",
-                     "steps": [
-                        {
-                           "color": "green",
-                           "value": null
-                        },
-                        {
-                           "color": "red",
-                           "value": 80
-                        }
-                     ]
-                  }
-               },
-               "overrides": [ ]
-            },
-            "gridPos": {
-               "h": 10,
-               "w": 12,
-               "x": 12,
-               "y": 20
-            },
-            "id": 9,
-            "options": {
-               "legend": {
-                  "calcs": [ ],
-                  "displayMode": "list",
-                  "placement": "bottom"
-               },
-               "tooltip": {
-                  "mode": "single"
-               }
-            },
-            "targets": [
-               {
-                  "datasource": "${datasource}",
-                  "exemplar": true,
-                  "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0",
-                  "interval": "",
-                  "legendFormat": "{{tenant}}",
-                  "refId": "A"
-               }
-            ],
-            "title": "Pending Samples",
-            "type": "timeseries"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [ ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "loki"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": null,
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "cluster",
-               "multi": false,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(loki_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 2,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": null,
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "namespace",
-               "multi": false,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 2,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "hide": 0,
-               "label": null,
-               "name": "loki_datasource",
-               "options": [ ],
-               "query": "loki",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".+",
-               "current": { },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": null,
-               "multi": false,
-               "name": "tenant",
-               "options": [ ],
-               "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))",
-               "refresh": 0,
-               "regex": "/\"([^\"]+)\"/",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Loki / Recording Rules",
-      "uid": "recording-rules",
-      "version": 0,
-      "weekStart": ""
-   }

charts/meta-monitoring/src/dashboards/mimir-alertmanager-resources.json

@@ -1,836 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 1,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [
-                     {
-                        "alias": "request",
-                        "color": "#FFC000",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     },
-                     {
-                        "alias": "limit",
-                        "color": "#E02F44",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     }
-                  ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "limit",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"cpu\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "request",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "CPU",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 2,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [
-                     {
-                        "alias": "request",
-                        "color": "#FFC000",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     },
-                     {
-                        "alias": "limit",
-                        "color": "#E02F44",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     }
-                  ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} > 0)",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "limit",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"memory\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "request",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Memory (workingset)",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "bytes",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 3,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Memory (go heap inuse)",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "bytes",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Alertmanager",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 4,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Receive bandwidth",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 5,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Transmit bandwidth",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Network",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 6,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"alertmanager\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk writes",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 7,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"alertmanager\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk reads",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Disk",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 8,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 12,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(alertmanager).*\"\n  }\n)\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{persistentvolumeclaim}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk space utilization",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "percentunit",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".*",
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": false,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": null,
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "namespace",
-               "multi": false,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Alertmanager resources",
-      "uid": "a6883fb22799ac74479c7db872451092",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-compactor-resources.json

@@ -1,940 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 1,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [
-                     {
-                        "alias": "request",
-                        "color": "#FFC000",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     },
-                     {
-                        "alias": "limit",
-                        "color": "#E02F44",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     }
-                  ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "limit",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"cpu\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "request",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "CPU",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 2,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Memory (go heap inuse)",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "bytes",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "CPU and memory",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 3,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [
-                     {
-                        "alias": "request",
-                        "color": "#FFC000",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     },
-                     {
-                        "alias": "limit",
-                        "color": "#E02F44",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     }
-                  ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "limit",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "request",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Memory (RSS)",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "bytes",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 4,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [
-                     {
-                        "alias": "request",
-                        "color": "#FFC000",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     },
-                     {
-                        "alias": "limit",
-                        "color": "#E02F44",
-                        "dashLength": 5,
-                        "dashes": true,
-                        "fill": 0
-                     }
-                  ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "limit",
-                        "legendLink": null
-                     },
-                     {
-                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "request",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Memory (workingset)",
-                  "tooltip": {
-                     "sort": 2
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "bytes",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 5,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Receive bandwidth",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 6,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Transmit bandwidth",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Network",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 7,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"compactor\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk writes",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 8,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"compactor\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk reads",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "Bps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 0,
-                  "id": 9,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(compactor).*\"\n  }\n)\n",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{persistentvolumeclaim}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Disk space utilization",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "percentunit",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Disk",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": true,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "namespace",
-               "multi": true,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Compactor resources",
-      "uid": "09a5c49e9cdb2f2b24c6d184574a07fd",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-config.json

@@ -1,312 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 1,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 12,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "sha256:{{sha256}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Startup config file hashes",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "instances",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Startup config file",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 2,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 12,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "sha256:{{sha256}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Runtime config file hashes",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "instances",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Runtime config file",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": true,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "namespace",
-               "multi": true,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Config",
-      "uid": "5d9d0b4724c0f80d68467088ec61e003",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-object-store.json

@@ -1,938 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 1,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{component}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "RPS / component",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "reqps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "max": 1,
-                        "min": 0,
-                        "noValue": "0",
-                        "unit": "percentunit"
-                     }
-                  },
-                  "id": 2,
-                  "links": [ ],
-                  "options": {
-                     "legend": {
-                        "showLegend": true
-                     },
-                     "tooltip": {
-                        "mode": "single",
-                        "sort": "none"
-                     }
-                  },
-                  "span": 6,
-                  "targets": [
-                     {
-                        "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{component}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "title": "Error rate / component",
-                  "type": "timeseries"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Components",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 10,
-                  "id": 3,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 6,
-                  "stack": true,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{operation}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "RPS / operation",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "reqps",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "max": 1,
-                        "min": 0,
-                        "noValue": "0",
-                        "unit": "percentunit"
-                     }
-                  },
-                  "id": 4,
-                  "links": [ ],
-                  "options": {
-                     "legend": {
-                        "showLegend": true
-                     },
-                     "tooltip": {
-                        "mode": "single",
-                        "sort": "none"
-                     }
-                  },
-                  "span": 6,
-                  "targets": [
-                     {
-                        "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "{{operation}}",
-                        "legendLink": null
-                     }
-                  ],
-                  "title": "Error rate / operation",
-                  "type": "timeseries"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Operations",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 5,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: Get",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 6,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: GetRange",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 7,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: Exists",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 8,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: Attributes",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 9,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: Upload",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               },
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 10,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "spaceLength": 10,
-                  "span": 4,
-                  "stack": false,
-                  "steppedLine": false,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "99th Percentile",
-                        "refId": "A"
-                     },
-                     {
-                        "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "50th Percentile",
-                        "refId": "B"
-                     },
-                     {
-                        "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))",
-                        "format": "time_series",
-                        "intervalFactor": 2,
-                        "legendFormat": "Average",
-                        "refId": "C"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Op: Delete",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "type": "graph",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "ms",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": true,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "namespace",
-               "multi": true,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Object Store",
-      "uid": "e1324ee2a434f4158c00a9ee279d3292",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-overrides.json

@@ -1,266 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "datasource": "${datasource}",
-                  "id": 1,
-                  "span": 12,
-                  "targets": [
-                     {
-                        "expr": "max by(limit_name) (cortex_limits_defaults{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
-                        "instant": true,
-                        "legendFormat": "",
-                        "refId": "A"
-                     }
-                  ],
-                  "title": "Defaults",
-                  "transformations": [
-                     {
-                        "id": "labelsToFields",
-                        "options": { }
-                     },
-                     {
-                        "id": "merge",
-                        "options": { }
-                     },
-                     {
-                        "id": "organize",
-                        "options": {
-                           "excludeByName": {
-                              "Time": true
-                           },
-                           "indexByName": {
-                              "Value": 1,
-                              "limit_name": 0
-                           }
-                        }
-                     },
-                     {
-                        "id": "sortBy",
-                        "options": {
-                           "fields": { },
-                           "sort": [
-                              {
-                                 "field": "limit_name"
-                              }
-                           ]
-                        }
-                     }
-                  ],
-                  "type": "table"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "datasource": "${datasource}",
-                  "id": 2,
-                  "span": 12,
-                  "targets": [
-                     {
-                        "expr": "max by(user, limit_name) (cortex_limits_overrides{cluster=~\"$cluster\",namespace=~\"$namespace\",user=~\"${tenant_id}\"})",
-                        "instant": true,
-                        "legendFormat": "",
-                        "refId": "A"
-                     }
-                  ],
-                  "title": "Per-tenant overrides",
-                  "transformations": [
-                     {
-                        "id": "labelsToFields",
-                        "options": {
-                           "mode": "columns",
-                           "valueLabel": "limit_name"
-                        }
-                     },
-                     {
-                        "id": "merge",
-                        "options": { }
-                     },
-                     {
-                        "id": "organize",
-                        "options": {
-                           "excludeByName": {
-                              "Time": true
-                           },
-                           "indexByName": {
-                              "user": 0
-                           }
-                        }
-                     }
-                  ],
-                  "type": "table"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".*",
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": false,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": null,
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "namespace",
-               "multi": false,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "current": {
-                  "selected": true,
-                  "text": ".*",
-                  "value": ".*"
-               },
-               "hide": 0,
-               "label": "Tenant ID",
-               "name": "tenant_id",
-               "options": [
-                  {
-                     "selected": true,
-                     "text": ".*",
-                     "value": ".*"
-                  }
-               ],
-               "query": ".*",
-               "type": "textbox"
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Overrides",
-      "uid": "1e2c358600ac53f09faea133f811b5bb",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-scaling.json

@@ -1,362 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "10s",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "200px",
-            "panels": [
-               {
-                  "id": 1,
-                  "options": {
-                     "content": "This dashboard identifies scaling-related issues by suggesting services that you might want to scale up.\nThe table that follows contains a suggested number of replicas and the reason why.\nIf the system is failing and depending on the reason, try scaling up to the specified number.\nThe specified numbers are intended as helpful guidelines when things go wrong, rather than prescriptive guidelines.\n\nReasons:\n- **sample_rate**: There are not enough replicas to handle the\n  sample rate.  Applies to distributor and ingesters.\n- **active_series**: There are not enough replicas\n  to handle the number of active series.  Applies to ingesters.\n- **cpu_usage**: There are not enough replicas\n  based on the CPU usage of the jobs vs the resource requests.\n  Applies to all jobs.\n- **memory_usage**: There are not enough replicas based on the memory\n  usage vs the resource requests.  Applies to all jobs.\n- **active_series_limits**: There are not enough replicas to hold 60% of the\n  sum of all the per tenant series limits.\n- **sample_rate_limits**: There are not enough replicas to handle 60% of the\n  sum of all the per tenant rate limits.\n",
-                     "mode": "markdown"
-                  },
-                  "span": 12,
-                  "title": "",
-                  "type": "text"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Service scaling",
-            "titleSize": "h6"
-         },
-         {
-            "collapse": false,
-            "height": "400px",
-            "panels": [
-               {
-                  "aliasColors": { },
-                  "bars": false,
-                  "dashLength": 10,
-                  "dashes": false,
-                  "datasource": "$datasource",
-                  "fill": 1,
-                  "id": 2,
-                  "legend": {
-                     "avg": false,
-                     "current": false,
-                     "max": false,
-                     "min": false,
-                     "show": true,
-                     "total": false,
-                     "values": false
-                  },
-                  "lines": true,
-                  "linewidth": 1,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "percentage": false,
-                  "pointradius": 5,
-                  "points": false,
-                  "renderer": "flot",
-                  "seriesOverrides": [ ],
-                  "sort": {
-                     "col": 0,
-                     "desc": false
-                  },
-                  "spaceLength": 10,
-                  "span": 12,
-                  "stack": false,
-                  "steppedLine": false,
-                  "styles": [
-                     {
-                        "alias": "Time",
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "pattern": "Time",
-                        "type": "hidden"
-                     },
-                     {
-                        "alias": "Required Replicas",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 0,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "Value",
-                        "thresholds": [ ],
-                        "type": "number",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "Cluster",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "__name__",
-                        "thresholds": [ ],
-                        "type": "hidden",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "Cluster",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "cluster",
-                        "thresholds": [ ],
-                        "type": "number",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "Service",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "deployment",
-                        "thresholds": [ ],
-                        "type": "number",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "Namespace",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "namespace",
-                        "thresholds": [ ],
-                        "type": "number",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "Reason",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "link": false,
-                        "linkTargetBlank": false,
-                        "linkTooltip": "Drill down",
-                        "linkUrl": "",
-                        "pattern": "reason",
-                        "thresholds": [ ],
-                        "type": "number",
-                        "unit": "short"
-                     },
-                     {
-                        "alias": "",
-                        "colorMode": null,
-                        "colors": [ ],
-                        "dateFormat": "YYYY-MM-DD HH:mm:ss",
-                        "decimals": 2,
-                        "pattern": "/.*/",
-                        "thresholds": [ ],
-                        "type": "string",
-                        "unit": "short"
-                     }
-                  ],
-                  "targets": [
-                     {
-                        "expr": "sort_desc(\n  cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    > ignoring(reason) group_left\n  cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n",
-                        "format": "table",
-                        "instant": true,
-                        "intervalFactor": 2,
-                        "legendFormat": "",
-                        "refId": "A"
-                     }
-                  ],
-                  "thresholds": [ ],
-                  "timeFrom": null,
-                  "timeShift": null,
-                  "title": "Workload-based scaling",
-                  "tooltip": {
-                     "shared": false,
-                     "sort": 0,
-                     "value_type": "individual"
-                  },
-                  "transform": "table",
-                  "type": "table",
-                  "xaxis": {
-                     "buckets": null,
-                     "mode": "time",
-                     "name": null,
-                     "show": true,
-                     "values": [ ]
-                  },
-                  "yaxes": [
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": 0,
-                        "show": true
-                     },
-                     {
-                        "format": "short",
-                        "label": null,
-                        "logBase": 1,
-                        "max": null,
-                        "min": null,
-                        "show": false
-                     }
-                  ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Scaling",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": true,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": ".+",
-               "current": {
-                  "selected": true,
-                  "text": "All",
-                  "value": "$__all"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "namespace",
-               "multi": true,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Scaling",
-      "uid": "64bbad83507b7289b514725658e10352",
-      "version": 0
-   }

charts/meta-monitoring/src/dashboards/mimir-slow-queries.json

@@ -1,323 +0,0 @@
-{
-      "__requires": [
-         {
-            "id": "grafana",
-            "name": "Grafana",
-            "type": "grafana",
-            "version": "8.0.0"
-         }
-      ],
-      "annotations": {
-         "list": [ ]
-      },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 1,
-      "hideControls": false,
-      "links": [
-         {
-            "asDropdown": true,
-            "icon": "external link",
-            "includeVars": true,
-            "keepTime": true,
-            "tags": [
-               "mimir"
-            ],
-            "targetBlank": false,
-            "title": "Mimir dashboards",
-            "type": "dashboards"
-         }
-      ],
-      "refresh": "",
-      "rows": [
-         {
-            "collapse": false,
-            "height": "250px",
-            "panels": [
-               {
-                  "datasource": "${lokidatasource}",
-                  "fieldConfig": {
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "Time range"
-                           },
-                           "properties": [
-                              {
-                                 "id": "mappings",
-                                 "value": [
-                                    {
-                                       "from": "",
-                                       "id": 1,
-                                       "text": "Instant query",
-                                       "to": "",
-                                       "type": 1,
-                                       "value": "0"
-                                    }
-                                 ]
-                              },
-                              {
-                                 "id": "unit",
-                                 "value": "s"
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "Step"
-                           },
-                           "properties": [
-                              {
-                                 "id": "unit",
-                                 "value": "s"
-                              }
-                           ]
-                        }
-                     ]
-                  },
-                  "id": 1,
-                  "span": 12,
-                  "targets": [
-                     {
-                        "expr": "{cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | response_time > ${min_duration}",
-                        "instant": false,
-                        "legendFormat": "",
-                        "range": true,
-                        "refId": "A"
-                     }
-                  ],
-                  "title": "Slow queries",
-                  "transformations": [
-                     {
-                        "id": "extractFields",
-                        "options": {
-                           "source": "labels"
-                        }
-                     },
-                     {
-                        "id": "calculateField",
-                        "options": {
-                           "alias": "Time range",
-                           "binary": {
-                              "left": "param_end",
-                              "operator": "-",
-                              "reducer": "sum",
-                              "right": "param_start"
-                           },
-                           "mode": "binary",
-                           "reduce": {
-                              "reducer": "sum"
-                           },
-                           "replaceFields": false
-                        }
-                     },
-                     {
-                        "id": "organize",
-                        "options": {
-                           "excludeByName": {
-                              "Line": true,
-                              "Time": true,
-                              "caller": true,
-                              "cluster": true,
-                              "container": true,
-                              "host": true,
-                              "id": true,
-                              "job": true,
-                              "labels": true,
-                              "level": true,
-                              "line": true,
-                              "method": true,
-                              "msg": true,
-                              "name": true,
-                              "namespace": true,
-                              "param_end": true,
-                              "param_start": true,
-                              "param_time": true,
-                              "path": true,
-                              "pod": true,
-                              "pod_template_hash": true,
-                              "query_wall_time_seconds": true,
-                              "stream": true,
-                              "traceID": true,
-                              "tsNs": true
-                           },
-                           "indexByName": {
-                              "Time range": 3,
-                              "param_query": 2,
-                              "param_step": 4,
-                              "response_time": 5,
-                              "ts": 0,
-                              "user": 1
-                           },
-                           "renameByName": {
-                              "org_id": "Tenant ID",
-                              "param_query": "Query",
-                              "param_step": "Step",
-                              "response_time": "Duration"
-                           }
-                        }
-                     }
-                  ],
-                  "type": "table"
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "",
-            "titleSize": "h6"
-         }
-      ],
-      "schemaVersion": 14,
-      "style": "dark",
-      "tags": [
-         "mimir"
-      ],
-      "templating": {
-         "list": [
-            {
-               "current": {
-                  "text": "default",
-                  "value": "default"
-               },
-               "hide": 0,
-               "label": "Data Source",
-               "name": "datasource",
-               "options": [ ],
-               "query": "prometheus",
-               "refresh": 1,
-               "regex": "",
-               "type": "datasource"
-            },
-            {
-               "allValue": ".*",
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": true,
-               "label": "cluster",
-               "multi": false,
-               "name": "cluster",
-               "options": [ ],
-               "query": "label_values(cortex_build_info, cluster)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "allValue": null,
-               "current": {
-                  "text": "prod",
-                  "value": "prod"
-               },
-               "datasource": "$datasource",
-               "hide": 0,
-               "includeAll": false,
-               "label": "namespace",
-               "multi": false,
-               "name": "namespace",
-               "options": [ ],
-               "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
-               "refresh": 1,
-               "regex": "",
-               "sort": 1,
-               "tagValuesQuery": "",
-               "tags": [ ],
-               "tagsQuery": "",
-               "type": "query",
-               "useTags": false
-            },
-            {
-               "hide": 0,
-               "includeAll": false,
-               "label": "Logs datasource",
-               "multi": false,
-               "name": "lokidatasource",
-               "query": "loki",
-               "type": "datasource"
-            },
-            {
-               "current": {
-                  "selected": true,
-                  "text": "5s",
-                  "value": "5s"
-               },
-               "hide": 0,
-               "label": "Min duration",
-               "name": "min_duration",
-               "options": [
-                  {
-                     "selected": true,
-                     "text": "5s",
-                     "value": "5s"
-                  }
-               ],
-               "query": "5s",
-               "type": "textbox"
-            },
-            {
-               "current": {
-                  "selected": true,
-                  "text": ".*",
-                  "value": ".*"
-               },
-               "hide": 0,
-               "label": "Tenant ID",
-               "name": "tenant_id",
-               "options": [
-                  {
-                     "selected": true,
-                     "text": ".*",
-                     "value": ".*"
-                  }
-               ],
-               "query": ".*",
-               "type": "textbox"
-            }
-         ]
-      },
-      "time": {
-         "from": "now-1h",
-         "to": "now"
-      },
-      "timepicker": {
-         "refresh_intervals": [
-            "5s",
-            "10s",
-            "30s",
-            "1m",
-            "5m",
-            "15m",
-            "30m",
-            "1h",
-            "2h",
-            "1d"
-         ],
-         "time_options": [
-            "5m",
-            "15m",
-            "1h",
-            "6h",
-            "12h",
-            "24h",
-            "2d",
-            "7d",
-            "30d"
-         ]
-      },
-      "timezone": "utc",
-      "title": "Mimir / Slow queries",
-      "uid": "6089e1ce1e678788f46312a0a1e647e6",
-      "version": 0
-   }

charts/meta-monitoring/src/rules/loki-rules.yaml

@@ -1,53 +1,53 @@
 groups:
-- name: loki_rules
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:loki_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:loki_request_duration_seconds:50quantile
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[1m]))
-      by (cluster, job)
-    record: cluster_job:loki_request_duration_seconds:avg
-  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job)
-    record: cluster_job:loki_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job)
-    record: cluster_job:loki_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job)
-    record: cluster_job:loki_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:loki_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:loki_request_duration_seconds:50quantile
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
-    record: cluster_job_route:loki_request_duration_seconds:avg
-  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job,
-      route)
-    record: cluster_job_route:loki_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
-    record: cluster_job_route:loki_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)
-    record: cluster_job_route:loki_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:loki_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:loki_request_duration_seconds:50quantile
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster,
-      namespace, job, route)
-    record: cluster_namespace_job_route:loki_request_duration_seconds:avg
-  - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate
+  - name: "loki_rules"
+    rules:
+    - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, job))"
+      record: "cluster_job:loki_request_duration_seconds:99quantile"
+    - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, job))"
+      record: "cluster_job:loki_request_duration_seconds:50quantile"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[5m]))
+        by (cluster, job)"
+      record: "cluster_job:loki_request_duration_seconds:avg"
+    - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job)"
+      record: "cluster_job:loki_request_duration_seconds_bucket:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job)"
+      record: "cluster_job:loki_request_duration_seconds_sum:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job)"
+      record: "cluster_job:loki_request_duration_seconds_count:sum_rate"
+    - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, job, route))"
+      record: "cluster_job_route:loki_request_duration_seconds:99quantile"
+    - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, job, route))"
+      record: "cluster_job_route:loki_request_duration_seconds:50quantile"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)
+        / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
+      record: "cluster_job_route:loki_request_duration_seconds:avg"
+    - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job,
+        route)"
+      record: "cluster_job_route:loki_request_duration_seconds_bucket:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)"
+      record: "cluster_job_route:loki_request_duration_seconds_sum:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
+      record: "cluster_job_route:loki_request_duration_seconds_count:sum_rate"
+    - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, namespace, job, route))"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds:99quantile"
+    - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+        by (le, cluster, namespace, job, route))"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds:50quantile"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
+        job, route) / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster,
+        namespace, job, route)"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds:avg"
+    - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, namespace,
+        job, route)"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
+        job, route)"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate"
+    - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, namespace,
+        job, route)"
+      record: "cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate"

charts/meta-monitoring/src/rules/mimir-rules.yaml

@@ -1,571 +0,0 @@
-groups:
-- name: mimir_api_1
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
-      by (cluster, job)
-    record: cluster_job:cortex_request_duration_seconds:avg
-  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
-    record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_request_duration_seconds_count:sum_rate
-- name: mimir_api_2
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:cortex_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:cortex_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
-    record: cluster_job_route:cortex_request_duration_seconds:avg
-  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
-      route)
-    record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
-    record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
-    record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
-- name: mimir_api_3
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
-      namespace, job, route)
-    record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
-  - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
-      job, route)
-    record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
-- name: mimir_querier_api
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_querier_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_querier_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_querier_request_duration_seconds:avg
-  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job)
-    record: cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_querier_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:cortex_querier_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, route))
-    record: cluster_job_route:cortex_querier_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
-      (cluster, job, route)
-    record: cluster_job_route:cortex_querier_request_duration_seconds:avg
-  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job, route)
-    record: cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job, route)
-    record: cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
-      job, route)
-    record: cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
-      by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
-      by (cluster, namespace, job, route)
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg
-  - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
-      namespace, job, route)
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      namespace, job, route)
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
-      namespace, job, route)
-    record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate
-- name: mimir_cache
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, method))
-    record: cluster_job_method:cortex_memcache_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, method))
-    record: cluster_job_method:cortex_memcache_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
-      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
-      by (cluster, job, method)
-    record: cluster_job_method:cortex_memcache_request_duration_seconds:avg
-  - expr: sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job, method)
-    record: cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
-      job, method)
-    record: cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
-      job, method)
-    record: cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_cache_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_cache_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_cache_request_duration_seconds:avg
-  - expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job)
-    record: cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_cache_request_duration_seconds_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, method))
-    record: cluster_job_method:cortex_cache_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job, method))
-    record: cluster_job_method:cortex_cache_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
-      method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
-      job, method)
-    record: cluster_job_method:cortex_cache_request_duration_seconds:avg
-  - expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job, method)
-    record: cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
-      method)
-    record: cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
-      job, method)
-    record: cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate
-- name: mimir_storage
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_kv_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_kv_request_duration_seconds:50quantile
-  - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_kv_request_duration_seconds:avg
-  - expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
-      job)
-    record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
-- name: mimir_queries
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_query_frontend_retries:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_query_frontend_retries:50quantile
-  - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
-      by (cluster, job)
-    record: cluster_job:cortex_query_frontend_retries:avg
-  - expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
-    record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
-  - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
-  - expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_query_frontend_retries_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
-  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
-      (cluster, job)
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
-  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
-      cluster, job)
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
-  - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
-      job)
-    record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
-- name: mimir_ingester_queries
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_series:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_series:50quantile
-  - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
-      by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_series:avg
-  - expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
-    record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_series_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_samples:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_samples:50quantile
-  - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
-      by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_samples:avg
-  - expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
-    record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
-  - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_exemplars:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
-      by (le, cluster, job))
-    record: cluster_job:cortex_ingester_queried_exemplars:50quantile
-  - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
-      sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_exemplars:avg
-  - expr: sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
-      job)
-    record: cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate
-  - expr: sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
-    record: cluster_job:cortex_ingester_queried_exemplars_count:sum_rate
-- name: mimir_received_samples
-  rules:
-  - expr: |
-      sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
-    record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
-- name: mimir_exemplars_in
-  rules:
-  - expr: |
-      sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))
-    record: cluster_namespace_job:cortex_distributor_exemplars_in:rate5m
-- name: mimir_received_exemplars
-  rules:
-  - expr: |
-      sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))
-    record: cluster_namespace_job:cortex_distributor_received_exemplars:rate5m
-- name: mimir_exemplars_ingested
-  rules:
-  - expr: |
-      sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))
-    record: cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m
-- name: mimir_exemplars_appended
-  rules:
-  - expr: |
-      sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))
-    record: cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m
-- name: mimir_scaling_rules
-  rules:
-  - expr: |
-      # Convenience rule to get the number of replicas for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
-      sum by (cluster, namespace, deployment) (
-        label_replace(
-          kube_deployment_spec_replicas,
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-          # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-        )
-      )
-      or
-      sum by (cluster, namespace, deployment) (
-        label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
-      )
-    record: cluster_namespace_deployment:actual_replicas:count
-  - expr: |
-      ceil(
-        quantile_over_time(0.99,
-          sum by (cluster, namespace) (
-            cluster_namespace_job:cortex_distributor_received_samples:rate5m
-          )[24h:]
-        )
-        / 240000
-      )
-    labels:
-      deployment: distributor
-      reason: sample_rate
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
-        * 0.59999999999999998 / 240000
-      )
-    labels:
-      deployment: distributor
-      reason: sample_rate_limits
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        quantile_over_time(0.99,
-          sum by (cluster, namespace) (
-            cluster_namespace_job:cortex_distributor_received_samples:rate5m
-          )[24h:]
-        )
-        * 3 / 80000
-      )
-    labels:
-      deployment: ingester
-      reason: sample_rate
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        quantile_over_time(0.99,
-          sum by(cluster, namespace) (
-            cortex_ingester_memory_series
-          )[24h:]
-        )
-        / 1500000
-      )
-    labels:
-      deployment: ingester
-      reason: active_series
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
-        * 3 * 0.59999999999999998 / 1500000
-      )
-    labels:
-      deployment: ingester
-      reason: active_series_limits
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
-        * 0.59999999999999998 / 80000
-      )
-    labels:
-      deployment: ingester
-      reason: sample_rate_limits
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      ceil(
-        (sum by (cluster, namespace) (
-          cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
-        ) / 4)
-          /
-        avg by (cluster, namespace) (
-          memcached_limit_bytes{job=~".+/memcached"}
-        )
-      )
-    labels:
-      deployment: memcached
-      reason: active_series
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      sum by (cluster, namespace, deployment) (
-        label_replace(
-          label_replace(
-            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
-            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-          ),
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-          # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-        )
-      )
-    record: cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate
-  - expr: |
-      # Convenience rule to get the CPU request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
-      # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
-      # that remove resource metrics, ref:
-      # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
-      # - https://github.com/kubernetes/kube-state-metrics/pull/1004
-      #
-      # This is the old expression, compatible with kube-state-metrics < v2.0.0,
-      # where kube_pod_container_resource_requests_cpu_cores was removed:
-      (
-        sum by (cluster, namespace, deployment) (
-          label_replace(
-            label_replace(
-              kube_pod_container_resource_requests_cpu_cores,
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-            ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-            # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-          )
-        )
-      )
-      or
-      # This expression is compatible with kube-state-metrics >= v1.4.0,
-      # where kube_pod_container_resource_requests was introduced.
-      (
-        sum by (cluster, namespace, deployment) (
-          label_replace(
-            label_replace(
-              kube_pod_container_resource_requests{resource="cpu"},
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-            ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-            # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-          )
-        )
-      )
-    record: cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
-  - expr: |
-      # Jobs should be sized to their CPU usage.
-      # We do this by comparing 99th percentile usage over the last 24hrs to
-      # their current provisioned #replicas and resource requests.
-      ceil(
-        cluster_namespace_deployment:actual_replicas:count
-          *
-        quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
-          /
-        cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
-      )
-    labels:
-      reason: cpu_usage
-    record: cluster_namespace_deployment_reason:required_replicas:count
-  - expr: |
-      # Convenience rule to get the Memory utilization for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
-      sum by (cluster, namespace, deployment) (
-        label_replace(
-          label_replace(
-            container_memory_usage_bytes{image!=""},
-            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-          ),
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-          # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-        )
-      )
-    record: cluster_namespace_deployment:container_memory_usage_bytes:sum
-  - expr: |
-      # Convenience rule to get the Memory request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
-      # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
-      # that remove resource metrics, ref:
-      # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
-      # - https://github.com/kubernetes/kube-state-metrics/pull/1004
-      #
-      # This is the old expression, compatible with kube-state-metrics < v2.0.0,
-      # where kube_pod_container_resource_requests_memory_bytes was removed:
-      (
-        sum by (cluster, namespace, deployment) (
-          label_replace(
-            label_replace(
-              kube_pod_container_resource_requests_memory_bytes,
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-            ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-            # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-          )
-        )
-      )
-      or
-      # This expression is compatible with kube-state-metrics >= v1.4.0,
-      # where kube_pod_container_resource_requests was introduced.
-      (
-        sum by (cluster, namespace, deployment) (
-          label_replace(
-            label_replace(
-              kube_pod_container_resource_requests{resource="memory"},
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
-            ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
-            # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
-          )
-        )
-      )
-    record: cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
-  - expr: |
-      # Jobs should be sized to their Memory usage.
-      # We do this by comparing 99th percentile usage over the last 24hrs to
-      # their current provisioned #replicas and resource requests.
-      ceil(
-        cluster_namespace_deployment:actual_replicas:count
-          *
-        quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
-          /
-        cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
-      )
-    labels:
-      reason: memory_usage
-    record: cluster_namespace_deployment_reason:required_replicas:count
-- name: mimir_alertmanager_rules
-  rules:
-  - expr: |
-      sum by (cluster, job, pod) (cortex_alertmanager_alerts)
-    record: cluster_job_pod:cortex_alertmanager_alerts:sum
-  - expr: |
-      sum by (cluster, job, pod) (cortex_alertmanager_silences)
-    record: cluster_job_pod:cortex_alertmanager_silences:sum
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))
-    record: cluster_job:cortex_alertmanager_alerts_received_total:rate5m
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))
-    record: cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m
-  - expr: |
-      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))
-    record: cluster_job_integration:cortex_alertmanager_notifications_total:rate5m
-  - expr: |
-      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))
-    record: cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))
-    record: cluster_job:cortex_alertmanager_state_replication_total:rate5m
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))
-    record: cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))
-    record: cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m
-  - expr: |
-      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))
-    record: cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m
-- name: mimir_ingester_rules
-  rules:
-  - expr: |
-      sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))
-    record: cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m

charts/meta-monitoring/src/rules/tempo-rules.yaml

@@ -1,15 +0,0 @@
-groups:
-- name: tempo_rules
-  rules:
-  - expr: histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:tempo_request_duration_seconds:99quantile
-  - expr: histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))
-    record: cluster_namespace_job_route:tempo_request_duration_seconds:50quantile
-  - expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
-    record: cluster_namespace_job_route:tempo_request_duration_seconds:avg
-  - expr: sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route)
-    record: cluster_namespace_job_route:tempo_request_duration_seconds_bucket:sum_rate
-  - expr: sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route)
-    record: cluster_namespace_job_route:tempo_request_duration_seconds_sum:sum_rate
-  - expr: sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)
-    record: cluster_namespace_job_route:tempo_request_duration_seconds_count:sum_rate

charts/meta-monitoring/templates/_helpers.tpl

@@ -0,0 +1,33 @@
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "ingress.apiVersion" -}}
+  {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}}
+      {{- print "networking.k8s.io/v1" -}}
+  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
+    {{- print "networking.k8s.io/v1beta1" -}}
+  {{- else -}}
+    {{- print "extensions/v1beta1" -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+Return if ingress is stable.
+*/}}
+{{- define "ingress.isStable" -}}
+  {{- eq (include "ingress.apiVersion" .) "networking.k8s.io/v1" -}}
+{{- end -}}
+
+{{/*
+Return if ingress supports ingressClassName.
+*/}}
+{{- define "ingress.supportsIngressClassName" -}}
+  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
+{{- end -}}
+
+{{/*
+Return if ingress supports pathType.
+*/}}
+{{- define "ingress.supportsPathType" -}}
+  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
+{{- end -}}

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -6,6 +6,24 @@
 {{- join ", " $list }}
 {{- end }}
 
+{{- define "agent.all_namespaces" -}}
+{{- $list := list }}
+{{- range .Values.namespacesToMonitor }}
+{{- $list = append $list (printf "\"%s\"" .) }}
+{{- end }}
+{{- $list = append $list (printf "\"%s\"" .Release.Namespace) }}
+{{- join ", " $list }}
+{{- end }}
+
+{{- define "agent.all_namespaces_bar" -}}
+{{- $list := list }}
+{{- range .Values.namespacesToMonitor }}
+{{- $list = append $list (printf "%s" .) }}
+{{- end }}
+{{- $list = append $list .Release.Namespace }}
+{{- join "|" $list }}
+{{- end }}
+
 {{- define "agent.loki_write_targets" -}}
 {{- $list := list }}
 {{- if .Values.local.logs.enabled }}
@@ -18,10 +36,10 @@
 {{- end }}
 
 {{- define "agent.loki_process_targets" -}}
-{{- if empty .Values.logs.piiRegexes }}
+{{- if and (empty .Values.logs.piiRegexes) (empty .Values.logs.retain) }}
 {{- include "agent.loki_write_targets" . }}
 {{- else }}
-{{- printf "loki.process.PII.receiver" }}
+{{- printf "loki.process.filter.receiver" }}
 {{- end }}
 {{- end }}
 
@@ -39,10 +57,32 @@
 {{- define "agent.tempo_write_targets" -}}
 {{- $list := list }}
 {{- if .Values.local.traces.enabled }}
-{{- $list = append $list ("otelcol.exporter.otlp.local.input") }}
+{{- $list = append $list ("otelcol.exporter.otlphttp.local.input") }}
 {{- end }}
 {{- if .Values.cloud.traces.enabled }}
-{{- $list = append $list ("otelcol.exporter.otlp.cloud.input") }}
+{{- $list = append $list ("otelcol.exporter.otlphttp.cloud.input") }}
 {{- end }}
 {{- join ", " $list }}
 {{- end }}
+
+{{- define "agent.all_logs" -}}
+{{- $list := list }}
+{{- range .Values.logs.retain }}
+{{- $list = append $list . }}
+{{- end }}
+{{- range .Values.logs.extraLogs }}
+{{- $list = append $list . }}
+{{- end }}
+{{- join "|" $list }}
+{{- end }}
+
+{{- define "agent.all_metrics" -}}
+{{- $list := list }}
+{{- range .Values.metrics.retain }}
+{{- $list = append $list . }}
+{{- end }}
+{{- range .Values.metrics.extraMetrics }}
+{{- $list = append $list . }}
+{{- end }}
+{{- join "|" $list }}
+{{- end }}

charts/meta-monitoring/templates/agent/config.yaml

@@ -8,7 +8,7 @@ data:
     discovery.kubernetes "pods" {
       role = "pod"
       namespaces {
-        own_namespace = false
+        own_namespace = true
         names = [ {{ include "agent.namespaces" . }} ]
       }
     }
@@ -33,22 +33,40 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
+        replacement = "{{- .Values.clusterLabelValue -}}"
       }
     }
 
+    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     // Logs
 
-    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
+    {{- if .Values.cloud.logs.enabled }}
+    remote.kubernetes.secret "logs_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.logs.secret -}}"
+    }
+    {{- end }}
+
     loki.source.kubernetes "pods" {
+      clustering {
+        enabled = true
+      }
       targets    = discovery.relabel.rename_meta_labels.output
       forward_to = [ {{ include "agent.loki_process_targets" . }} ]
     }
 
-    {{- if not (empty .Values.logs.piiRegexes) }}
-    loki.process "PII" {
+    {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.piiRegexes)) }}
+    loki.process "filter" {
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
+      {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.extraLogs)) }}
+      stage.match {
+        selector = "{cluster=\"{{- .Values.clusterLabelValue -}}\", namespace=~\"{{- join "|" .Values.namespacesToMonitor -}}|{{- $.Release.Namespace -}}\", pod=~\"loki.*\"} !~ \"{{ include "agent.all_logs" . }}\""
+        action   = "drop"
+      }
+      {{- end }}
+
+      {{- if not (empty .Values.logs.piiRegexes) }}
       {{- range .Values.logs.piiRegexes }}
       stage.replace {
         expression = "{{ .expression }}"
@@ -56,149 +74,121 @@ data:
         replace = "{{ .replace }}"
       }
       {{- end }}
+      {{- end }}
     }
     {{- end }}
     {{- end }}
 
-    // Metrics
-
     {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
+    // Metrics
+
+    {{- if .Values.cloud.metrics.enabled }}
+    remote.kubernetes.secret "metrics_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.metrics.secret -}}"
+    }
+    {{- end }}
+
+    discovery.kubernetes "metric_pods" {
+      role = "pod"
+      namespaces {
+        own_namespace = true
+        names = [ {{ include "agent.all_namespaces" . }} ]
+      }
+    }
+
+    discovery.relabel "only_http_metrics" {
+      targets = discovery.kubernetes.metric_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        regex = "(.*)/(.*)/(.*)"
+        replacement = "${1}/${2}-${3}"
+        target_label = "job"
+      }
+      rule {
+        target_label = "cluster"
+        replacement = "{{- .Values.clusterLabelValue -}}"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_port_name"]
+        action = "keep"
+        regex = ".*metrics.*"
+      }
+    }
+
     prometheus.scrape "pods" {
-      targets    = discovery.relabel.rename_meta_labels.output
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
-    }
-    {{- if .Values.kubeStateMetrics.enabled }}
-
-    prometheus.scrape "kubeStateMetrics" {
-      targets    = [ { "__address__" = "{{ .Values.kubeStateMetrics.endpoint }}" } ]
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
-    }
-    {{- end }}
-
-    // cAdvisor and Kubelete metrics
-    // Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
-    discovery.kubernetes "all_nodes" {
-      role = "node"
+      clustering {
+        enabled = true
+      }
+      targets    = discovery.relabel.only_http_metrics.output
+      forward_to = [ prometheus.relabel.filter.receiver ]
     }
 
-    discovery.relabel "all_nodes" {
-      targets = discovery.kubernetes.all_nodes.targets
-      rule {
-        source_labels = ["__meta_kubernetes_node_name"]
-        target_label = "node"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_namespace"]
-        target_label  = "namespace"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_pod_name"]
-        target_label  = "pod"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
-        separator = "/"
-        regex = "(.*)/(.*)/(.*)"
-        replacement = "${1}/${2}-${3}"
-        target_label = "job"
-      }
+    prometheus.relabel "filter" {
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
+        replacement = "{{- .Values.clusterLabelValue -}}"
+      }
+
+      rule {
+        source_labels = ["__name__"]
+        regex = "({{ include "agent.all_metrics" . }})"
+        action = "keep"
+      }
+
+      rule {
+        source_labels = ["namespace"]
+        regex = "{{ include "agent.all_namespaces_bar" . }}"
+
+        action = "keep"
       }
-    }
 
-    prometheus.scrape "cadvisor" {
-      targets    = discovery.relabel.all_nodes.output
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
-
-      scrape_interval = "15s"
-      metrics_path = "/metrics/cadvisor"
-      scheme = "https"
-
-      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-      tls_config {
-        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-      }
     }
 
-    prometheus.scrape "kubelet" {
-      targets    = discovery.relabel.all_nodes.output
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
-
-      scrape_interval = "15s"
-      metrics_path = "/metrics"
-      scheme = "https"
-
-      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-      tls_config {
-        ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-      }
-    }
-
-    prometheus.exporter.unix {}
-
-    prometheus.scrape "node_exporter" {
-      targets = prometheus.exporter.unix.targets
-      forward_to = [prometheus.relabel.node_exporter.receiver]
-
-      job_name = "node-exporter"
-      scrape_interval = "15s"
-    }
-
-    prometheus.relabel "node_exporter" {
-      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
-
-      rule {
-        replacement = env("HOSTNAME")
-        target_label = "nodename"
-      }
-      rule {
-        replacement = "node-exporter"
-        target_label = "job"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_node_name"]
-        target_label = "node"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_namespace"]
-        target_label  = "namespace"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_pod_name"]
-        target_label  = "pod"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
-        separator = "/"
-        regex = "(.*)/(.*)/(.*)"
-        replacement = "${1}/${2}-${3}"
-        target_label = "job"
-      }
-      rule {
-        target_label = "cluster"
-        replacement = "{{- .Values.clusterName -}}"
-      }
-    }
     {{- end }}
 
-    // Traces
-
     {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
+    // Traces
+
+    {{- if .Values.cloud.traces.enabled }}
+    remote.kubernetes.secret "traces_credentials" {
+      namespace = "{{- $.Release.Namespace -}}"
+      name = "{{- .Values.cloud.traces.secret -}}"
+    }
+    {{- end }}
+
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {
       // We don't technically need this, but it shows how to change listen address and incoming port.
       // In this case, the Agent is listening on all available bindable addresses on port 4317 (which is the
       // default OTLP gRPC port) for the OTLP protocol.
-      grpc {
-          endpoint = "0.0.0.0:4317"
-      }
+      grpc {}
 
       // We define where to send the output of all ingested traces. In this case, to the OpenTelemetry batch processor
       // named 'default'.
       output {
-          traces = [otelcol.processor.batch.default.input]
+        traces = [otelcol.processor.batch.default.input]
+      }
+    }
+
+    otelcol.receiver.jaeger "jaeger" {
+      protocols {
+        thrift_http {}
+      }
+
+      output {
+        traces = [otelcol.processor.batch.default.input]
       }
     }
 
@@ -219,7 +209,7 @@ data:
     {{- if .Values.local.logs.enabled }}
     loki.write "local" {
       endpoint {
-        url = "http://loki-gateway.{{- .Release.Namespace -}}.svc.cluster.local:80/loki/api/v1/push"
+        url = "http://loki-write.{{- .Release.Namespace -}}.svc.cluster.local:3100/loki/api/v1/push"
       }
     }
     {{- end }}
@@ -232,21 +222,10 @@ data:
     }
     {{- end }}
 
-    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
-    // The OpenTelemetry exporter exports processed trace spans to another target that is listening for OTLP format traces.
-    // A unique label, 'local', is added to uniquely identify this exporter.
-    otelcol.exporter.otlp "local" {
-        // Define the client for exporting.
+    {{- if .Values.local.traces.enabled }}
+    otelcol.exporter.otlphttp "local" {
         client {
-            // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC).
-            endpoint = "meta-tempo-distributor:4317"
-            // Configure TLS settings for communicating with the endpoint.
-            tls {
-                // The connection is insecure.
-                insecure = true
-                // Do not verify TLS certificates when connecting.
-                insecure_skip_verify = true
-            }
+            endpoint = "http://{{- .Release.Name -}}-tempo-distributor.{{- .Release.Namespace -}}.svc:4318"
         }
     }
     {{- end }}
@@ -254,11 +233,10 @@ data:
     {{- if .Values.cloud.logs.enabled }}
     loki.write "cloud" {
       endpoint {
-        url = "{{- .Values.cloud.logs.endpoint -}}/loki/api/v1/push"
-
+        url = nonsensitive(remote.kubernetes.secret.logs_credentials.data["endpoint"])
         basic_auth {
-          username = "{{- .Values.cloud.logs.username -}}"
-          password = "{{- .Values.cloud.logs.password -}}"
+          username = nonsensitive(remote.kubernetes.secret.logs_credentials.data["username"])
+          password = remote.kubernetes.secret.logs_credentials.data["password"]
         }
       }
     }
@@ -267,26 +245,25 @@ data:
     {{- if .Values.cloud.metrics.enabled }}
     prometheus.remote_write "cloud" {
       endpoint {
-        url = "{{- .Values.cloud.metrics.endpoint -}}/api/prom/push"
-
+        url = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["endpoint"])
         basic_auth {
-          username = "{{- .Values.cloud.metrics.username -}}"
-          password = "{{- .Values.cloud.metrics.password -}}"
+          username = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["username"])
+          password = remote.kubernetes.secret.metrics_credentials.data["password"]
         }
       }
     }
     {{- end }}
 
     {{- if .Values.cloud.traces.enabled }}
-    otelcol.exporter.otlp "cloud" {
+    otelcol.exporter.otlphttp "cloud" {
         client {
-            endpoint = "{{- .Values.cloud.traces.endpoint -}}"
+            endpoint = nonsensitive(remote.kubernetes.secret.traces_credentials.data["endpoint"])
             auth = otelcol.auth.basic.creds.handler
         }
     }
 
     otelcol.auth.basic "creds" {
-      username = "{{- .Values.cloud.traces.username -}}"
-      password = "{{- .Values.cloud.traces.password -}}"
+      username = nonsensitive(remote.kubernetes.secret.traces_credentials.data["username"])
+      password = remote.kubernetes.secret.traces_credentials.data["password"]
     }
     {{- end }}

charts/meta-monitoring/templates/grafana/alloy-dashboards-1.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-dashboards-1
+  namespace: {{ $.Release.Namespace }}
+data:
+  "alloy-cluster-node.json": |
+    {{ $.Files.Get "src/dashboards/alloy-cluster-node.json" | fromJson | toJson }}
+  "alloy-cluster-overview.json": |
+    {{ $.Files.Get "src/dashboards/alloy-cluster-overview.json" | fromJson | toJson }}
+  "alloy-controller.json": |
+    {{ $.Files.Get "src/dashboards/alloy-controller.json" | fromJson | toJson }}
+  "alloy-opentelemetry.json": |
+    {{ $.Files.Get "src/dashboards/alloy-opentelemetry.json" | fromJson | toJson }}
+  "alloy-prometheus.json": |
+    {{ $.Files.Get "src/dashboards/alloy-prometheus.json" | fromJson | toJson }}
+  "alloy-resources.json": |
+    {{ $.Files.Get "src/dashboards/alloy-resources.json" | fromJson | toJson }}
+{{- end }}

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,4 +1,4 @@
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -28,56 +28,12 @@ data:
         orgId: 1
         type: file
 {{- end }}
-{{- if .Values.dashboards.metrics.enabled }}
       - disableDeletion: true
         editable: false
-        folder: Mimir
-        name: mimir-1
+        folder: Alloy
+        name: alloy-1
         options:
-          path: /var/lib/grafana/dashboards/mimir-1
-        orgId: 1
-        type: file
-      - disableDeletion: true
-        editable: false
-        folder: Mimir
-        name: mimir-2
-        options:
-          path: /var/lib/grafana/dashboards/mimir-2
-        orgId: 1
-        type: file
-      - disableDeletion: true
-        editable: false
-        folder: Mimir
-        name: mimir-3
-        options:
-          path: /var/lib/grafana/dashboards/mimir-3
-        orgId: 1
-        type: file
-      - disableDeletion: true
-        editable: false
-        folder: Mimir
-        name: mimir-4
-        options:
-          path: /var/lib/grafana/dashboards/mimir-4
-        orgId: 1
-        type: file
-      - disableDeletion: true
-        editable: false
-        folder: Mimir
-        name: mimir-5
-        options:
-          path: /var/lib/grafana/dashboards/mimir-5
+          path: /var/lib/grafana/dashboards/alloy-1
         orgId: 1
         type: file
 {{- end }}
-{{- if .Values.dashboards.traces.enabled }}
-      - disableDeletion: true
-        editable: false
-        folder: Tempo
-        name: tempo-1
-        options:
-          path: /var/lib/grafana/dashboards/tempo-1
-        orgId: 1
-        type: file
-{{- end }}
-{{- end }}

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -1,4 +1,4 @@
-{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
+{{- if .Values.local.grafana.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -32,7 +32,7 @@ data:
         uid: loki_ds
         # <string> Sets the data source's URL, including the
         # port.
-        url: http://loki-gateway.{{- $.Release.Namespace -}}.svc.cluster.local
+        url: http://{{- $.Release.Namespace -}}-loki-gateway.{{- $.Release.Namespace -}}.svc.cluster.local
         # <bool> Toggles whether the data source is pre-selected
         # for new panels. You can set only one default
         # data source per organization.
@@ -61,6 +61,10 @@ data:
         # <bool> Allows users to edit data sources from the
         # Grafana UI.
         editable: true
+        # Extra config.
+        jsonData:
+          # Scrape interval
+          timeInterval: 1m
       {{- end }}
       {{- if .Values.local.traces.enabled }}
       - name: Tempo

charts/meta-monitoring/templates/grafana/grafana-deployment.yaml

@@ -1,16 +1,4 @@
-{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: grafana-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
----
+{{- if .Values.local.grafana.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -32,7 +20,7 @@ spec:
           - 0
       containers:
         - name: grafana
-          image: grafana/grafana:10.0.0
+          image: grafana/grafana:{{- .Values.grafana.version }}
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 3000
@@ -65,7 +53,7 @@ spec:
               name: grafana-pv
             - mountPath: /etc/grafana/provisioning/datasources
               name: datasources-provisioning
-            {{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+            {{- if .Values.dashboards.logs.enabled }}
             - mountPath: /etc/grafana/provisioning/dashboards
               name: dashboards-provisioning
             {{- end }}
@@ -75,22 +63,8 @@ spec:
             - mountPath: /var/lib/grafana/dashboards/loki-2
               name: loki-dashboards-2
             {{- end }}
-            {{- if .Values.dashboards.metrics.enabled }}
-            - mountPath: /var/lib/grafana/dashboards/mimir-1
-              name: mimir-dashboards-1
-            - mountPath: /var/lib/grafana/dashboards/mimir-2
-              name: mimir-dashboards-2
-            - mountPath: /var/lib/grafana/dashboards/mimir-3
-              name: mimir-dashboards-3
-            - mountPath: /var/lib/grafana/dashboards/mimir-4
-              name: mimir-dashboards-4
-            - mountPath: /var/lib/grafana/dashboards/mimir-5
-              name: mimir-dashboards-5
-            {{- end }}
-            {{- if .Values.dashboards.traces.enabled }}
-            - mountPath: /var/lib/grafana/dashboards/tempo-1
-              name: tempo-dashboards-1
-            {{- end }}
+            - mountPath: /var/lib/grafana/dashboards/alloy-1
+              name: alloy-dashboards-1
       volumes:
         - name: grafana-pv
           persistentVolumeClaim:
@@ -109,41 +83,7 @@ spec:
           configMap:
             name: loki-dashboards-2
         {{- end }}
-        {{- if .Values.dashboards.metrics.enabled }}
-        - name: mimir-dashboards-1
+        - name: alloy-dashboards-1
           configMap:
-            name: mimir-dashboards-1
-        - name: mimir-dashboards-2
-          configMap:
-            name: mimir-dashboards-2
-        - name: mimir-dashboards-3
-          configMap:
-            name: mimir-dashboards-3
-        - name: mimir-dashboards-4
-          configMap:
-            name: mimir-dashboards-4
-        - name: mimir-dashboards-5
-          configMap:
-            name: mimir-dashboards-5
-        {{- end }}
-        {{- if .Values.dashboards.traces.enabled }}
-        - name: tempo-dashboards-1
-          configMap:
-            name: tempo-dashboards-1
-        {{- end }}
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: grafana
-spec:
-  ports:
-    - port: 3000
-      protocol: TCP
-      targetPort: http-grafana
-  selector:
-    app: grafana
-  sessionAffinity: None
-  type: ClusterIP  # Make this configurable
+            name: alloy-dashboards-1
 {{- end }}

charts/meta-monitoring/templates/grafana/grafana-ingress.yaml

@@ -0,0 +1,57 @@
+{{- if and .Values.local.grafana.enabled .Values.grafana.ingress.enabled -}}
+{{- $ingressApiIsStable := eq (include "ingress.isStable" .) "true" -}}
+{{- $ingressSupportsIngressClassName := eq (include "ingress.supportsIngressClassName" .) "true" -}}
+{{- $ingressSupportsPathType := eq (include "ingress.supportsPathType" .) "true" -}}
+apiVersion: {{ include "ingress.apiVersion" . }}
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: grafana
+    {{- range $labelKey, $labelValue := .Values.grafana.ingress.labels }}
+    {{ $labelKey }}: {{ $labelValue | toYaml }}
+    {{- end }}
+  {{- with .Values.grafana.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and $ingressSupportsIngressClassName .Values.grafana.ingress.ingressClassName }}
+  ingressClassName: {{ .Values.grafana.ingress.ingressClassName }}
+  {{- end -}}
+  {{- if .Values.grafana.ingress.tls }}
+  tls:
+    {{- range .Values.grafana.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ tpl . $ | quote }}
+        {{- end }}
+      {{- with .secretName }}
+      secretName: {{ . }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.grafana.ingress.hosts }}
+    - host: {{ tpl .host $ | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if $ingressSupportsPathType }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if $ingressApiIsStable }}
+              service:
+                name: grafana
+                port:
+                  number: 3000
+              {{- else }}
+              serviceName: grafana
+              servicePort: 3000
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}

charts/meta-monitoring/templates/grafana/grafana-pvc.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.local.grafana.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+{{- end }}

charts/meta-monitoring/templates/grafana/grafana-service.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.local.grafana.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+spec:
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+  selector:
+    app: grafana
+  sessionAffinity: None
+  type: ClusterIP  # Make this configurable
+{{- end }}

charts/meta-monitoring/templates/grafana/loki-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -12,8 +12,6 @@ data:
     {{ $.Files.Get "src/dashboards/loki-deletion.json" | fromJson | toJson }}
   "loki-logs.json": |
     {{ $.Files.Get "src/dashboards/loki-logs.json" | fromJson | toJson }}
-  "loki-mixin-recording-rules.json": |
-    {{ $.Files.Get "src/dashboards/loki-mixin-recording-rules.json" | fromJson | toJson }}
   "loki-operational.json": |
     {{ $.Files.Get "src/dashboards/loki-operational.json" | fromJson | toJson }}
 {{- end }}

charts/meta-monitoring/templates/grafana/loki-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.dashboards.logs.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dahboards-5.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.dashboards.metrics.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mimir-dashboards-5
-  namespace: {{ $.Release.Namespace }}
-data:
-  "mimir-tenants.json": |
-    {{ $.Files.Get "src/dashboards/mimir-tenants.json" | fromJson | toJson }}
-  "mimir-top-tenants.json": |
-    {{ $.Files.Get "src/dashboards/mimir-top-tenants.json" | fromJson | toJson }}
-  "mimir-writes-networking.json": |
-    {{ $.Files.Get "src/dashboards/mimir-writes-networking.json" | fromJson | toJson }}
-  "mimir-writes-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-writes-resources.json" | fromJson | toJson }}
-  "mimir-writes.json": |
-    {{ $.Files.Get "src/dashboards/mimir-writes.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.dashboards.metrics.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mimir-dashboards-1
-  namespace: {{ $.Release.Namespace }}
-data:
-  "mimir-alertmanager-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-alertmanager-resources.json" | fromJson | toJson }}
-  "mimir-alertmanager.json": |
-    {{ $.Files.Get "src/dashboards/mimir-alertmanager.json" | fromJson | toJson }}
-  "mimir-compactor-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-compactor-resources.json" | fromJson | toJson }}
-  "mimir-compactor.json": |
-    {{ $.Files.Get "src/dashboards/mimir-compactor.json" | fromJson | toJson }}
-  "mimir-config.json": |
-    {{ $.Files.Get "src/dashboards/mimir-config.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.dashboards.metrics.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mimir-dashboards-2
-  namespace: {{ $.Release.Namespace }}
-data:
-  "mimir-object-store.json": |
-    {{ $.Files.Get "src/dashboards/mimir-object-store.json" | fromJson | toJson }}
-  "mimir-overrides.json": |
-    {{ $.Files.Get "src/dashboards/mimir-overrides.json" | fromJson | toJson }}
-  "mimir-overview-networking.json": |
-    {{ $.Files.Get "src/dashboards/mimir-overview-networking.json" | fromJson | toJson }}
-  "mimir-overview-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-overview-resources.json" | fromJson | toJson }}
-  "mimir-overview.json": |
-    {{ $.Files.Get "src/dashboards/mimir-overview.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.dashboards.metrics.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mimir-dashboards-3
-  namespace: {{ $.Release.Namespace }}
-data:
-  "mimir-queries.json": |
-    {{ $.Files.Get "src/dashboards/mimir-queries.json" | fromJson | toJson }}
-  "mimir-reads-networking.json": |
-    {{ $.Files.Get "src/dashboards/mimir-reads-networking.json" | fromJson | toJson }}
-  "mimir-reads-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-reads-resources.json" | fromJson | toJson }}
-  "mimir-reads.json": |
-    {{ $.Files.Get "src/dashboards/mimir-reads.json" | fromJson | toJson }}
-  "mimir-remote-ruler-reads-resources.json": |
-    {{ $.Files.Get "src/dashboards/mimir-remote-ruler-reads-resources.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.dashboards.metrics.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mimir-dashboards-4
-  namespace: {{ $.Release.Namespace }}
-data:
-  "mimir-remote-ruler-reads.json": |
-    {{ $.Files.Get "src/dashboards/mimir-remote-ruler-reads.json" | fromJson | toJson }}
-  "mimir-rollout-progress.json": |
-    {{ $.Files.Get "src/dashboards/mimir-rollout-progress.json" | fromJson | toJson }}
-  "mimir-ruler.json": |
-    {{ $.Files.Get "src/dashboards/mimir-ruler.json" | fromJson | toJson }}
-  "mimir-scaling.json": |
-    {{ $.Files.Get "src/dashboards/mimir-scaling.json" | fromJson | toJson }}
-  "mimir-slow-queries.json": |
-    {{ $.Files.Get "src/dashboards/mimir-slow-queries.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml

@@ -1,21 +0,0 @@
-{{- if .Values.dashboards.traces.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: tempo-dashboards-1
-  namespace: {{ $.Release.Namespace }}
-data:
-  "tempo-operational.json": |
-    {{ $.Files.Get "src/dashboards/tempo-operational.json" | fromJson | toJson }}
-  "tempo-reads.json": |
-    {{ $.Files.Get "src/dashboards/tempo-reads.json" | fromJson | toJson }}
-  "tempo-resources.json": |
-    {{ $.Files.Get "src/dashboards/tempo-resources.json" | fromJson | toJson }}
-  "tempo-rollout-progress.json": |
-    {{ $.Files.Get "src/dashboards/tempo-rollout-progress.json" | fromJson | toJson }}
-  "tempo-tenants.json": |
-    {{ $.Files.Get "src/dashboards/tempo-tenants.json" | fromJson | toJson }}
-  "tempo-writes.json": |
-    {{ $.Files.Get "src/dashboards/tempo-writes.json" | fromJson | toJson }}
-{{- end }}

charts/meta-monitoring/templates/ruler/ruler.yaml

@@ -1,9 +1,10 @@
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if .Values.local.grafana.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: meta-mimir-ruler-for-dashboards
-  namespace: meta
+  name: {{ $.Release.Namespace }}-mimir-ruler-for-dashboards
+  namespace: {{ $.Release.Namespace }}
 spec:
   progressDeadlineSeconds: 600
   replicas: 1
@@ -24,7 +25,7 @@ spec:
         app.kubernetes.io/component: ruler-for-dashboards
         app.kubernetes.io/instance: meta
         app.kubernetes.io/name: mimir
-      namespace: meta
+      namespace: {{ $.Release.Namespace }}
     spec:
       containers:
         - args:
@@ -48,6 +49,9 @@ spec:
             - containerPort: 7946
               name: memberlist
               protocol: TCP
+          envFrom:
+          - secretRef:
+             name: minio
           readinessProbe:
             failureThreshold: 3
             httpGet:
@@ -91,8 +95,6 @@ spec:
         runAsUser: 10001
         seccompProfile:
           type: RuntimeDefault
-      serviceAccount: meta-mimir
-      serviceAccountName: meta-mimir
       terminationGracePeriodSeconds: 180
       topologySpreadConstraints:
         - labelSelector:
@@ -109,11 +111,11 @@ spec:
             items:
               - key: mimir.yaml
                 path: mimir.yaml
-            name: meta-mimir-config
+            name: {{ $.Release.Namespace }}-mimir-config
           name: config
         - configMap:
             defaultMode: 420
-            name: meta-mimir-runtime
+            name: {{ $.Release.Namespace }}-mimir-runtime
           name: runtime-config
         - emptyDir: {}
           name: storage
@@ -124,3 +126,4 @@ spec:
             name: rules
           name: rules
 {{- end }}
+{{- end }}

charts/meta-monitoring/templates/ruler/rules-configmap.yaml

@@ -1,4 +1,5 @@
-{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
+{{- if .Values.local.metrics.enabled }}
+{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -9,10 +10,5 @@ data:
 {{- if .Values.dashboards.logs.enabled }}
 {{ ($.Files.Glob "src/rules/loki-rules.yaml").AsConfig | indent 2 }}
 {{- end }}
-{{- if .Values.dashboards.metrics.enabled }}
-{{ ($.Files.Glob "src/rules/mimir-rules.yaml").AsConfig | indent 2 }}
-{{- end }}
-{{- if .Values.dashboards.traces.enabled }}
-{{ ($.Files.Glob "src/rules/tempo-rules.yaml").AsConfig | indent 2 }}
 {{- end }}
 {{- end }}