Rename agent dashboard file

Signed-off-by: Michel Hollands <michel.hollands@gmail.com>

.github/configs/cr.yaml

@@ -1,3 +0,0 @@
-owner: grafana
-git-repo: helm-charts
-skip-existing: true

.github/configs/ct.yaml

@@ -1,15 +0,0 @@
-## Reference: https://github.com/helm/chart-testing/blob/master/doc/ct_lint-and-install.md
-remote: origin
-target-branch: main
-chart-dirs:
-- charts
-chart-repos:
-  - grafana=https://grafana.github.io/helm-charts
-  - minio=https://charts.min.io
-validate-chart-schema: true
-validate-maintainers: true
-validate-yaml: true
-exclude-deprecated: true
-excluded-charts: []
-namespace: meta-monitoring # Need to set the namespace because we create the secret there
-release-label: app.kubernetes.io/instance

.github/configs/updatecli.d/alloy.yaml

@@ -1,30 +0,0 @@
-name: Bump dependency "alloy" for Helm chart "meta-monitoring"
-sources:
-    alloy:
-        name: Get latest "alloy" Helm chart version
-        kind: helmchart
-        spec:
-            name: alloy
-            url: https://grafana.github.io/helm-charts
-            versionfilter:
-                kind: semver
-                pattern: '*'
-conditions:
-    alloy:
-        name: Ensure Helm chart dependency "alloy" is specified
-        kind: yaml
-        spec:
-            file: charts/meta-monitoring/Chart.yaml
-            key: $.dependencies[1].name
-            value: alloy
-        disablesourceinput: true
-targets:
-    alloy:
-        name: Bump Helm chart dependency "alloy" for Helm chart "meta-monitoring"
-        kind: helmchart
-        spec:
-            file: Chart.yaml
-            key: $.dependencies[1].version
-            name: charts/meta-monitoring
-            versionincrement: none
-        sourceid: alloy

.github/configs/updatecli.d/loki.yaml

@@ -1,30 +0,0 @@
-name: Bump dependency "loki" for Helm chart "meta-monitoring"
-sources:
-    loki:
-        name: Get latest "loki" Helm chart version
-        kind: helmchart
-        spec:
-            name: loki
-            url: https://grafana.github.io/helm-charts
-            versionfilter:
-                kind: semver
-                pattern: '*'
-conditions:
-    loki:
-        name: Ensure Helm chart dependency "loki" is specified
-        kind: yaml
-        spec:
-            file: charts/meta-monitoring/Chart.yaml
-            key: $.dependencies[0].name
-            value: loki
-        disablesourceinput: true
-targets:
-    loki:
-        name: Bump Helm chart dependency "loki" for Helm chart "meta-monitoring"
-        kind: helmchart
-        spec:
-            file: Chart.yaml
-            key: $.dependencies[0].version
-            name: charts/meta-monitoring
-            versionincrement: none
-        sourceid: loki

.github/configs/updatecli.d/mimir-distributed.yaml

@@ -1,30 +0,0 @@
-name: Bump dependency "mimir-distributed" for Helm chart "meta-monitoring"
-sources:
-    mimir-distributed:
-        name: Get latest "mimir-distributed" Helm chart version
-        kind: helmchart
-        spec:
-            name: mimir-distributed
-            url: https://grafana.github.io/helm-charts
-            versionfilter:
-                kind: semver
-                pattern: '*'
-conditions:
-    mimir-distributed:
-        name: Ensure Helm chart dependency "mimir-distributed" is specified
-        kind: yaml
-        spec:
-            file: charts/meta-monitoring/Chart.yaml
-            key: $.dependencies[2].name
-            value: mimir-distributed
-        disablesourceinput: true
-targets:
-    mimir-distributed:
-        name: Bump Helm chart dependency "mimir-distributed" for Helm chart "meta-monitoring"
-        kind: helmchart
-        spec:
-            file: Chart.yaml
-            key: $.dependencies[2].version
-            name: charts/meta-monitoring
-            versionincrement: none
-        sourceid: mimir-distributed

.github/configs/updatecli.d/minio.yaml

@@ -1,30 +0,0 @@
-name: Bump dependency "minio" for Helm chart "meta-monitoring"
-sources:
-    minio:
-        name: Get latest "minio" Helm chart version
-        kind: helmchart
-        spec:
-            name: minio
-            url: https://charts.min.io
-            versionfilter:
-                kind: semver
-                pattern: '*'
-conditions:
-    minio:
-        name: Ensure Helm chart dependency "minio" is specified
-        kind: yaml
-        spec:
-            file: charts/meta-monitoring/Chart.yaml
-            key: $.dependencies[4].name
-            value: minio
-        disablesourceinput: true
-targets:
-    minio:
-        name: Bump Helm chart dependency "minio" for Helm chart "meta-monitoring"
-        kind: helmchart
-        spec:
-            file: Chart.yaml
-            key: $.dependencies[4].version
-            name: charts/meta-monitoring
-            versionincrement: none
-        sourceid: minio

.github/configs/updatecli.d/tempo-distributed.yaml

@@ -1,30 +0,0 @@
-name: Bump dependency "tempo-distributed" for Helm chart "meta-monitoring"
-sources:
-    tempo-distributed:
-        name: Get latest "tempo-distributed" Helm chart version
-        kind: helmchart
-        spec:
-            name: tempo-distributed
-            url: https://grafana.github.io/helm-charts
-            versionfilter:
-                kind: semver
-                pattern: '*'
-conditions:
-    tempo-distributed:
-        name: Ensure Helm chart dependency "tempo-distributed" is specified
-        kind: yaml
-        spec:
-            file: charts/meta-monitoring/Chart.yaml
-            key: $.dependencies[3].name
-            value: tempo-distributed
-        disablesourceinput: true
-targets:
-    tempo-distributed:
-        name: Bump Helm chart dependency "tempo-distributed" for Helm chart "meta-monitoring"
-        kind: helmchart
-        spec:
-            file: Chart.yaml
-            key: $.dependencies[3].version
-            name: charts/meta-monitoring
-            versionincrement: none
-        sourceid: tempo-distributed

.github/workflows/check-for-dependency-updates.yaml

@@ -1,177 +0,0 @@
----
-name: Check for dependency updates
-
-on:
-    workflow_dispatch:
-    schedule:
-      # Run once a day
-      - cron: '0 7 * * *'
-
-permissions:
-    contents: "write"
-    pull-requests: "write"
-
-env:
-    UPDATECLI_CONFIG_DIR: "${{ github.workspace }}/.github/configs/updatecli.d"
-    UPDATECLI_GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-
-jobs:
-    updateLoki:
-        name: Update the Loki subchart
-        runs-on: "ubuntu-latest"
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v2
-
-            - name: Install Updatecli
-              uses: updatecli/updatecli-action@v2
-
-            - name: Run Updatecli
-              id: update-loki
-              run: |
-                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/loki.yaml
-                if ! git diff --exit-code > /dev/null; then
-                  echo "changed=true" >> "${GITHUB_OUTPUT}"
-                fi
-
-            - name: Create pull request
-              if: steps.update-loki.outputs.changed == 'true'
-              uses: peter-evans/create-pull-request@v5
-              with:
-                title: "[dependency] Update the Loki subchart"
-                body: "Updates the Loki subchart"
-                base: main
-                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-                committer: "GitHub <noreply@github.com>"
-                commit-message: Update loki
-                labels: dependencies
-                branch: chore/update-loki
-                delete-branch: true
-
-    updateGrafanaAlloy:
-        name: Update the Grafana Alloy subchart
-        runs-on: "ubuntu-latest"
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v2
-
-            - name: Install Updatecli
-              uses: updatecli/updatecli-action@v2
-
-            - name: Run Updatecli
-              id: update-grafana-alloy
-              run: |
-                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/alloy.yaml
-                if ! git diff --exit-code > /dev/null; then
-                  echo "changed=true" >> "${GITHUB_OUTPUT}"
-                fi
-
-            - name: Create pull request
-              if: steps.update-grafana-alloy.outputs.changed == 'true'
-              uses: peter-evans/create-pull-request@v5
-              with:
-                title: "[dependency] Update the Grafana Alloy subchart"
-                body: "Updates the Grafana Alloy subchart"
-                base: main
-                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-                committer: "GitHub <noreply@github.com>"
-                commit-message: Update Grafana Alloy
-                labels: dependencies
-                branch: chore/update-grafana-alloy
-                delete-branch: true
-
-    updateMimirDistributed:
-        name: Update the Mimir Distributed subchart
-        runs-on: "ubuntu-latest"
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v2
-
-            - name: Install Updatecli
-              uses: updatecli/updatecli-action@v2
-
-            - name: Run Updatecli
-              id: update-mimir-distributed
-              run: |
-                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/mimir-distributed.yaml
-                if ! git diff --exit-code > /dev/null; then
-                  echo "changed=true" >> "${GITHUB_OUTPUT}"
-                fi
-
-            - name: Create pull request
-              if: steps.update-mimir-distributed.outputs.changed == 'true'
-              uses: peter-evans/create-pull-request@v5
-              with:
-                title: "[dependency] Update the Mimir Distributed subchart"
-                body: "Updates the Mimir Distributed subchart"
-                base: main
-                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-                committer: "GitHub <noreply@github.com>"
-                commit-message: Update Mimir Distributed
-                labels: dependencies
-                branch: chore/update-mimir-distributed
-                delete-branch: true
-
-    updateTempoDistributed:
-        name: Update the Tempo Distributed subchart
-        runs-on: "ubuntu-latest"
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v2
-
-            - name: Install Updatecli
-              uses: updatecli/updatecli-action@v2
-
-            - name: Run Updatecli
-              id: update-tempo-distributed
-              run: |
-                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/tempo-distributed.yaml
-                if ! git diff --exit-code > /dev/null; then
-                  echo "changed=true" >> "${GITHUB_OUTPUT}"
-                fi
-
-            - name: Create pull request
-              if: steps.update-tempo-distributed.outputs.changed == 'true'
-              uses: peter-evans/create-pull-request@v5
-              with:
-                title: "[dependency] Update the Tempo Distributed subchart"
-                body: "Updates the tempo Distributed subchart"
-                base: main
-                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-                committer: "GitHub <noreply@github.com>"
-                commit-message: Update Tempo Distributed
-                labels: dependencies
-                branch: chore/update-tempo-distributed
-                delete-branch: true
-
-    updateMinio:
-        name: Update the Minio subchart
-        runs-on: "ubuntu-latest"
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v2
-
-            - name: Install Updatecli
-              uses: updatecli/updatecli-action@v2
-
-            - name: Run Updatecli
-              id: update-minio
-              run: |
-                updatecli apply --config ${UPDATECLI_CONFIG_DIR}/minio.yaml
-                if ! git diff --exit-code > /dev/null; then
-                  echo "changed=true" >> "${GITHUB_OUTPUT}"
-                fi
-
-            - name: Create pull request
-              if: steps.update-minio.outputs.changed == 'true'
-              uses: peter-evans/create-pull-request@v5
-              with:
-                title: "[dependency] Update the Minio subchart"
-                body: "Updates the Minio subchart"
-                base: main
-                author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-                committer: "GitHub <noreply@github.com>"
-                commit-message: Update minio
-                labels: dependencies
-                branch: chore/update-minio
-                delete-branch: true

.github/workflows/helm-ci.yml

@@ -16,51 +16,86 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v3
 
+      # - name: Check Docs
+      #   run: |
+      #     docker run --rm --volume "$(pwd):/helm-docs" -u "$(id -u)" jnorwood/helm-docs:v1.11.0
+      #     if ! git diff --exit-code; then
+      #         echo "Documentation not up to date. Please run helm-docs and commit changes!" >&2
+      #         exit 1
+      #     fi
+
       - name: Lint Yaml
         run: make helm-lint
 
-  # call-test:
+      # - name: Lint Code Base
-  #   name: Test Helm Chart
+      #   uses: docker://github/super-linter:v3.12.0
-  #   runs-on: ubuntu-latest
+      #   env:
-  #   steps:
+      #     FILTER_REGEX_EXCLUDE: .*(README\.md|Chart\.yaml|NOTES.txt).*
-  #     - name: Checkout
+      #     FILTER_REGEX_INCLUDE: .*charts/meta-monitoring/.*
-  #       uses: actions/checkout@v3
+      #     VALIDATE_ALL_CODEBASE: false
-  #       with:
+      #     VALIDATE_KUBERNETES_KUBEVAL: false
-  #         fetch-depth: 0
+      #     VALIDATE_YAML: false
+      #     VALIDATE_GO: false
+      #     DEFAULT_BRANCH: main
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  call-test:
+    name: Test Helm Chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-  #     - name: Set up Helm
+      - name: Set up Helm
-  #       uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v3
-  #       with:
+        with:
-  #         version: v3.8.2
+          version: v3.8.2
 
-  #     # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
-  #     # yamllint (https://github.com/adrienverge/yamllint) which require Python
+      # yamllint (https://github.com/adrienverge/yamllint) which require Python
-  #     - name: Set up Python
+      - name: Set up Python
-  #       uses: actions/setup-python@v4
+        uses: actions/setup-python@v4
-  #       with:
+        with:
-  #         python-version: 3.7
+          python-version: 3.7
 
-  #     - name: Set up chart-testing
+      - name: Set up chart-testing
-  #       uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.4.0
 
-  #     - name: Run chart-testing (list-changed)
+      - name: Run chart-testing (list-changed)
-  #       id: list-changed
+        id: list-changed
-  #       run: |
+        run: |
-  #         changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+          changed=$(ct list-changed --config "${CT_CONFIGFILE}")
-  #         if [[ -n "$changed" ]]; then
+          if [[ -n "$changed" ]]; then
-  #           echo "changed=true" >> $GITHUB_OUTPUT
+            echo "changed=true" >> $GITHUB_OUTPUT
-  #         fi
+          fi
 
-  #     - name: Run chart-testing (lint)
+      - name: Run chart-testing (lint)
-  #       run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false
+        run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false
 
-  #     - name: Create kind cluster
+      - name: Create kind cluster
-  #       uses: helm/kind-action@v1.8.0
+        uses: helm/kind-action@v1.8.0
+        if: steps.list-changed.outputs.changed == 'true'
+
+      # - name: Install prometheus operator
+      #   id: install-prometheus
       #   if: steps.list-changed.outputs.changed == 'true'
-  #       with:
-  #         config: tools/kind.config
-
-  #     - name: Run chart-testing (install)
       #   run: |
-  #         changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+      #     kubectl create namespace prometheus
-  #         ct install --config "${CT_CONFIGFILE}"
+
+      #     helm install prometheus prometheus-community/kube-prometheus-stack \
+      #       --namespace prometheus \
+      #       --set grafana.enabled=false \
+      #       --set prometheus.prometheusSpec.serviceMonitorSelector.matchLabels.release=prometheus
+
+      #     kubectl --namespace prometheus get pods -l "release=prometheus"
+      #     kubectl --namespace prometheus get services -l "release=prometheus"
+
+      - name: Run chart-testing (install)
+        run: |
+          changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+          if [[ "$changed" == "charts/enterprise-metrics" ]]; then
+            # Do not run `ct install` for enterprise-metrics
+            exit 0
+          fi
+          ct install --config "${CT_CONFIGFILE}"

.github/workflows/helm-release.yml

@@ -1,175 +0,0 @@
-name: Release Helm chart
-on:
-  workflow_dispatch:
-env:
-  CR_CONFIGFILE: "${{ github.workspace }}/source/.github/configs/cr.yaml"
-  CT_CONFIGFILE: "${{ github.workspace }}/source/.github/configs/ct.yaml"
-  CR_INDEX_PATH: "${{ github.workspace }}/.cr-index"
-  CR_PACKAGE_PATH: "${{ github.workspace }}/.cr-release-packages"
-  CR_TOOL_PATH: "${{ github.workspace }}/.cr-tool"
-  CR_VERSION: "1.5.0"
-jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      changed: ${{ steps.list-changed.outputs.changed }}
-      chartpath: ${{ steps.list-changed.outputs.chartpath }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        path: source
-
-    - name: Install chart-testing
-      uses: helm/chart-testing-action@v2
-
-    - name: List changed charts
-      id: list-changed
-      run: |
-        cd source
-
-        latest_tag=$( if ! git describe --tags --abbrev=0 --match='helm-chart/*' 2> /dev/null ; then git rev-list --max-parents=0 --first-parent HEAD; fi )
-
-        echo "Running: ct list-changed --config ${CT_CONFIGFILE} --since ${latest_tag} --target-branch ${{ github.ref_name }}"
-        changed=$(ct list-changed --config "${CT_CONFIGFILE}" --since "${latest_tag}" --target-branch "${{ github.ref_name }}")
-        echo "${changed}"
-
-        num_changed=$(wc -l <<< ${changed})
-        if [[ "${num_changed}" -gt "1" ]] ; then
-          echo "More than one chart changed, exiting"
-          exit 1
-        fi
-        if [[ -n "${changed}" ]]; then
-          name=$(yq ".name" < ${changed}/Chart.yaml)
-          version=$(yq ".version" < ${changed}/Chart.yaml)
-          tagname="v${version}"
-
-          if [ $(git tag -l "${tagname}") ]; then
-            echo "Tag ${tagname} already exists, skipping release"
-            echo "changed=false" >> $GITHUB_OUTPUT
-          else
-            echo "Releasing ${changed}"
-            echo "changed=true" >> $GITHUB_OUTPUT
-            echo "chartpath=${changed}" >> $GITHUB_OUTPUT
-          fi
-        else
-          echo "No charts have changed, skipping release"
-          echo "changed=false" >> $GITHUB_OUTPUT
-        fi
-
-  release:
-    needs: [setup]
-    runs-on: ubuntu-latest
-    if: needs.setup.outputs.changed == 'true'
-    permissions:
-      contents: write
-      id-token: write
-    steps:
-    - id: get-secrets
-      uses: grafana/shared-workflows/actions/get-vault-secrets@main
-      with:
-        # Secrets placed in the ci/repo/grafana/<repo>/<path> path in Vault
-        repo_secrets: |
-          APP_ID=github-app:app-id
-          PRIVATE_KEY=github-app:private-key
-    - uses: actions/create-github-app-token@v1
-      id: app-token
-      with:
-        app-id: ${{ env.APP_ID }}
-        private-key: ${{ env.PRIVATE_KEY }}
-        owner: ${{ github.repository_owner }}
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        path: source
-
-    - name: Configure Git
-      run: |
-        cd source
-        git config user.name "$GITHUB_ACTOR"
-        git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-    - name: Checkout helm-charts
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        repository: grafana/helm-charts
-        path: helm-charts
-        token: "${{ steps.app-token.outputs.token }}"
-
-    - name: Configure Git for helm-charts
-      run: |
-        cd helm-charts
-        git config user.name "$GITHUB_ACTOR"
-        git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-    - name: Set up Helm
-      uses: azure/setup-helm@v4
-
-    - name: Parse Chart.yaml
-      id: parse-chart
-      run: |
-        cd source
-        changed="${{ needs.setup.outputs.chartpath }}"
-        description=$(yq ".description" < ${changed}/Chart.yaml)
-        name=$(yq ".name" < ${changed}/Chart.yaml)
-        version=$(yq ".version" < ${changed}/Chart.yaml)
-
-        echo "chartpath=${changed}" >> $GITHUB_OUTPUT
-        echo "desc=${description}" >> $GITHUB_OUTPUT
-        echo "tagname=v${version}" >> $GITHUB_OUTPUT
-        echo "packagename=${name}-${version}" >> $GITHUB_OUTPUT
-
-    - name: Install CR tool
-      run: |
-        mkdir "${CR_TOOL_PATH}"
-        mkdir "${CR_PACKAGE_PATH}"
-        mkdir "${CR_INDEX_PATH}"
-        curl -sSLo cr.tar.gz "https://github.com/helm/chart-releaser/releases/download/v${CR_VERSION}/chart-releaser_${CR_VERSION}_linux_amd64.tar.gz"
-        tar -xzf cr.tar.gz -C "${CR_TOOL_PATH}"
-        rm -f cr.tar.gz
-
-    - name: Create Helm package
-      run: |
-        cd source
-        helm repo add grafana https://grafana.github.io/helm-charts
-        helm repo add minio https://charts.min.io
-
-        "${CR_TOOL_PATH}/cr" package "${{ steps.parse-chart.outputs.chartpath }}" --config "${CR_CONFIGFILE}" --package-path "${CR_PACKAGE_PATH}"
-
-    - name: Make a release on this repo
-      uses: softprops/action-gh-release@v2
-      with:
-        name: ${{ steps.parse-chart.outputs.tagname }}
-        repository: grafana/meta-monitoring-chart
-        tag_name: ${{ steps.parse-chart.outputs.tagname }}
-        token: ${{ steps.app-token.outputs.token }}
-        generate_release_notes: true
-        files: |
-          ${{ env.CR_PACKAGE_PATH }}/${{ steps.parse-chart.outputs.packagename }}.tgz
-
-    # Note that this creates a release in grafana/helm-charts with a new tag.
-    # The tag name in grafana/helm-charts is <package>-<version>, while the
-    # tag name for grafana/meta-monitoring-chart is <version>.
-    - name: Make release on Helm Charts
-      uses: softprops/action-gh-release@v2
-      with:
-        name: ${{ steps.parse-chart.outputs.packagename }}
-        repository: grafana/helm-charts
-        tag_name: ${{ steps.parse-chart.outputs.packagename }}
-        token: ${{ steps.app-token.outputs.token }}
-        body: |
-          ${{ steps.parse-chart.outputs.desc }}
-
-          Source commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }}
-
-          Tag on source: https://github.com/${{ github.repository }}/releases/tag/${{ steps.parse-chart.outputs.tagname }}
-        files: |
-          ${{ env.CR_PACKAGE_PATH }}/${{ steps.parse-chart.outputs.packagename }}.tgz
-
-    - name: Update helm-charts index.yaml
-      run: |
-        cd helm-charts
-        "${CR_TOOL_PATH}/cr" index --config "${CR_CONFIGFILE}" --token "${{ steps.app-token.outputs.token }}" --index-path "${CR_INDEX_PATH}" --package-path "${CR_PACKAGE_PATH}" --push

Makefile

@@ -6,5 +6,5 @@ help:
 
 .PHONY: helm-lint
 
-helm-lint: ## Run helm linter
+helm-lint: ## run helm linter
 	$(MAKE) -BC charts/meta-monitoring lint

README.md

@@ -1,9 +1,20 @@
 # meta-monitoring-chart
 
-This is a meta-monitoring chart for Loki.
+This is a meta-monitoring chart for GEL, GEM and GET. It should be installed in a
+separate namespace next to GEM, GEL or GET installations.
 
 Note that this is pre-production software at the moment.
 
+## Preparation
+
+Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml).
+
+1. Add or remove the namespaces to monitor in the `namespacesToMonitor` setting
+
+1. Set the cluster name in the `clusterName` setting. This will be added as a label to all logs, metrics and traces.
+
+1. Create a `meta` namespace.
+
 ## Local and cloud modes
 
 The chart has 2 modes: local and cloud. In the local mode logs, metrics and/or traces are sent
@@ -23,6 +34,12 @@ Both modes can be enabled at the same time.
 
 ## Installation
 
+```
+helm install -n meta --skip-crds -f values.yaml meta ./charts/meta-monitoring
+```
+
+If the platform supports CRDs the `--skip-crds` option can be removed. However the CRDs are not used by this chart.
+
 For more instructions including how to update the chart go to the [installation](docs/installation.md) page.
 
 ## Supported features
@@ -42,6 +59,7 @@ Most of these features are enabled by default. See the values.yaml file for how
 
 ## Caveats
 
+- The [loki.source.kubernetes](https://grafana.com/docs/agent/latest/flow/reference/components/loki.source.kubernetes/) component of the Grafana Agent is used to scrape Kubernetes log files. This component is marked experimental at the moment.
 - This has not been tested on Openshift yet.
 - The underlying Loki, Mimir and Tempo are at the default size installed by the Helm chart. This might need changing when monitoring bigger Loki, Mimir or Tempo installations.
 - MinIO is used as storage at the moment with a limited retention. At the moment this chart cannot be used for monitoring over longer periods.

charts/meta-monitoring/Chart.lock

@@ -1,18 +1,18 @@
 dependencies:
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: 6.3.4
+  version: 5.8.0
-- name: alloy
+- name: grafana-agent
   repository: https://grafana.github.io/helm-charts
-  version: 0.1.1
+  version: 0.15.0
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 5.3.0
+  version: 4.4.1
 - name: tempo-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 1.9.3
+  version: 1.4.7
 - name: minio
   repository: https://charts.min.io
-  version: 5.1.0
+  version: 5.0.11
-digest: sha256:3a47359025acb478698760e2f27b1608fab95174508e32f86e31a06d7856fe32
+digest: sha256:da0e744b5046eb7972e0bf82d1d0ba4786e9600af63b65f35b16118105248074
-generated: "2024-04-23T07:02:48.466787696Z"
+generated: "2023-08-16T16:08:36.406791+01:00"

charts/meta-monitoring/Chart.yaml

@@ -1,6 +1,7 @@
 apiVersion: v2
 name: meta-monitoring
 description: A Helm chart for meta monitoring Grafana Loki, Mimir and Tempo
+
 # A chart can be either an 'application' or a 'library' chart.
 #
 # Application charts are a collection of templates that can be packaged into versioned archives
@@ -10,32 +11,35 @@ description: A Helm chart for meta monitoring Grafana Loki, Mimir and Tempo
 # a dependency of application charts to inject those utilities and functions into the rendering
 # pipeline. Library charts do not define any templates and therefore cannot be deployed.
 type: application
+
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 version: 0.0.1
+
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
 appVersion: "0.0.1"
+
 dependencies:
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: 6.3.4
+  version: "5.8.0"
   condition: local.logs.enabled
-- name: alloy
+- name: grafana-agent
   repository: https://grafana.github.io/helm-charts
-  version: 0.1.1
+  version: "0.15.0"
 - name: mimir-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 5.3.0
+  version: "4.4.1"
   condition: local.metrics.enabled
 - name: tempo-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 1.9.3
+  version: "1.4.7"
   condition: local.traces.enabled
 - name: minio
   repository: https://charts.min.io
-  version: 5.1.0
+  version: "5.0.11"
   condition: local.minio.enabled

charts/meta-monitoring/Makefile

@@ -1,4 +1,4 @@
-.DEFAULT_GOAL := lint
+.DEFAULT_GOAL := all
 .PHONY: lint lint-yaml
 
 lint: lint-yaml

charts/meta-monitoring/src/dashboards/loki-deletion.json

@@ -62,6 +62,7 @@
                         "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                         "format": "time_series",
                         "instant": true,
+                        "intervalFactor": 2,
                         "refId": "A"
                      }
                   ],
@@ -137,6 +138,7 @@
                         "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                         "format": "time_series",
                         "instant": true,
+                        "intervalFactor": 2,
                         "refId": "A"
                      }
                   ],
@@ -189,145 +191,232 @@
             "height": "250px",
             "panels": [
                {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 3,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "in progress",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "# of Delete Requests (received - processed) ",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 4,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "received",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Delete Requests Received / Day",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 5,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "processed",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Delete Requests Processed / Day",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
                }
             ],
             "repeat": null,
@@ -342,145 +431,232 @@
             "height": "250px",
             "panels": [
                {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 6,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Compactor CPU usage",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 7,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": " {{pod}} ",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Compactor memory usage (MiB)",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 8,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 4,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Compaction run duration (seconds)",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
                }
             ],
             "repeat": null,
@@ -495,98 +671,156 @@
             "height": "250px",
             "panels": [
                {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 9,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "failures",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Failures in Loading Delete Requests / Hour",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 10,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
                   "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{user}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Lines Deleted / Sec",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
                }
             ],
             "repeat": null,
@@ -648,7 +882,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",

charts/meta-monitoring/src/dashboards/loki-logs.json

@@ -77,7 +77,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -164,7 +164,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -250,7 +250,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -336,7 +336,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -422,7 +422,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -508,7 +508,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -595,7 +595,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -682,7 +682,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -718,7 +718,7 @@
             "bars": false,
             "dashLength": 10,
             "dashes": false,
-            "datasource": "$loki_datasource",
+            "datasource": "$logs",
             "fill": 1,
             "fillGradient": 0,
             "gridPos": {
@@ -787,7 +787,7 @@
                "sort": 2,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -819,7 +819,7 @@
             }
          },
          {
-            "datasource": "$loki_datasource",
+            "datasource": "$logs",
             "gridPos": {
                "h": 19,
                "w": 24,
@@ -861,7 +861,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",
@@ -918,7 +918,7 @@
             {
                "hide": 0,
                "label": null,
-               "name": "loki_datasource",
+               "name": "logs",
                "options": [ ],
                "query": "loki",
                "refresh": 1,

charts/meta-monitoring/src/dashboards/loki-mixin-recording-rules.json

@@ -600,7 +600,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",

charts/meta-monitoring/src/dashboards/loki-operational.json

@@ -102,7 +102,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -198,7 +198,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -293,7 +293,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -389,7 +389,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -485,7 +485,7 @@
                "sort": 2,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -591,7 +591,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -697,7 +697,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -794,7 +794,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -903,7 +903,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1000,7 +1000,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1109,7 +1109,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1215,7 +1215,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1312,7 +1312,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1420,7 +1420,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1517,7 +1517,7 @@
                "sort": 0,
                "value_type": "individual"
             },
-            "type": "timeseries",
+            "type": "graph",
             "xaxis": {
                "buckets": null,
                "mode": "time",
@@ -1618,7 +1618,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -1799,9 +1799,9 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))",
+                        "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}",
                         "intervalFactor": 3,
-                        "legendFormat": "{{pod}}",
+                        "legendFormat": "{{pod}}-{{container}}",
                         "refId": "A"
                      }
                   ],
@@ -1815,7 +1815,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -1907,7 +1907,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -1943,7 +1943,7 @@
                   "bars": true,
                   "dashLength": 10,
                   "dashes": false,
-                  "datasource": "$loki_datasource",
+                  "datasource": "$logs",
                   "fill": 1,
                   "fillGradient": 0,
                   "gridPos": {
@@ -1999,7 +1999,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2031,7 +2031,7 @@
                   }
                },
                {
-                  "datasource": "$loki_datasource",
+                  "datasource": "$logs",
                   "gridPos": {
                      "h": 18,
                      "w": 12,
@@ -2116,7 +2116,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2205,7 +2205,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2294,7 +2294,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2383,7 +2383,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2490,7 +2490,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2581,7 +2581,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2697,7 +2697,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -2850,7 +2850,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3006,9 +3006,9 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))",
+                        "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}",
                         "intervalFactor": 3,
-                        "legendFormat": "{{pod}}",
+                        "legendFormat": "{{pod}}-{{container}}",
                         "refId": "A"
                      }
                   ],
@@ -3022,7 +3022,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3114,7 +3114,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3150,7 +3150,7 @@
                   "bars": true,
                   "dashLength": 10,
                   "dashes": false,
-                  "datasource": "$loki_datasource",
+                  "datasource": "$logs",
                   "fill": 1,
                   "fillGradient": 0,
                   "gridPos": {
@@ -3206,7 +3206,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3238,7 +3238,7 @@
                   }
                },
                {
-                  "datasource": "$loki_datasource",
+                  "datasource": "$logs",
                   "gridPos": {
                      "h": 18,
                      "w": 18,
@@ -3323,7 +3323,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3443,7 +3443,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3535,7 +3535,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3655,7 +3655,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3747,7 +3747,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3861,7 +3861,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -3964,7 +3964,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4067,7 +4067,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4165,7 +4165,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4255,7 +4255,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4345,7 +4345,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4435,7 +4435,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4525,7 +4525,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4645,7 +4645,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4737,7 +4737,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4825,7 +4825,7 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
+                        "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
                         "refId": "A"
                      }
                   ],
@@ -4839,7 +4839,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4911,7 +4911,7 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
+                        "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
                         "refId": "A"
                      }
                   ],
@@ -4925,7 +4925,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -4997,7 +4997,7 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
+                        "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
                         "refId": "A"
                      }
                   ],
@@ -5011,7 +5011,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5083,7 +5083,7 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
+                        "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
                         "refId": "A"
                      }
                   ],
@@ -5097,7 +5097,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5169,17 +5169,17 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
+                        "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
                         "legendFormat": ".99",
                         "refId": "A"
                      },
                      {
-                        "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
+                        "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
                         "legendFormat": ".9",
                         "refId": "B"
                      },
                      {
-                        "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
+                        "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
                         "legendFormat": ".5",
                         "refId": "C"
                      }
@@ -5194,7 +5194,7 @@
                      "sort": 0,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5269,19 +5269,19 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
+                        "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
                         "intervalFactor": 1,
                         "legendFormat": ".99-{{operation}}",
                         "refId": "A"
                      },
                      {
-                        "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
+                        "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
                         "hide": false,
                         "legendFormat": ".9-{{operation}}",
                         "refId": "B"
                      },
                      {
-                        "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
+                        "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
                         "hide": false,
                         "legendFormat": ".5-{{operation}}",
                         "refId": "C"
@@ -5297,7 +5297,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5372,7 +5372,7 @@
                   "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
+                        "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                         "intervalFactor": 1,
                         "legendFormat": "{{status_code}}-{{operation}}",
                         "refId": "A"
@@ -5388,7 +5388,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5507,7 +5507,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5598,7 +5598,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5717,7 +5717,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5808,7 +5808,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -5927,7 +5927,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -6018,7 +6018,7 @@
                      "sort": 2,
                      "value_type": "individual"
                   },
-                  "type": "timeseries",
+                  "type": "graph",
                   "xaxis": {
                      "buckets": null,
                      "mode": "time",
@@ -6070,7 +6070,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",
@@ -6081,7 +6081,7 @@
             {
                "hide": 0,
                "label": null,
-               "name": "loki_datasource",
+               "name": "logs",
                "options": [ ],
                "query": "loki",
                "refresh": 1,

charts/meta-monitoring/src/dashboards/loki-reads.json

@@ -33,234 +33,113 @@
                      "3xx": "#6ED0E0",
                      "4xx": "#EF843C",
                      "5xx": "#E24D42",
-                     "OK": "#7EB26D",
-                     "cancel": "#A9A9A9",
                      "error": "#E24D42",
                      "success": "#7EB26D"
                   },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 100,
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
-                           }
-                        },
-                        "min": 0,
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "1xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EAB839",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "2xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "3xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#6ED0E0",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "4xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EF843C",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "5xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "OK"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "cancel"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#A9A9A9",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "error"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "success"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        }
-                     ]
-                  },
                   "fill": 10,
                   "id": 1,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
                   "linewidth": 0,
                   "links": [ ],
-                  "options": {
+                  "nullPointMode": "null as zero",
-                     "legend": {
+                  "percentage": false,
-                        "showLegend": true
+                  "pointradius": 5,
-                     },
+                  "points": false,
-                     "tooltip": {
+                  "renderer": "flot",
-                        "mode": "single",
+                  "seriesOverrides": [ ],
-                        "sort": "none"
+                  "spaceLength": 10,
-                     }
+                  "span": 6,
-                  },
-                  "span": 4,
                   "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{status}}",
-                        "refId": "A"
+                        "refId": "A",
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "QPS",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 10,
+                        "show": false
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "ms"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 1,
-                  },
                   "id": 2,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
+                  "linewidth": 1,
                   "links": [ ],
                   "nullPointMode": "null as zero",
-                  "options": {
+                  "percentage": false,
-                     "legend": {
+                  "pointradius": 5,
-                        "showLegend": true
+                  "points": false,
-                     },
+                  "renderer": "flot",
-                     "tooltip": {
+                  "seriesOverrides": [ ],
-                        "mode": "single",
+                  "spaceLength": 10,
-                        "sort": "none"
+                  "span": 6,
-                     }
+                  "stack": false,
-                  },
+                  "steppedLine": false,
-                  "span": 4,
                   "targets": [
                      {
                         "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3",
@@ -287,8 +166,23 @@
                         "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Latency",
-                  "type": "timeseries",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
                   "yaxes": [
                      {
                         "format": "ms",
@@ -307,57 +201,6 @@
                         "show": false
                      }
                   ]
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "ms"
-                     },
-                     "overrides": [ ]
-                  },
-                  "id": 3,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "options": {
-                     "legend": {
-                        "showLegend": true
-                     },
-                     "tooltip": {
-                        "mode": "single",
-                        "sort": "none"
-                     }
-                  },
-                  "span": 4,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])) by (le,pod)) * 1e3",
-                        "format": "time_series",
-                        "interval": "1m",
-                        "intervalFactor": 2,
-                        "legendFormat": "__auto",
-                        "refId": "A",
-                        "step": 10
-                     }
-                  ],
-                  "title": "Per Pod Latency (p99)",
-                  "type": "timeseries"
                }
             ],
             "repeat": null,
@@ -378,256 +221,156 @@
                      "3xx": "#6ED0E0",
                      "4xx": "#EF843C",
                      "5xx": "#E24D42",
-                     "OK": "#7EB26D",
-                     "cancel": "#A9A9A9",
                      "error": "#E24D42",
                      "success": "#7EB26D"
                   },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 100,
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
-                           }
-                        },
-                        "min": 0,
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "1xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EAB839",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "2xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "3xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#6ED0E0",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "4xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EF843C",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "5xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "OK"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "cancel"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#A9A9A9",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "error"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "success"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        }
-                     ]
-                  },
                   "fill": 10,
-                  "id": 4,
+                  "id": 3,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
                   "linewidth": 0,
                   "links": [ ],
-                  "options": {
+                  "nullPointMode": "null as zero",
-                     "legend": {
+                  "percentage": false,
-                        "showLegend": true
+                  "pointradius": 5,
-                     },
+                  "points": false,
-                     "tooltip": {
+                  "renderer": "flot",
-                        "mode": "single",
+                  "seriesOverrides": [ ],
-                        "sort": "none"
+                  "spaceLength": 10,
-                     }
+                  "span": 6,
-                  },
-                  "span": 4,
                   "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{status}}",
-                        "refId": "A"
+                        "refId": "A",
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "QPS",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 10,
+                        "show": false
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
+                  "datasource": "$datasource",
+                  "fill": 1,
+                  "id": 4,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
                   },
-                        "unit": "ms"
+                  "lines": true,
-                     },
+                  "linewidth": 1,
-                     "overrides": [ ]
-                  },
-                  "id": 5,
                   "links": [ ],
                   "nullPointMode": "null as zero",
-                  "options": {
+                  "percentage": false,
-                     "legend": {
+                  "pointradius": 5,
-                        "showLegend": true
+                  "points": false,
-                     },
+                  "renderer": "flot",
-                     "tooltip": {
+                  "seriesOverrides": [ ],
-                        "mode": "single",
+                  "spaceLength": 10,
-                        "sort": "none"
+                  "span": 6,
-                     }
+                  "stack": false,
-                  },
+                  "steppedLine": false,
-                  "span": 4,
                   "targets": [
                      {
                         "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "99th Percentile",
-                        "refId": "A"
+                        "refId": "A",
+                        "step": 10
                      },
                      {
                         "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "50th Percentile",
-                        "refId": "B"
+                        "refId": "B",
+                        "step": 10
                      },
                      {
                         "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "Average",
-                        "refId": "C"
+                        "refId": "C",
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Latency",
-                  "type": "timeseries",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
                   "yaxes": [
                      {
                         "format": "ms",
@@ -646,57 +389,6 @@
                         "show": false
                      }
                   ]
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "ms"
-                     },
-                     "overrides": [ ]
-                  },
-                  "id": 6,
-                  "links": [ ],
-                  "nullPointMode": "null as zero",
-                  "options": {
-                     "legend": {
-                        "showLegend": true
-                     },
-                     "tooltip": {
-                        "mode": "single",
-                        "sort": "none"
-                     }
-                  },
-                  "span": 4,
-                  "targets": [
-                     {
-                        "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3",
-                        "format": "time_series",
-                        "interval": "1m",
-                        "intervalFactor": 2,
-                        "legendFormat": "__auto",
-                        "refId": "A",
-                        "step": 10
-                     }
-                  ],
-                  "title": "Per Pod Latency (p99)",
-                  "type": "timeseries"
                }
             ],
             "repeat": null,
@@ -720,7 +412,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",

charts/meta-monitoring/src/dashboards/loki-writes-resources.json

@@ -27,447 +27,589 @@
             "collapsed": false,
             "panels": [
                {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "gridPos": { },
                   "id": 1,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "In-memory streams",
                   "tooltip": {
                      "sort": 2
                   },
-                  "type": "timeseries"
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "request"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#FFC000",
-                                    "mode": "fixed"
-                                 }
-                              },
-                              {
-                                 "id": "custom.fillOpacity",
-                                 "value": 0
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "limit"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E02F44",
-                                    "mode": "fixed"
-                                 }
-                              },
-                              {
-                                 "id": "custom.fillOpacity",
-                                 "value": 0
-                              }
-                           ]
-                        }
-                     ]
-                  },
                   "gridPos": { },
                   "id": 2,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
                   },
-                     "tooltip": {
+                  "lines": true,
-                        "mode": "single",
+                  "linewidth": 1,
-                        "sort": "none"
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [
+                     {
+                        "alias": "request",
+                        "color": "#FFC000",
+                        "fill": 0
+                     },
+                     {
+                        "alias": "limit",
+                        "color": "#E02F44",
+                        "fill": 0
                      }
-                  },
+                  ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      },
                      {
                         "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "request",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      },
                      {
                         "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "limit",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "CPU",
                   "tooltip": {
                      "sort": 2
                   },
-                  "type": "timeseries"
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "bytes"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "request"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#FFC000",
-                                    "mode": "fixed"
-                                 }
-                              },
-                              {
-                                 "id": "custom.fillOpacity",
-                                 "value": 0
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "limit"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E02F44",
-                                    "mode": "fixed"
-                                 }
-                              },
-                              {
-                                 "id": "custom.fillOpacity",
-                                 "value": 0
-                              }
-                           ]
-                        }
-                     ]
-                  },
                   "gridPos": { },
                   "id": 3,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
                   },
-                     "tooltip": {
+                  "lines": true,
-                        "mode": "single",
+                  "linewidth": 1,
-                        "sort": "none"
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [
+                     {
+                        "alias": "request",
+                        "color": "#FFC000",
+                        "fill": 0
+                     },
+                     {
+                        "alias": "limit",
+                        "color": "#E02F44",
+                        "fill": 0
                      }
-                  },
+                  ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      },
                      {
                         "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "request",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      },
                      {
                         "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "limit",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Memory (workingset)",
                   "tooltip": {
                      "sort": 2
                   },
-                  "type": "timeseries"
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "bytes",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 10,
+                        "show": false
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "bytes"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 1,
-                  },
                   "gridPos": { },
                   "id": 4,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Memory (go heap inuse)",
                   "tooltip": {
                      "sort": 2
                   },
-                  "type": "timeseries"
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "bytes",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 100,
+                        "show": false
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "Bps"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 10,
-                  },
                   "gridPos": { },
                   "id": 5,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 0,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Disk Writes",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "Bps",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 100,
+                        "show": false
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "Bps"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 10,
-                  },
                   "gridPos": { },
                   "id": 6,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 0,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{pod}} - {{device}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Disk Reads",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "Bps",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 10,
+                        "show": false
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "percentunit"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 1,
-                  },
                   "gridPos": { },
                   "id": 7,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
-                     },
+                     "current": false,
-                     "tooltip": {
+                     "max": false,
-                        "mode": "single",
+                     "min": false,
-                        "sort": "none"
+                     "show": true,
-                     }
+                     "total": false,
+                     "values": false
                   },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{persistentvolumeclaim}}",
-                        "legendLink": null
+                        "legendLink": null,
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Disk Space Utilization",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "percentunit",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
                }
             ],
             "repeat": null,
@@ -492,7 +634,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",

charts/meta-monitoring/src/dashboards/loki-writes.json

@@ -33,237 +33,116 @@
                      "3xx": "#6ED0E0",
                      "4xx": "#EF843C",
                      "5xx": "#E24D42",
-                     "OK": "#7EB26D",
-                     "cancel": "#A9A9A9",
                      "error": "#E24D42",
                      "success": "#7EB26D"
                   },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 100,
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
-                           }
-                        },
-                        "min": 0,
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "1xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EAB839",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "2xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "3xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#6ED0E0",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "4xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EF843C",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "5xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "OK"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "cancel"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#A9A9A9",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "error"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "success"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        }
-                     ]
-                  },
                   "fill": 10,
                   "id": 1,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
                   "linewidth": 0,
                   "links": [ ],
-                  "options": {
+                  "nullPointMode": "null as zero",
-                     "legend": {
+                  "percentage": false,
-                        "showLegend": true
+                  "pointradius": 5,
-                     },
+                  "points": false,
-                     "tooltip": {
+                  "renderer": "flot",
-                        "mode": "single",
+                  "seriesOverrides": [ ],
-                        "sort": "none"
+                  "spaceLength": 10,
-                     }
-                  },
                   "span": 6,
                   "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "{{status}}",
-                        "refId": "A"
+                        "refId": "A",
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "QPS",
-                  "type": "timeseries"
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
                      },
                      {
-                  "datasource": "$datasource",
+                        "format": "short",
-                  "fieldConfig": {
+                        "label": null,
-                     "defaults": {
+                        "logBase": 1,
-                        "custom": {
+                        "max": null,
-                           "drawStyle": "line",
+                        "min": null,
-                           "fillOpacity": 10,
+                        "show": false
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
                      }
+                  ]
                },
-                        "thresholds": {
+               {
-                           "mode": "absolute",
+                  "aliasColors": { },
-                           "steps": [ ]
+                  "bars": false,
-                        },
+                  "dashLength": 10,
-                        "unit": "ms"
+                  "dashes": false,
-                     },
+                  "datasource": "$datasource",
-                     "overrides": [ ]
+                  "fill": 1,
-                  },
                   "id": 2,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
+                  "linewidth": 1,
                   "links": [ ],
                   "nullPointMode": "null as zero",
-                  "options": {
+                  "percentage": false,
-                     "legend": {
+                  "pointradius": 5,
-                        "showLegend": true
+                  "points": false,
-                     },
+                  "renderer": "flot",
-                     "tooltip": {
+                  "seriesOverrides": [ ],
-                        "mode": "single",
+                  "spaceLength": 10,
-                        "sort": "none"
-                     }
-                  },
                   "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
+                        "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})) * 1e3",
                         "format": "time_series",
                         "intervalFactor": 2,
                         "legendFormat": "99th Percentile",
@@ -271,7 +150,7 @@
                         "step": 10
                      },
                      {
-                        "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
+                        "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})) * 1e3",
                         "format": "time_series",
                         "intervalFactor": 2,
                         "legendFormat": "50th Percentile",
@@ -279,7 +158,7 @@
                         "step": 10
                      },
                      {
-                        "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})",
+                        "expr": "1e3 * sum(cluster_job:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}) / sum(cluster_job:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
                         "format": "time_series",
                         "intervalFactor": 2,
                         "legendFormat": "Average",
@@ -287,8 +166,23 @@
                         "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Latency",
-                  "type": "timeseries",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
                   "yaxes": [
                      {
                         "format": "ms",
@@ -321,105 +215,76 @@
             "height": "250px",
             "panels": [
                {
+                  "aliasColors": {
+                     "1xx": "#EAB839",
+                     "2xx": "#7EB26D",
+                     "3xx": "#6ED0E0",
+                     "4xx": "#EF843C",
+                     "5xx": "#E24D42",
+                     "error": "#E24D42",
+                     "success": "#7EB26D"
+                  },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 10,
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
                   "id": 3,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
                   },
-                     "tooltip": {
+                  "lines": true,
-                        "mode": "single",
+                  "linewidth": 0,
-                        "sort": "none"
-                     }
-                  },
-                  "span": 6,
-                  "targets": [
-                     {
-                        "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))",
-                        "format": "time_series",
-                        "legendFormat": "bytes",
-                        "legendLink": null
-                     }
-                  ],
-                  "title": "Per Total Received Bytes",
-                  "type": "timeseries"
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [ ]
-                  },
-                  "id": 4,
                   "links": [ ],
-                  "options": {
+                  "nullPointMode": "null as zero",
-                     "legend": {
+                  "percentage": false,
-                        "showLegend": true
+                  "pointradius": 5,
-                     },
+                  "points": false,
-                     "tooltip": {
+                  "renderer": "flot",
-                        "mode": "single",
+                  "seriesOverrides": [ ],
-                        "sort": "none"
+                  "spaceLength": 10,
-                     }
-                  },
                   "span": 6,
                   "stack": true,
+                  "steppedLine": false,
                   "targets": [
                      {
-                        "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))",
+                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                         "format": "time_series",
-                        "legendFormat": "{{tenant}}",
+                        "intervalFactor": 2,
-                        "legendLink": null
+                        "legendFormat": "{{status}}",
+                        "refId": "A",
+                        "step": 10
                      }
                   ],
-                  "title": "Per Tenant",
+                  "thresholds": [ ],
-                  "type": "timeseries",
+                  "timeFrom": null,
+                  "timeShift": null,
+                  "title": "QPS",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
                   "yaxes": [
                      {
                         "format": "short",
                         "label": null,
                         "logBase": 1,
-                        "max": 1,
+                        "max": null,
                         "min": 0,
                         "show": true
                      },
@@ -427,281 +292,85 @@
                         "format": "short",
                         "label": null,
                         "logBase": 1,
-                        "max": 1,
+                        "max": null,
                         "min": null,
                         "show": false
                      }
                   ]
-               }
-            ],
-            "repeat": null,
-            "repeatIteration": null,
-            "repeatRowId": null,
-            "showTitle": true,
-            "title": "Write Path",
-            "titleSize": "h6"
                },
                {
-            "collapse": false,
+                  "aliasColors": { },
-            "height": "250px",
+                  "bars": false,
-            "panels": [
+                  "dashLength": 10,
-               {
+                  "dashes": false,
-                  "aliasColors": {
-                     "1xx": "#EAB839",
-                     "2xx": "#7EB26D",
-                     "3xx": "#6ED0E0",
-                     "4xx": "#EF843C",
-                     "5xx": "#E24D42",
-                     "OK": "#7EB26D",
-                     "cancel": "#A9A9A9",
-                     "error": "#E24D42",
-                     "success": "#7EB26D"
-                  },
                   "datasource": "$datasource",
-                  "fieldConfig": {
+                  "fill": 1,
-                     "defaults": {
+                  "id": 4,
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 100,
-                           "lineWidth": 0,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "normal"
-                           }
-                        },
-                        "min": 0,
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "short"
-                     },
-                     "overrides": [
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "1xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EAB839",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "2xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "3xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#6ED0E0",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "4xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#EF843C",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "5xx"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "OK"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "cancel"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#A9A9A9",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "error"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#E24D42",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        },
-                        {
-                           "matcher": {
-                              "id": "byName",
-                              "options": "success"
-                           },
-                           "properties": [
-                              {
-                                 "id": "color",
-                                 "value": {
-                                    "fixedColor": "#7EB26D",
-                                    "mode": "fixed"
-                                 }
-                              }
-                           ]
-                        }
-                     ]
-                  },
-                  "fill": 10,
-                  "id": 5,
-                  "linewidth": 0,
-                  "links": [ ],
-                  "options": {
                   "legend": {
-                        "showLegend": true
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
                   },
-                     "tooltip": {
+                  "lines": true,
-                        "mode": "single",
+                  "linewidth": 1,
-                        "sort": "none"
-                     }
-                  },
-                  "span": 6,
-                  "stack": true,
-                  "targets": [
-                     {
-                        "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
-                        "format": "time_series",
-                        "legendFormat": "{{status}}",
-                        "refId": "A"
-                     }
-                  ],
-                  "title": "QPS",
-                  "type": "timeseries"
-               },
-               {
-                  "datasource": "$datasource",
-                  "fieldConfig": {
-                     "defaults": {
-                        "custom": {
-                           "drawStyle": "line",
-                           "fillOpacity": 10,
-                           "lineWidth": 1,
-                           "pointSize": 5,
-                           "showPoints": "never",
-                           "spanNulls": false,
-                           "stacking": {
-                              "group": "A",
-                              "mode": "none"
-                           }
-                        },
-                        "thresholds": {
-                           "mode": "absolute",
-                           "steps": [ ]
-                        },
-                        "unit": "ms"
-                     },
-                     "overrides": [ ]
-                  },
-                  "id": 6,
                   "links": [ ],
                   "nullPointMode": "null as zero",
-                  "options": {
+                  "percentage": false,
-                     "legend": {
+                  "pointradius": 5,
-                        "showLegend": true
+                  "points": false,
-                     },
+                  "renderer": "flot",
-                     "tooltip": {
+                  "seriesOverrides": [ ],
-                        "mode": "single",
+                  "spaceLength": 10,
-                        "sort": "none"
-                     }
-                  },
                   "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
                   "targets": [
                      {
                         "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "99th Percentile",
-                        "refId": "A"
+                        "refId": "A",
+                        "step": 10
                      },
                      {
                         "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "50th Percentile",
-                        "refId": "B"
+                        "refId": "B",
+                        "step": 10
                      },
                      {
                         "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))",
                         "format": "time_series",
+                        "intervalFactor": 2,
                         "legendFormat": "Average",
-                        "refId": "C"
+                        "refId": "C",
+                        "step": 10
                      }
                   ],
+                  "thresholds": [ ],
+                  "timeFrom": null,
+                  "timeShift": null,
                   "title": "Latency",
-                  "type": "timeseries",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "graph",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
                   "yaxes": [
                      {
                         "format": "ms",
@@ -743,7 +412,7 @@
                   "value": "default"
                },
                "hide": 0,
-               "label": "Data source",
+               "label": "Data Source",
                "name": "datasource",
                "options": [ ],
                "query": "prometheus",

charts/meta-monitoring/src/rules/loki-rules.yaml

@@ -1,53 +1,53 @@
 groups:
 - name: "loki_rules"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[5m]))
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(loki_request_duration_seconds_count[1m]))
       by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job)"
     record: "cluster_job:loki_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
+      / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, job,
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, job,
       route)"
     record: "cluster_job_route:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, job, route)"
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, job, route)"
+  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, job, route)"
     record: "cluster_job_route:loki_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:50quantile"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(loki_request_duration_seconds_count[5m])) by (cluster,
+      job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds:avg"
-  - expr: "sum(rate(loki_request_duration_seconds_bucket[5m])) by (le, cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_sum[5m])) by (cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_sum[1m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(loki_request_duration_seconds_count[5m])) by (cluster, namespace,
+  - expr: "sum(rate(loki_request_duration_seconds_count[1m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate"

charts/meta-monitoring/src/rules/mimir-rules.yaml

@@ -1,317 +1,322 @@
 groups:
 - name: "mimir_api_1"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[5m]))
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
       by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_api_2"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job, route)
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
-      / sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job, route)"
+      / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, job,
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
       route)"
     record: "cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, job, route)"
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, job, route)"
+  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)"
     record: "cluster_job_route:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_api_3"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
-      job, route) / sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster,
+      job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_request_duration_seconds_bucket[5m])) by (le, cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_sum[5m])) by (cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_request_duration_seconds_count[5m])) by (cluster, namespace,
+  - expr: "sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
       job, route)"
     record: "cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate"
 - name: "mimir_querier_api"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
+      job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_querier_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, route))"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[5m])) by
+      job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
       (cluster, job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
       job, route)"
     record: "cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
       by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
-      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[5m]))
+      namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
       by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
       namespace, job, route)"
     record: "cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate"
 - name: "mimir_cache"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
-      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[5m]))
+      job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
       by (cluster, job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_memcache_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_cache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_cache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job)
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster, job)"
+      / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_cache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_cache_request_duration_seconds_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
       by (le, cluster, job, method))"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
-      method) / sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
+      method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[5m])) by (cluster, job,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
       method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
       job, method)"
     record: "cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate"
 - name: "mimir_storage"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_kv_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_kv_request_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[5m])) by (cluster, job)
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
-      / sum(rate(cortex_kv_request_duration_seconds_count[5m])) by (cluster, job)"
+      / sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds:avg"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_kv_request_duration_seconds_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_kv_request_duration_seconds_count:sum_rate"
 - name: "mimir_queries"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_retries:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_retries:50quantile"
-  - expr: "sum(rate(cortex_query_frontend_retries_sum[5m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[5m]))
+  - expr: "sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
       by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries:avg"
-  - expr: "sum(rate(cortex_query_frontend_retries_bucket[5m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_bucket:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_retries_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_sum:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_retries_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_query_frontend_retries_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
-      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[5m])) by
+      job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
       (cluster, job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds:avg"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[5m])) by (le,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
       cluster, job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[5m])) by (cluster,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_count[5m])) by (cluster,
+  - expr: "sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
       job)"
     record: "cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate"
 - name: "mimir_ingester_queries"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_series:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_series:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_series_sum[5m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[5m]))
+  - expr: "sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
       by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series:avg"
-  - expr: "sum(rate(cortex_ingester_queried_series_bucket[5m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_series_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_series_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_series_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_samples:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_samples:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_samples_sum[5m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[5m]))
+  - expr: "sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
       by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples:avg"
-  - expr: "sum(rate(cortex_ingester_queried_samples_bucket[5m])) by (le, cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_samples_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_samples_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_samples_count:sum_rate"
-  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[5m]))
+  - expr: "histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_exemplars:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[5m]))
+  - expr: "histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
       by (le, cluster, job))"
     record: "cluster_job:cortex_ingester_queried_exemplars:50quantile"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[5m])) by (cluster, job) /
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
-      sum(rate(cortex_ingester_queried_exemplars_count[5m])) by (cluster, job)"
+      sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars:avg"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_bucket[5m])) by (le, cluster,
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
       job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate"
-  - expr: "sum(rate(cortex_ingester_queried_exemplars_count[5m])) by (cluster, job)"
+  - expr: "sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)"
     record: "cluster_job:cortex_ingester_queried_exemplars_count:sum_rate"
 - name: "mimir_received_samples"
   rules:
-  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))"
+  - expr: "|
+      sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_received_samples:rate5m"
 - name: "mimir_exemplars_in"
   rules:
-  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))"
+  - expr: "|
+      sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_exemplars_in:rate5m"
 - name: "mimir_received_exemplars"
   rules:
-  - expr: "sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))"
+  - expr: "|
+      sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))"
     record: "cluster_namespace_job:cortex_distributor_received_exemplars:rate5m"
 - name: "mimir_exemplars_ingested"
   rules:
-  - expr: "sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))"
+  - expr: "|
+      sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))"
     record: "cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m"
 - name: "mimir_exemplars_appended"
   rules:
-  - expr: "sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))"
+  - expr: "|
+      sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))"
     record: "cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m"
 - name: "mimir_scaling_rules"
   rules:
-  - expr: |
+  - expr: "|
       # Convenience rule to get the number of replicas for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
       sum by (cluster, namespace, deployment) (
         label_replace(
           kube_deployment_spec_replicas,
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
         )
       )
       or
       sum by (cluster, namespace, deployment) (
-        label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
+        label_replace(kube_statefulset_replicas, \"deployment\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\")
-      )
+      )"
     record: "cluster_namespace_deployment:actual_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
         quantile_over_time(0.99,
           sum by (cluster, namespace) (
@@ -319,21 +324,21 @@ groups:
           )[24h:]
         )
         / 240000
-      )
+      )"
     labels:
       deployment: "distributor"
       reason: "sample_rate"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"ingestion_rate\"})
         * 0.59999999999999998 / 240000
-      )
+      )"
     labels:
       deployment: "distributor"
       reason: "sample_rate_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
         quantile_over_time(0.99,
           sum by (cluster, namespace) (
@@ -341,12 +346,12 @@ groups:
           )[24h:]
         )
         * 3 / 80000
-      )
+      )"
     labels:
       deployment: "ingester"
       reason: "sample_rate"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
         quantile_over_time(0.99,
           sum by(cluster, namespace) (
@@ -354,59 +359,59 @@ groups:
           )[24h:]
         )
         / 1500000
-      )
+      )"
     labels:
       deployment: "ingester"
       reason: "active_series"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"max_global_series_per_user\"})
         * 3 * 0.59999999999999998 / 1500000
-      )
+      )"
     labels:
       deployment: "ingester"
       reason: "active_series_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
-        sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+        sum by (cluster, namespace) (cortex_limits_overrides{limit_name=\"ingestion_rate\"})
         * 0.59999999999999998 / 80000
-      )
+      )"
     labels:
       deployment: "ingester"
       reason: "sample_rate_limits"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       ceil(
         (sum by (cluster, namespace) (
-          cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
+          cortex_ingester_tsdb_storage_blocks_bytes{job=~\".+/ingester.*\"}
         ) / 4)
           /
         avg by (cluster, namespace) (
-          memcached_limit_bytes{job=~".+/memcached"}
+          memcached_limit_bytes{job=~\".+/memcached\"}
-        )
         )
+      )"
     labels:
       deployment: "memcached"
       reason: "active_series"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       sum by (cluster, namespace, deployment) (
         label_replace(
           label_replace(
-            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[5m])),
+            sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
-            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
           ),
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
-        )
         )
+      )"
     record: "cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate"
-  - expr: |
+  - expr: "|
       # Convenience rule to get the CPU request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
       # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
       # that remove resource metrics, ref:
       # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
@@ -419,11 +424,11 @@ groups:
           label_replace(
             label_replace(
               kube_pod_container_resource_requests_cpu_cores,
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
             ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
           )
         )
       )
@@ -434,17 +439,17 @@ groups:
         sum by (cluster, namespace, deployment) (
           label_replace(
             label_replace(
-              kube_pod_container_resource_requests{resource="cpu"},
+              kube_pod_container_resource_requests{resource=\"cpu\"},
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
             ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
-          )
           )
         )
+      )"
     record: "cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum"
-  - expr: |
+  - expr: "|
       # Jobs should be sized to their CPU usage.
       # We do this by comparing 99th percentile usage over the last 24hrs to
       # their current provisioned #replicas and resource requests.
@@ -454,28 +459,28 @@ groups:
         quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
           /
         cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
-      )
+      )"
     labels:
       reason: "cpu_usage"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
-  - expr: |
+  - expr: "|
       # Convenience rule to get the Memory utilization for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
       sum by (cluster, namespace, deployment) (
         label_replace(
           label_replace(
-            container_memory_usage_bytes{image!=""},
+            container_memory_usage_bytes{image!=\"\"},
-            "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+            \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
           ),
-          # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+          # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
           # always matches everything and the (optional) zone is not removed.
-          "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+          \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
-        )
         )
+      )"
     record: "cluster_namespace_deployment:container_memory_usage_bytes:sum"
-  - expr: |
+  - expr: "|
       # Convenience rule to get the Memory request for both a deployment and a statefulset.
-      # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+      # Multi-zone deployments are grouped together removing the \"zone-X\" suffix.
       # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
       # that remove resource metrics, ref:
       # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
@@ -488,11 +493,11 @@ groups:
           label_replace(
             label_replace(
               kube_pod_container_resource_requests_memory_bytes,
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
             ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
           )
         )
       )
@@ -503,17 +508,17 @@ groups:
         sum by (cluster, namespace, deployment) (
           label_replace(
             label_replace(
-              kube_pod_container_resource_requests{resource="memory"},
+              kube_pod_container_resource_requests{resource=\"memory\"},
-              "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+              \"deployment\", \"$1\", \"pod\", \"(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))\"
             ),
-            # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+            # The question mark in \"(.*?)\" is used to make it non-greedy, otherwise it
             # always matches everything and the (optional) zone is not removed.
-            "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+            \"deployment\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"
-          )
           )
         )
+      )"
     record: "cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum"
-  - expr: |
+  - expr: "|
       # Jobs should be sized to their Memory usage.
       # We do this by comparing 99th percentile usage over the last 24hrs to
       # their current provisioned #replicas and resource requests.
@@ -523,33 +528,44 @@ groups:
         quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
           /
         cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
-      )
+      )"
     labels:
       reason: "memory_usage"
     record: "cluster_namespace_deployment_reason:required_replicas:count"
 - name: "mimir_alertmanager_rules"
   rules:
-  - expr: "sum by (cluster, job, pod) (cortex_alertmanager_alerts)"
+  - expr: "|
+      sum by (cluster, job, pod) (cortex_alertmanager_alerts)"
     record: "cluster_job_pod:cortex_alertmanager_alerts:sum"
-  - expr: "sum by (cluster, job, pod) (cortex_alertmanager_silences)"
+  - expr: "|
+      sum by (cluster, job, pod) (cortex_alertmanager_silences)"
     record: "cluster_job_pod:cortex_alertmanager_silences:sum"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))"
     record: "cluster_job:cortex_alertmanager_alerts_received_total:rate5m"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))"
     record: "cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m"
-  - expr: "sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))"
+  - expr: "|
+      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))"
     record: "cluster_job_integration:cortex_alertmanager_notifications_total:rate5m"
-  - expr: "sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))"
+  - expr: "|
+      sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))"
     record: "cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))"
     record: "cluster_job:cortex_alertmanager_state_replication_total:rate5m"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))"
     record: "cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))"
     record: "cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m"
-  - expr: "sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))"
+  - expr: "|
+      sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))"
     record: "cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m"
 - name: "mimir_ingester_rules"
   rules:
-  - expr: "sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[5m]))"
+  - expr: "|
+      sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))"
     record: "cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m"

charts/meta-monitoring/src/rules/tempo-rules.yaml

@@ -1,15 +1,15 @@
 groups:
 - name: "tempo_rules"
   rules:
-  - expr: "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route))"
+  - expr: "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:99quantile"
-  - expr: "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route))"
+  - expr: "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route))"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:50quantile"
-  - expr: "sum(rate(tempo_request_duration_seconds_sum[5m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[5m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route) / sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds:avg"
-  - expr: "sum(rate(tempo_request_duration_seconds_bucket[5m])) by (le, cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_bucket[1m])) by (le, cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_bucket:sum_rate"
-  - expr: "sum(rate(tempo_request_duration_seconds_sum[5m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_sum[1m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_sum:sum_rate"
-  - expr: "sum(rate(tempo_request_duration_seconds_count[5m])) by (cluster, namespace, job, route)"
+  - expr: "sum(rate(tempo_request_duration_seconds_count[1m])) by (cluster, namespace, job, route)"
     record: "cluster_namespace_job_route:tempo_request_duration_seconds_count:sum_rate"

charts/meta-monitoring/templates/_helpers.tpl

@@ -1,33 +0,0 @@
-{{/*
-Return the appropriate apiVersion for ingress.
-*/}}
-{{- define "ingress.apiVersion" -}}
-  {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}}
-      {{- print "networking.k8s.io/v1" -}}
-  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
-    {{- print "networking.k8s.io/v1beta1" -}}
-  {{- else -}}
-    {{- print "extensions/v1beta1" -}}
-  {{- end -}}
-{{- end -}}
-
-{{/*
-Return if ingress is stable.
-*/}}
-{{- define "ingress.isStable" -}}
-  {{- eq (include "ingress.apiVersion" .) "networking.k8s.io/v1" -}}
-{{- end -}}
-
-{{/*
-Return if ingress supports ingressClassName.
-*/}}
-{{- define "ingress.supportsIngressClassName" -}}
-  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
-{{- end -}}
-
-{{/*
-Return if ingress supports pathType.
-*/}}
-{{- define "ingress.supportsPathType" -}}
-  {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
-{{- end -}}

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -18,10 +18,10 @@
 {{- end }}
 
 {{- define "agent.loki_process_targets" -}}
-{{- if and (empty .Values.logs.piiRegexes) (empty .Values.logs.retain) }}
+{{- if empty .Values.logs.piiRegexes }}
 {{- include "agent.loki_write_targets" . }}
 {{- else }}
-{{- printf "loki.process.filter.receiver" }}
+{{- printf "loki.process.PII.receiver" }}
 {{- end }}
 {{- end }}
 

charts/meta-monitoring/templates/agent/config.yaml

@@ -8,7 +8,7 @@ data:
     discovery.kubernetes "pods" {
       role = "pod"
       namespaces {
-        own_namespace = true
+        own_namespace = false
         names = [ {{ include "agent.namespaces" . }} ]
       }
     }
@@ -33,38 +33,22 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterLabelValue -}}"
+        replacement = "{{- .Values.clusterName -}}"
       }
     }
 
-    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
     // Logs
 
-    remote.kubernetes.secret "logs_credentials" {
+    {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }}
-      namespace = "{{- $.Release.Namespace -}}"
-      name = "{{- .Values.cloud.logs.secret -}}"
-    }
-
     loki.source.kubernetes "pods" {
-      clustering {
-        enabled = true
-      }
       targets    = discovery.relabel.rename_meta_labels.output
       forward_to = [ {{ include "agent.loki_process_targets" . }} ]
     }
 
-    {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.piiRegexes)) }}
+    {{- if not (empty .Values.logs.piiRegexes) }}
-    loki.process "filter" {
+    loki.process "PII" {
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
-      {{- if not (empty .Values.logs.retain) }}
-      stage.match {
-        selector = "{cluster=\"{{- .Values.clusterLabelValue -}}\", namespace=~\"{{- join "|" .Values.namespacesToMonitor -}}|{{- $.Release.Namespace -}}\", pod=~\"loki.*\"} !~ \"{{ join "|" .Values.logs.retain }}\""
-        action   = "drop"
-      }
-      {{- end }}
-
-      {{- if not (empty .Values.logs.piiRegexes) }}
       {{- range .Values.logs.piiRegexes }}
       stage.replace {
         expression = "{{ .expression }}"
@@ -72,85 +56,26 @@ data:
         replace = "{{ .replace }}"
       }
       {{- end }}
-      {{- end }}
     }
     {{- end }}
     {{- end }}
 
-    {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
     // Metrics
 
-    remote.kubernetes.secret "metrics_credentials" {
+    {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }}
-      namespace = "{{- $.Release.Namespace -}}"
-      name = "{{- .Values.cloud.metrics.secret -}}"
-    }
-
-    discovery.kubernetes "metric_pods" {
-      role = "pod"
-      namespaces {
-        own_namespace = true
-        names = [ {{ include "agent.namespaces" . }} ]
-      }
-    }
-
-    discovery.relabel "only_http_metrics" {
-      targets = discovery.kubernetes.metric_pods.targets
-
-      rule {
-        source_labels = ["__meta_kubernetes_namespace"]
-        target_label  = "namespace"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_pod_name"]
-        target_label  = "pod"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"]
-        separator = "/"
-        regex = "(.*)/(.*)/(.*)"
-        replacement = "${1}/${2}-${3}"
-        target_label = "job"
-      }
-      rule {
-        target_label = "cluster"
-        replacement = "{{- .Values.clusterLabelValue -}}"
-      }
-      rule {
-        source_labels = ["__meta_kubernetes_pod_container_port_number"]
-        action = "drop"
-        regex = "9095"
-      }
-    }
-
     prometheus.scrape "pods" {
-      clustering {
+      targets    = discovery.relabel.rename_meta_labels.output
-        enabled = true
-      }
-      targets    = discovery.relabel.only_http_metrics.output
-      forward_to = [ prometheus.relabel.filter.receiver ]
-    }
-
-    prometheus.relabel "filter" {
-      rule {
-        source_labels = ["__name__"]
-        regex = "({{ join "|" .Values.metrics.retain }})"
-        action = "keep"
-      }
-
       forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
     {{- if .Values.kubeStateMetrics.enabled }}
 
     prometheus.scrape "kubeStateMetrics" {
-      clustering {
-        enabled = true
-      }
       targets    = [ { "__address__" = "{{ .Values.kubeStateMetrics.endpoint }}" } ]
-      forward_to = [ prometheus.relabel.filter.receiver ]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
     }
     {{- end }}
 
-    // cAdvisor and Kubelet metrics
+    // cAdvisor and Kubelete metrics
     // Based on https://github.com/Chewie/loutretelecom-manifests/blob/main/manifests/addons/monitoring/config.river
     discovery.kubernetes "all_nodes" {
       role = "node"
@@ -179,17 +104,15 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterLabelValue -}}"
+        replacement = "{{- .Values.clusterName -}}"
       }
     }
 
     prometheus.scrape "cadvisor" {
-      clustering {
-        enabled = true
-      }
       targets    = discovery.relabel.all_nodes.output
-      forward_to = [ prometheus.relabel.filter.receiver ]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
 
+      scrape_interval = "15s"
       metrics_path = "/metrics/cadvisor"
       scheme = "https"
 
@@ -200,12 +123,10 @@ data:
     }
 
     prometheus.scrape "kubelet" {
-      clustering {
-        enabled = true
-      }
       targets    = discovery.relabel.all_nodes.output
-      forward_to = [ prometheus.relabel.filter.receiver ]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
 
+      scrape_interval = "15s"
       metrics_path = "/metrics"
       scheme = "https"
 
@@ -215,20 +136,18 @@ data:
       }
     }
 
-    prometheus.exporter.unix "promexporter" {}
+    prometheus.exporter.unix {}
 
     prometheus.scrape "node_exporter" {
-      clustering {
+      targets = prometheus.exporter.unix.targets
-        enabled = true
-      }
-      targets = prometheus.exporter.unix.promexporter.targets
       forward_to = [prometheus.relabel.node_exporter.receiver]
 
       job_name = "node-exporter"
+      scrape_interval = "15s"
     }
 
     prometheus.relabel "node_exporter" {
-      forward_to = [ prometheus.relabel.filter.receiver ]
+      forward_to = [ {{ include "agent.prometheus_write_targets" . }} ]
 
       rule {
         replacement = env("HOSTNAME")
@@ -259,19 +178,14 @@ data:
       }
       rule {
         target_label = "cluster"
-        replacement = "{{- .Values.clusterLabelValue -}}"
+        replacement = "{{- .Values.clusterName -}}"
       }
     }
     {{- end }}
 
-    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
     // Traces
 
-    remote.kubernetes.secret "traces_credentials" {
+    {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }}
-      namespace = "{{- $.Release.Namespace -}}"
-      name = "{{- .Values.cloud.traces.secret -}}"
-    }
-
     // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river
     otelcol.receiver.otlp "otlp_receiver" {
       // We don't technically need this, but it shows how to change listen address and incoming port.
@@ -340,10 +254,11 @@ data:
     {{- if .Values.cloud.logs.enabled }}
     loki.write "cloud" {
       endpoint {
-        url = nonsensitive(remote.kubernetes.secret.logs_credentials.data["endpoint"])
+        url = "{{- .Values.cloud.logs.endpoint -}}/loki/api/v1/push"
+
         basic_auth {
-          username = nonsensitive(remote.kubernetes.secret.logs_credentials.data["username"])
+          username = "{{- .Values.cloud.logs.username -}}"
-          password = remote.kubernetes.secret.logs_credentials.data["password"]
+          password = "{{- .Values.cloud.logs.password -}}"
         }
       }
     }
@@ -352,10 +267,11 @@ data:
     {{- if .Values.cloud.metrics.enabled }}
     prometheus.remote_write "cloud" {
       endpoint {
-        url = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["endpoint"])
+        url = "{{- .Values.cloud.metrics.endpoint -}}/api/prom/push"
+
         basic_auth {
-          username = nonsensitive(remote.kubernetes.secret.metrics_credentials.data["username"])
+          username = "{{- .Values.cloud.metrics.username -}}"
-          password = remote.kubernetes.secret.metrics_credentials.data["password"]
+          password = "{{- .Values.cloud.metrics.password -}}"
         }
       }
     }
@@ -364,13 +280,13 @@ data:
     {{- if .Values.cloud.traces.enabled }}
     otelcol.exporter.otlp "cloud" {
         client {
-            endpoint = nonsensitive(remote.kubernetes.secret.traces_credentials.data["endpoint"])
+            endpoint = "{{- .Values.cloud.traces.endpoint -}}"
             auth = otelcol.auth.basic.creds.handler
         }
     }
 
     otelcol.auth.basic "creds" {
-      username = nonsensitive(remote.kubernetes.secret.traces_credentials.data["username"])
+      username = "{{- .Values.cloud.traces.username -}}"
-      password = remote.kubernetes.secret.traces_credentials.data["password"]
+      password = "{{- .Values.cloud.traces.password -}}"
     }
     {{- end }}

charts/meta-monitoring/templates/grafana/agent-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
+{{- if .Values.dashboards.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -14,6 +14,6 @@ data:
     {{ $.Files.Get "src/dashboards/agent-remote-write.json" | fromJson | toJson }}
   "agent-tracing-pipeline.json": |
     {{ $.Files.Get "src/dashboards/agent-tracing-pipeline.json" | fromJson | toJson }}
-  "agent.json": |
+  "agent-overview.json": |
     {{ $.Files.Get "src/dashboards/agent.json" | fromJson | toJson }}
 {{- end }}

charts/meta-monitoring/templates/grafana/dashboard.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/datasources.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.grafana.enabled }}
+{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/grafana-ingress.yaml

@@ -1,57 +0,0 @@
-{{- if and .Values.local.grafana.enabled .Values.grafana.ingress.enabled -}}
-{{- $ingressApiIsStable := eq (include "ingress.isStable" .) "true" -}}
-{{- $ingressSupportsIngressClassName := eq (include "ingress.supportsIngressClassName" .) "true" -}}
-{{- $ingressSupportsPathType := eq (include "ingress.supportsPathType" .) "true" -}}
-apiVersion: {{ include "ingress.apiVersion" . }}
-kind: Ingress
-metadata:
-  name: grafana
-  namespace: {{ $.Release.Namespace }}
-  labels:
-    app: grafana
-    {{- range $labelKey, $labelValue := .Values.grafana.ingress.labels }}
-    {{ $labelKey }}: {{ $labelValue | toYaml }}
-    {{- end }}
-  {{- with .Values.grafana.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- if and $ingressSupportsIngressClassName .Values.grafana.ingress.ingressClassName }}
-  ingressClassName: {{ .Values.grafana.ingress.ingressClassName }}
-  {{- end -}}
-  {{- if .Values.grafana.ingress.tls }}
-  tls:
-    {{- range .Values.grafana.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ tpl . $ | quote }}
-        {{- end }}
-      {{- with .secretName }}
-      secretName: {{ . }}
-      {{- end }}
-    {{- end }}
-  {{- end }}
-  rules:
-    {{- range .Values.grafana.ingress.hosts }}
-    - host: {{ tpl .host $ | quote }}
-      http:
-        paths:
-          {{- range .paths }}
-          - path: {{ .path }}
-            {{- if $ingressSupportsPathType }}
-            pathType: {{ .pathType }}
-            {{- end }}
-            backend:
-              {{- if $ingressApiIsStable }}
-              service:
-                name: grafana
-                port:
-                  number: 3000
-              {{- else }}
-              serviceName: grafana
-              servicePort: 3000
-              {{- end }}
-          {{- end }}
-    {{- end }}
-{{- end }}

charts/meta-monitoring/templates/grafana/grafana.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.local.grafana.enabled }}
+{{- if or (or .Values.local.logs.enabled .Values.local.metrics.enabled) .Values.local.traces.enabled }}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim

charts/meta-monitoring/templates/grafana/loki-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
+{{- if .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/loki-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.logs.enabled }}
+{{- if .Values.dashboards.logs.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dahboards-5.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-2.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-3.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/mimir-dashboards-4.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.metrics.enabled }}
+{{- if .Values.dashboards.metrics.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/grafana/tempo-dashboards-1.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.local.grafana.enabled .Values.dashboards.traces.enabled }}
+{{- if .Values.dashboards.traces.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap

charts/meta-monitoring/templates/ruler/ruler.yaml

@@ -1,9 +1,8 @@
-{{- if .Values.local.grafana.enabled }}
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
-{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ $.Release.Namespace }}-mimir-ruler-for-dashboards
+  name: meta-mimir-ruler-for-dashboards
   namespace: {{ $.Release.Namespace }}
 spec:
   progressDeadlineSeconds: 600
@@ -92,6 +91,8 @@ spec:
         runAsUser: 10001
         seccompProfile:
           type: RuntimeDefault
+      serviceAccount: meta-mimir
+      serviceAccountName: meta-mimir
       terminationGracePeriodSeconds: 180
       topologySpreadConstraints:
         - labelSelector:
@@ -108,11 +109,11 @@ spec:
             items:
               - key: mimir.yaml
                 path: mimir.yaml
-            name: {{ $.Release.Namespace }}-mimir-config
+            name: meta-mimir-config
           name: config
         - configMap:
             defaultMode: 420
-            name: {{ $.Release.Namespace }}-mimir-runtime
+            name: meta-mimir-runtime
           name: runtime-config
         - emptyDir: {}
           name: storage
@@ -123,4 +124,3 @@ spec:
             name: rules
           name: rules
 {{- end }}
-{{- end }}

charts/meta-monitoring/templates/ruler/rules-configmap.yaml

@@ -1,5 +1,4 @@
-{{- if .Values.local.metrics.enabled }}
+{{- if or (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled) .Values.dashboards.traces.enabled }}
-{{- if and .Values.local.grafana.enabled (or .Values.dashboards.logs.enabled .Values.dashboards.metrics.enabled .Values.dashboards.traces.enabled) }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -17,4 +16,3 @@ data:
 {{ ($.Files.Glob "src/rules/tempo-rules.yaml").AsConfig | indent 2 }}
 {{- end }}
 {{- end }}
-{{- end }}

charts/meta-monitoring/templates/validate.yaml

@@ -3,20 +3,20 @@
 {{- end -}}
 
 {{- if eq .Values.cloud.logs.enabled true -}}
-  {{- if empty .Values.cloud.logs.secret -}}
+  {{- if or (empty .Values.cloud.logs.endpoint) (or (empty .Values.cloud.logs.username) (empty .Values.cloud.logs.password)) -}}
-    {{- fail "if cloud.logs is enabled then the secret has to be filled in" -}}
+    {{- fail "if cloud.logs is enabled then the endpoint, username and password have to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
 {{- if eq .Values.cloud.metrics.enabled true -}}
-  {{- if empty .Values.cloud.metrics.secret -}}
+  {{- if or (empty .Values.cloud.metrics.endpoint) (or (empty .Values.cloud.metrics.username) (empty .Values.cloud.metrics.password)) -}}
-    {{- fail "if cloud.metrics is enabled then the secret has to be filled in" -}}
+    {{- fail "if cloud.metrics is enabled then the endpoint, username and password have to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
 {{- if eq .Values.cloud.traces.enabled true -}}
-  {{- if empty .Values.cloud.traces.secret -}}
+  {{- if or (empty .Values.cloud.traces.endpoint) (or (empty .Values.cloud.traces.username) (empty .Values.cloud.traces.password)) -}}
-    {{- fail "if cloud.traces is enabled then the secret has to be filled in" -}}
+    {{- fail "if cloud.traces is enabled then the endpoint, username and password have to be filled in" -}}
   {{- end -}}
 {{- end -}}
 
@@ -37,7 +37,3 @@
 {{- if empty .Values.namespacesToMonitor -}}
   {{- fail "No namespaces have been specified in namespacesToMonitor" -}}
 {{- end -}}
-
-{{- if empty .Values.metrics.retain -}}
-  {{- fail "All metrics will be collected, please specify some in metrics.retain" -}}
-{{- end -}}

charts/meta-monitoring/values.yaml

@@ -1,178 +1,49 @@
 # Specify the namespaces to monitor here
 namespacesToMonitor:
 - loki
+- mimir
+- tempo
 # The name of the cluster where this will be installed
-clusterLabelValue: "meta-monitoring"
+clusterName: "meta-monitoring"
-
-# Set to true to write logs, metrics or traces to Grafana Cloud
-# The secrets have to be created first
-cloud:
-  logs:
-    enabled: true
-    secret: "logs"
-  metrics:
-    enabled: true
-    secret: "metrics"
-  traces:
-    enabled: true
-    secret: "traces"
 
 # Set to true for a local version of logs, metrics or traces
 local:
-  grafana:
+  logs:
-    enabled: false
+    enabled: true
+  metrics:
+    enabled: true
+  traces:
+    enabled: true
+  minio:
+    enabled: true  # This should be set to true if any of the previous is enabled
+
+# Set to true to write logs, metrics or traces to Grafana Cloud
+cloud:
   logs:
     enabled: false
+    endpoint:
+    username:
+    password:
   metrics:
     enabled: false
+    endpoint:
+    username:
+    password:
   traces:
     enabled: false
-  minio:
+    endpoint:
-    enabled: false  # This should be set to true if any of the previous is enabled
+    username:
+    password:
 
-grafana:
-  # Gateway ingress configuration
-  ingress:
-    # -- Specifies whether an ingress for the gateway should be created
-    enabled: true
-    # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
-    ingressClassName: ""
-    # -- Annotations for the gateway ingress
-    annotations: { }
-    # -- Labels for the gateway ingress
-    labels: { }
-    # -- Hosts configuration for the gateway ingress, passed through the `tpl` function to allow templating
-    hosts:
-      - host: monitoring.example.com
-        paths:
-          - path: /
-            # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
-            # pathType: Prefix
-    # -- TLS configuration for the gateway ingress. Hosts passed through the `tpl` function to allow templating
-    #tls:
-    #  - secretName: grafana-tls
-    #    hosts:
-    #      - monitoring.example.com
-
-
-logs:
 # Adding regexes here will add a stage.replace block for logs. For more information see
 # https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
+logs:
   piiRegexes:
   # This example replaces the word after password with *****
   # - expression: "password (\\\\S+)"
   #   source: ""         # Empty uses the log message
   #   replace: "*****""
 
-  # The lines matching these will be kept in Loki
-  retain:
-  # This shows the queries
-  - caller=metrics.go
-  # This shows any errors
-  - level=error
-  # This shows the ingest requests and is very noisy. Uncomment to include.
-  # - caller=push.go
-  # Log lines for delete requests
-  - delete request for user added
-  - Started processing delete request
-  - delete request for user marked as processed
-
-metrics:
-  # The list of metrics to retain for logging dashboards
-  retain:
-  - agent_config_last_load_success_timestamp_seconds
-  - agent_config_last_load_successful
-  - agent_config_load_failures_total
-  - container_cpu_usage_seconds_total
-  - container_fs_writes_bytes_total
-  - container_memory_working_set_bytes
-  - container_network_receive_bytes_total
-  - container_network_transmit_bytes_total
-  - container_spec_cpu_period
-  - container_spec_cpu_quota
-  - container_spec_memory_limit_bytes
-  - cortex_ingester_flush_queue_length
-  - cortex_prometheus_rule_group_iterations_total
-  - cortex_prometheus_rule_evaluation_failures_total
-  - cortex_prometheus_rule_group_rules
-  - cortex_prometheus_rule_group_last_duration_seconds
-  - cortex_prometheus_rule_group_last_evaluation_timestamp_seconds
-  - cortex_prometheus_rule_group_iterations_missed_total
-  - go_gc_duration_seconds
-  - go_goroutines
-  - go_memstats_heap_inuse_bytes
-  - kubelet_volume_stats_used_bytes
-  - kubelet_volume_stats_capacity_bytes
-  - kube_persistentvolumeclaim_labels
-  - kube_pod_container_resource_requests
-  - kube_pod_container_status_last_terminated_reason
-  - kube_pod_container_status_restarts_total
-  - loki_boltdb_shipper_compact_tables_operation_duration_seconds
-  - loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds
-  - loki_boltdb_shipper_retention_marker_count_total
-  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket
-  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count
-  - loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum
-  - loki_boltdb_shipper_retention_marker_table_processed_total
-  - loki_boltdb_shipper_request_duration_seconds_bucket
-  - loki_boltdb_shipper_request_duration_seconds_count
-  - loki_boltdb_shipper_request_duration_seconds_sum
-  - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket
-  - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count
-  - loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum
-  - loki_boltdb_shipper_retention_sweeper_marker_files_current
-  - loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time
-  - loki_build_info
-  - loki_chunk_store_deduped_chunks_total
-  - loki_chunk_store_index_entries_per_chunk_bucket
-  - loki_chunk_store_index_entries_per_chunk_count
-  - loki_chunk_store_index_entries_per_chunk_sum
-  - loki_compactor_delete_requests_processed_total
-  - loki_compactor_delete_requests_received_total
-  - loki_compactor_deleted_lines
-  - loki_compactor_oldest_pending_delete_request_age_seconds
-  - loki_compactor_pending_delete_requests_count
-  - loki_discarded_samples_total
-  - loki_distributor_bytes_received_total
-  - loki_distributor_lines_received_total
-  - loki_distributor_structured_metadata_bytes_received_total
-  - loki_ingester_chunk_age_seconds_bucket
-  - loki_ingester_chunk_age_seconds_count
-  - loki_ingester_chunk_age_seconds_sum
-  - loki_ingester_chunk_bounds_hours_bucket
-  - loki_ingester_chunk_bounds_hours_count
-  - loki_ingester_chunk_bounds_hours_sum
-  - loki_ingester_chunk_entries_bucket
-  - loki_ingester_chunk_entries_count
-  - loki_ingester_chunk_entries_sum
-  - loki_ingester_chunk_size_bytes_bucket
-  - loki_ingester_chunk_utilization_bucket
-  - loki_ingester_chunk_utilization_sum
-  - loki_ingester_chunks_flushed_total
-  - loki_ingester_flush_queue_length
-  - loki_ingester_memory_chunks
-  - loki_ingester_memory_streams
-  - loki_ingester_streams_created_total
-  - loki_request_duration_seconds_bucket
-  - loki_request_duration_seconds_count
-  - loki_request_duration_seconds_sum
-  - loki_ruler_wal_appender_ready
-  - loki_ruler_wal_disk_size
-  - loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds
-  - loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds
-  - loki_ruler_wal_prometheus_remote_storage_samples_pending
-  - loki_ruler_wal_prometheus_remote_storage_samples_total
-  - loki_ruler_wal_samples_appended_total
-  - loki_ruler_wal_storage_created_series_total
-  - loki_write_batch_retries_total
-  - loki_write_dropped_bytes_total
-  - loki_write_dropped_entries_total
-  - loki_write_sent_bytes_total
-  - loki_write_sent_entries_total
-  - node_disk_read_bytes_total
-  - node_disk_written_bytes_total
-  - promtail_custom_bad_words_total
-
 # Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
 dashboards:
   logs:
@@ -201,15 +72,6 @@ kubeStateMetrics:
 loki:
   loki:
     auth_enabled: false
-    schemaConfig:
-      configs:
-        - from: 2024-03-29
-          store: tsdb
-          object_store: s3
-          schema: v13
-          index:
-            prefix: index_
-            period: 24h
     storage:
       type: "s3"
       s3:
@@ -227,13 +89,8 @@ loki:
             secret_access_key: "{{ .Values.global.minio.rootPassword }}"
       compactor:
         retention_enabled: true
-        delete_request_store: s3
       limits_config:
-        retention_period: 30d
+        retention_period: 24h
-  lokiCanary:
-    enabled: false
-  test:
-    enabled: false
   monitoring:
     dashboards:
       enabled: false
@@ -250,28 +107,12 @@ loki:
   test:
     enabled: false
 
-alloy:
+grafana-agent:
-  alloy:
+  agent:
-    clustering:
-      enabled: true
     configMap:
       create: false
       name: "agent-configmap"
       key: 'config.river'
-    resources:
-      requests:
-        cpu: '1000m'
-        memory: '600Mi'
-      limits:
-        memory: '4Gi'
-  controller:
-    type: "statefulset"
-    autoscaling:
-      enabled: true
-      minReplicas: 3
-      maxReplicas: 30
-      targetMemoryUtilizationPercentage: 90
-      targetCPUUtilizationPercentage: 90
 
 mimir-distributed:
   minio:
@@ -301,7 +142,7 @@ mimir-distributed:
           secret_access_key: "{{ .Values.global.minio.rootPassword }}"
           insecure: true
       limits:
-        compactor_blocks_retention_period: 30d
+        compactor_blocks_retention_period: 24h
 
 tempo-distributed:
   tempo:
@@ -317,7 +158,7 @@ tempo-distributed:
             insecure: true
       compactor:
         compaction:
-          block_retention: 30d
+          block_retention: 24h
   traces:
     otlp:
       http:

docs/create_new_release.md

@@ -1,10 +0,0 @@
-# Create a new release
-
-1. Update the version field in charts/meta-monitoring/Chart.yaml in a new PR. Merge this PR if approved.
-
-2. On the [Actions tab](https://github.com/grafana/meta-monitoring-chart/actions):
-   - Select `Release Helm chart` in the workflows on the left
-   - Click the `Run workflow` button
-   - Leave the `main` branch as is
-   - Click the green `Run workflow` button
-

docs/installation.md

@@ -1,124 +1,23 @@
 # Install this chart
 
-## Preparation for Cloud mode (preferred)
-
-1. Use an existing Grafana Cloud account or setup a new one. Then create an access token:
-
-   1. In Grafana go to Administration -> Users and Access -> Cloud access policies.
-
-   1. Click `Create access policy`.
-
-   1. Fill in the `Display name` field and check the `Write` check box for metrics, logs and traces. Then click `Create`.
-
-   1. On the newly created access policy click `Add token`.
-
-   1. Fill in the `Token name` field and click `Create`. Make a copy of the token as it will be used later on.
-
 1. Create the meta namespace
 
    ```
    kubectl create namespace meta
    ```
 
-1. Create secrets with credentials and the endpoint when sending logs, metrics or traces to Grafana Cloud.
+1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml).
-
-   ```
-   kubectl create secret generic logs -n meta \
-    --from-literal=username=<logs username> \
-    --from-literal=password=<token>
-    --from-literal=endpoint='https://logs-prod-us-central1.grafana.net/loki/api/v1/push'
-
-   kubectl create secret generic metrics -n meta \
-    --from-literal=username=<metrics username> \
-    --from-literal=password=<token>
-    --from-literal=endpoint='https://prometheus-us-central1.grafana.net/api/prom/push'
-
-   kubectl create secret generic traces -n meta \
-    --from-literal=username=<traces username> \
-    --from-literal=password=<token>
-    --from-literal=endpoint='https://tempo-us-central1.grafana.net/tempo'
-   ```
-
-   The logs, metrics and traces usernames are the `User / Username / Instance IDs` of the Loki, Prometheus/Mimir and Tempo instances in Grafana Cloud. From `Home` in Grafana click on `Stacks`. Then go to the `Details` pages of Loki, Prometheus/Mimir and Tempo.
-
-1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml). Fill in the names of the secrets created above as needed. An example minimal values.yaml looks like this:
-
-   ```
-   namespacesToMonitor:
-   - loki
-
-   cloud:
-     logs:
-       enabled: true
-       secret: "logs"
-     metrics:
-       enabled: true
-       secret: "metrics"
-     traces:
-       enabled: true
-       secret: "traces"
-   ```
-
-## Preparation for Local mode
-
-1. Create the meta namespace
-
-   ```
-   kubectl create namespace meta
-   ```
-
-1. Create a values.yaml file based on the [default one](../charts/meta-monitoring/values.yaml). An example minimal values.yaml looks like this:
-
-   ```
-   namespacesToMonitor:
-   - loki
-
-   cloud:
-     logs:
-       enabled: false
-     metrics:
-       enabled: false
-     traces:
-       enabled: false
-
-   local:
-     grafana:
-       enabled:true
-     logs:
-       enabled: true
-     metrics:
-       enabled: true
-     traces:
-       enabled: true
-     minio:
-       enabled: true
-   ```
-
-## Installing the chart
-
-1. Add the repo
-
-   ```
-   helm repo add grafana https://grafana.github.io/helm-charts
-   ```
-
-1. Fetch the latest charts from the grafana repo
-
-   ```
-   helm repo update grafana
-   ```
-
 
 1. Install this helm chart
 
    ```
-   helm install -n meta -f values.yaml meta grafana/meta-monitoring
+   helm install -n meta -f values.yaml meta ./charts/meta-monitoring
    ```
 
 1. Upgrade
 
    ```
-   helm upgrade --install -f values.yaml -n meta meta grafana/meta-monitoring
+   helm upgrade --install -f values.yaml -n meta meta ./charts/meta-monitoring
    ```
 
 1. Delete this chart:
@@ -126,41 +25,3 @@
    ```
    helm delete -n meta meta
    ```
-
-## Installing the dashboards and rules on Grafana Cloud
-
-## Installing the dashboards on Grafana Cloud
-
-For each of the dashboard files in charts/meta-monitoring/src/dashboards do the following:
-
-1. Click on 'Dashboards' in Grafana
-
-1. Click on the 'New` button and select 'Import'
-
-1. Drop the dashboard file to the 'Upload dashboard JSON file' drop area
-
-1. Click 'Import'
-
-Only the files for the application monitored have to be copied. When monitoring Loki import dashboard files starting with 'loki-'.
-
-## Installing the rules on Grafana Cloud
-
-1. Select the rules files in charts/meta-monitoring/src/rules for the application to monitor. When monitoring Loki use loki-rules.yaml.
-
-1. Install mimirtool as per the [instructions](https://grafana.com/docs/mimir/latest/manage/tools/mimirtool/)
-
-1. Create an access policy with Read and Write permission for Rules. Also create a token and record the token.
-
-1. Get your cloud Prometheus endpoint and Instance ID from the `Prometheus` page in `Stacks`.
-
-1. Use them to load the rules using mimirtool as follows:
-
-   ```
-  mimirtool rules load --address=<your_cloud_prometheus_endpoint> --id=<your_instance_id> --key=<your_cloud_access_policy_token> *.yaml
-  ```
-
-1. To check the rules you have uploaded run:
-
-   ```
-  mimirtool rules print --address=<your_cloud_prometheus_endpoint> --id=<your_instance_id> --key=<your_cloud_access_policy_token>
-  ```

tools/kind.config

@@ -1,9 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-name: meta
-nodes:
-- role: control-plane
-- role: worker
-- role: worker
-- role: worker
-