Add parquet dashboards and alerts

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## master
 * [CHANGE] Use k8s-libsonnet 1.36. k8s-libsonnet removed 1.30, which broke the README instructions and `make test-readme`
 * [ENHANCEMENT] Add bigger tenants and configure default compactor tenant shards
+* [ENHANCEMENT] Add `Cortex / Parquet` and `Cortex / Parquet Converter Resources` dashboards
+* [ENHANCEMENT] Add parquet converter and parquet queryable alerts
 
 ## 1.17.1 / 2024-10-23
 * [CHANGE] Use cortex v1.17.1

cortex-mixin/alerts.libsonnet

@@ -6,7 +6,8 @@
     (if std.member($._config.storage_engine, 'blocks')
      then
        (import 'alerts/blocks.libsonnet') +
-       (import 'alerts/compactor.libsonnet')
+       (import 'alerts/compactor.libsonnet') +
+       (import 'alerts/parquet.libsonnet')
      else {}) +
 
     { _config:: $._config + $._group_config },

cortex-mixin/alerts/parquet.libsonnet

@@ -0,0 +1,136 @@
+{
+  groups+: [
+    {
+      name: 'cortex_parquet_alerts',
+      rules: [
+        {
+          // Alert if the parquet converter fails to convert the blocks of a tenant and
+          // converts none of them.
+          alert: 'CortexParquetConverterFailingConversions',
+          'for': '1h',
+          expr: |||
+            sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_block_convert_failures_total[1h])) > 0
+            unless
+            sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[1h])) > 0
+          ||| % $._config,
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              Cortex Parquet Converter in %(alert_aggregation_variables)s fails to convert blocks of tenant {{ $labels.user }} and converted none in the last hour.
+            ||| % $._config,
+          },
+        },
+        {
+          // Alert if a tenant has blocks that are due for conversion, but the parquet
+          // converter has not converted any of its blocks in the last 3h.
+          alert: 'CortexParquetConverterNotConverting',
+          'for': '3h',
+          expr: |||
+            max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count) > 0
+            unless
+            sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[3h])) > 0
+          ||| % $._config,
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} blocks that are due for parquet conversion, and the Cortex Parquet Converter converted none in the last 3 hours.
+            ||| % $._config,
+          },
+        },
+        {
+          // Alert if more than 20% of the conversions finish more than 5h (the top bucket
+          // of the histogram) after the TSDB block was uploaded.
+          alert: 'CortexParquetConversionDelayHigh',
+          'for': '1h',
+          expr: |||
+            (
+              1 - (
+                sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_bucket{le="300"}[3h]))
+                /
+                sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_count[3h]))
+              )
+            ) > 0.2
+          ||| % $._config,
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              More than 20%% of the parquet conversions in %(alert_aggregation_variables)s finish more than 5 hours after the TSDB block was uploaded.
+            ||| % $._config,
+          },
+        },
+        {
+          // Alert if the parquet converter ring has unhealthy members. The blocks of the
+          // tenants that an unhealthy member owns are not converted.
+          alert: 'CortexParquetConverterRingUnhealthy',
+          'for': '15m',
+          expr: |||
+            max by(%(alert_aggregation_labels)s) (cortex_ring_members{name="parquet-converter", state="Unhealthy"}) > 0
+          ||| % $._config,
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              The Cortex Parquet Converter ring in %(alert_aggregation_variables)s has {{ $value }} unhealthy members.
+            ||| % $._config,
+          },
+        },
+        {
+          // Alert if most of the blocks that the parquet queryable finds are TSDB blocks,
+          // which are read from the store-gateways in place of the parquet files.
+          alert: 'CortexParquetQueryableMostlyTSDB',
+          'for': '2h',
+          expr: |||
+            (
+              sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total{type="tsdb"}[1h]))
+              /
+              sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total[1h]))
+            ) > 0.7
+          ||| % $._config,
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              More than 70%% of the blocks queried through the parquet queryable in %(alert_aggregation_variables)s are TSDB blocks.
+            ||| % $._config,
+          },
+        },
+        {
+          // Alert if most of the blocks of a tenant are neither parquet blocks nor due for
+          // conversion. The converter only takes blocks wider than the first compactor block
+          // range, so this usually means that the compactor cannot compact the tenant past
+          // the first range and the parquet converter has nothing to convert.
+          alert: 'CortexParquetCompactionStuckAtFirstRange',
+          'for': '6h',
+          expr: |||
+            (
+              (
+                max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count)
+                - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_blocks_count)
+                - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count)
+              )
+              / max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count)
+            ) > 0.8
+            and
+            max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count) > 500
+          ||| % $._config,
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              {{ $value | humanizePercentage }} of the blocks of tenant {{ $labels.user }} in %(alert_aggregation_variables)s are not compacted past the first block range, so the Cortex Parquet Converter has nothing to convert.
+            ||| % $._config,
+          },
+        },
+      ],
+    },
+  ],
+}

cortex-mixin/config.libsonnet

@@ -25,6 +25,7 @@
       store_gateway: '(store-gateway|cortex$)',
       gateway: '(gateway|cortex-gw|cortex-gw-internal)',
       compactor: 'compactor.*',  // Match also custom compactor deployments.
+      parquet_converter: 'parquet-converter.*',  // Match also custom parquet-converter deployments.
       alertmanager: 'alertmanager',
     },
 
@@ -43,6 +44,7 @@
     // Name selectors for different application instances, using the "per_instance_label".
     instance_names: {
       compactor: 'compactor.*',
+      parquet_converter: 'parquet-converter.*',
       alertmanager: 'alertmanager.*',
     },
 

cortex-mixin/dashboards.libsonnet

@@ -11,6 +11,8 @@
     (import 'dashboards/rollout-progress.libsonnet') +
     (import 'dashboards/compactor.libsonnet') +
     (import 'dashboards/compactor-resources.libsonnet') +
+    (import 'dashboards/parquet.libsonnet') +
+    (import 'dashboards/parquet-converter-resources.libsonnet') +
     (import 'dashboards/object-store.libsonnet') +
 
     (if !$._config.resources_dashboards_enabled then {} else

cortex-mixin/dashboards/parquet-converter-resources.libsonnet

@@ -0,0 +1,49 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+  'cortex-parquet-converter-resources.json':
+    ($.dashboard('Cortex / Parquet Converter Resources') + { uid: 'c603a418fc4b1f77e6f13386b24f64f3' })
+    .addClusterSelectorTemplates()
+    .addRow(
+      $.row('CPU and Memory')
+      .addPanel(
+        $.containerCPUUsagePanel('CPU', 'parquet-converter'),
+      )
+      .addPanel(
+        $.containerMemoryWorkingSetPanel('Memory (workingset)', 'parquet-converter'),
+      )
+      .addPanel(
+        $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.parquet_converter),
+      )
+    )
+    .addRow(
+      $.row('Network')
+      .addPanel(
+        $.containerNetworkReceiveBytesPanel($._config.instance_names.parquet_converter),
+      )
+      .addPanel(
+        $.containerNetworkTransmitBytesPanel($._config.instance_names.parquet_converter),
+      )
+    )
+    .addRow(
+      $.row('Disk')
+      .addPanel(
+        $.containerDiskWritesPanel('Disk Writes', 'parquet-converter'),
+      )
+      .addPanel(
+        $.containerDiskReadsPanel('Disk Reads', 'parquet-converter'),
+      )
+      .addPanel(
+        $.containerDiskSpaceUtilization('Disk Space Utilization', 'parquet-converter'),
+      )
+    ) + {
+      templating+: {
+        list: [
+          // Do not allow to include all clusters/namespaces otherwise this dashboard
+          // risks to explode because it shows resources per pod.
+          l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {})
+          for l in super.list
+        ],
+      },
+    },
+}

cortex-mixin/dashboards/parquet.libsonnet

@@ -0,0 +1,135 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+  // Shows the parquet pipeline end to end: the parquet converter turns TSDB blocks into
+  // parquet files, the compactor reports how many blocks are converted, and the queriers
+  // and rulers read the parquet files through the parquet queryable.
+  local queryableRows(title, jobName) = [
+    $.row(title)
+    .addPanel(
+      $.timeseriesPanel('Blocks queried / sec', unit='ops') +
+      $.queryPanel('sum by(type) (rate(cortex_parquet_queryable_blocks_queried_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{type}}') +
+      $.stack +
+      $.panelDescription(
+        'Blocks queried / sec',
+        |||
+          Rate of blocks found to query, by the format that served them.
+          <tt>tsdb</tt> blocks have no parquet files yet and are read from the store-gateways.
+        |||
+      ),
+    )
+    .addPanel(
+      $.timeseriesPanel('Operations / sec', unit='ops') +
+      $.queryPanel('sum by(type, method) (rate(cortex_parquet_queryable_operations_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{method}} {{type}}') +
+      $.stack +
+      $.panelDescription(
+        'Operations / sec',
+        |||
+          Rate of Select, LabelNames and LabelValues operations, by the blocks they read.
+          <tt>parquet</tt> reads only parquet blocks, <tt>tsdb</tt> only TSDB blocks, and <tt>mixed</tt> both.
+        |||
+      ),
+    )
+    .addPanel(
+      $.timeseriesPanel('Shard cache hit ratio', unit='percentunit') +
+      $.queryPanel(
+        |||
+          sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval]))
+          /
+          (
+            sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval]))
+            +
+            sum(rate(cortex_parquet_cache_misses_total{%s}[$__rate_interval]))
+          )
+        ||| % [$.jobMatcher(jobName), $.jobMatcher(jobName), $.jobMatcher(jobName)],
+        'hit ratio'
+      ) +
+      { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
+    )
+    .addPanel(
+      $.timeseriesPanel('Shard cache items') +
+      $.queryPanel('sum by(%s) (cortex_parquet_cache_item_count{%s})' % [$._config.per_instance_label, $.jobMatcher(jobName)], '{{%s}}' % $._config.per_instance_label),
+    ),
+  ],
+
+  'cortex-parquet.json':
+    ($.dashboard('Cortex / Parquet') + { uid: 'dfef97d2b024dfd26ee9968cc0a095b9' })
+    .addClusterSelectorTemplates()
+    .addRow(
+      $.row('Converter')
+      .addPanel(
+        $.successFailurePanel(
+          'Block conversions / sec',
+          'sum(rate(cortex_parquet_converter_blocks_converted_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter),
+          'sum(rate(cortex_parquet_converter_block_convert_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter),
+        ) +
+        { yaxes: $.yaxes('ops') } +
+        $.panelDescription(
+          'Block conversions / sec',
+          |||
+            Rate of TSDB blocks converted to parquet, and of conversions that failed.
+          |||
+        ),
+      )
+      .addPanel(
+        $.timeseriesPanel('Tenants owned') +
+        $.queryPanel('sum by(%s) (cortex_parquet_converter_users_owned{%s})' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.parquet_converter)], '{{%s}}' % $._config.per_instance_label) +
+        $.panelDescription(
+          'Tenants owned',
+          |||
+            Number of tenants that each parquet converter instance owns in the ring.
+          |||
+        ),
+      )
+    )
+    .addRow(
+      $.row('')
+      .addPanel(
+        $.timeseriesPanel('Conversion delay', unit='m') +
+        $.latencyPanel('cortex_parquet_converter_convert_block_delay_minutes', '{%s}' % $.jobMatcher($._config.job_names.parquet_converter), multiplier='1') +
+        $.panelDescription(
+          'Conversion delay',
+          |||
+            Time from the upload of a TSDB block to object storage until its parquet files are written.
+          |||
+        ),
+      )
+      .addPanel(
+        $.timeseriesPanel('Latest block conversion duration', unit='s') +
+        $.queryPanel('topk(10, max by(user) (cortex_parquet_converter_convert_block_duration_seconds{%s}))' % $.jobMatcher($._config.job_names.parquet_converter), '{{user}}') +
+        $.panelDescription(
+          'Latest block conversion duration',
+          |||
+            Time that the latest block conversion took, for the 10 tenants with the slowest one.
+          |||
+        ),
+      )
+    )
+    .addRow(
+      $.row('Bucket')
+      .addPanel(
+        $.timeseriesPanel('Tenants with largest number of unconverted blocks') +
+        $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_unconverted_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') +
+        $.panelDescription(
+          'Tenants with largest number of unconverted blocks',
+          |||
+            The 10 tenants with the most blocks that are due for conversion but have no parquet files yet.
+            The compactor reports this value, so it only updates when the compactor cleans the tenant's bucket.
+          |||
+        ),
+      )
+      .addPanel(
+        $.timeseriesPanel('Tenants with largest number of parquet blocks') +
+        $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') +
+        $.panelDescription(
+          'Tenants with largest number of parquet blocks',
+          |||
+            The 10 tenants with the most parquet blocks in the bucket.
+          |||
+        ),
+      )
+    )
+    .addRows(queryableRows('Querier', $._config.job_names.querier))
+    .addRows(queryableRows('Ruler', $._config.job_names.ruler))
+    .addRows($.getObjectStoreRows('Object Store', 'parquet-converter')),
+}