diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7cb776b..2011200 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
## master
* [CHANGE] Use k8s-libsonnet 1.36. k8s-libsonnet removed 1.30, which broke the README instructions and `make test-readme`
* [ENHANCEMENT] Add bigger tenants and configure default compactor tenant shards
+* [ENHANCEMENT] Add `Cortex / Parquet` and `Cortex / Parquet Converter Resources` dashboards
+* [ENHANCEMENT] Add parquet converter and parquet queryable alerts
## 1.17.1 / 2024-10-23
* [CHANGE] Use cortex v1.17.1
diff --git a/cortex-mixin/alerts.libsonnet b/cortex-mixin/alerts.libsonnet
index 4dc1f85..40f11ee 100644
--- a/cortex-mixin/alerts.libsonnet
+++ b/cortex-mixin/alerts.libsonnet
@@ -6,7 +6,8 @@
(if std.member($._config.storage_engine, 'blocks')
then
(import 'alerts/blocks.libsonnet') +
- (import 'alerts/compactor.libsonnet')
+ (import 'alerts/compactor.libsonnet') +
+ (import 'alerts/parquet.libsonnet')
else {}) +
{ _config:: $._config + $._group_config },
diff --git a/cortex-mixin/alerts/parquet.libsonnet b/cortex-mixin/alerts/parquet.libsonnet
new file mode 100644
index 0000000..ec8a537
--- /dev/null
+++ b/cortex-mixin/alerts/parquet.libsonnet
@@ -0,0 +1,136 @@
+{
+ groups+: [
+ {
+ name: 'cortex_parquet_alerts',
+ rules: [
+ {
+ // Alert if the parquet converter fails to convert the blocks of a tenant and
+ // converts none of them.
+ alert: 'CortexParquetConverterFailingConversions',
+ 'for': '1h',
+ expr: |||
+ sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_block_convert_failures_total[1h])) > 0
+ unless
+ sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[1h])) > 0
+ ||| % $._config,
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ message: |||
+ Cortex Parquet Converter in %(alert_aggregation_variables)s fails to convert blocks of tenant {{ $labels.user }} and converted none in the last hour.
+ ||| % $._config,
+ },
+ },
+ {
+ // Alert if a tenant has blocks that are due for conversion, but the parquet
+ // converter has not converted any of its blocks in the last 3h.
+ alert: 'CortexParquetConverterNotConverting',
+ 'for': '3h',
+ expr: |||
+ max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count) > 0
+ unless
+ sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[3h])) > 0
+ ||| % $._config,
+ labels: {
+ severity: 'critical',
+ },
+ annotations: {
+ message: |||
+ Tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} blocks that are due for parquet conversion, and the Cortex Parquet Converter converted none in the last 3 hours.
+ ||| % $._config,
+ },
+ },
+ {
+ // Alert if more than 20% of the conversions finish more than 5h (the top bucket
+ // of the histogram) after the TSDB block was uploaded.
+ alert: 'CortexParquetConversionDelayHigh',
+ 'for': '1h',
+ expr: |||
+ (
+ 1 - (
+ sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_bucket{le="300"}[3h]))
+ /
+ sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_count[3h]))
+ )
+ ) > 0.2
+ ||| % $._config,
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ message: |||
+ More than 20%% of the parquet conversions in %(alert_aggregation_variables)s finish more than 5 hours after the TSDB block was uploaded.
+ ||| % $._config,
+ },
+ },
+ {
+ // Alert if the parquet converter ring has unhealthy members. The blocks of the
+ // tenants that an unhealthy member owns are not converted.
+ alert: 'CortexParquetConverterRingUnhealthy',
+ 'for': '15m',
+ expr: |||
+ max by(%(alert_aggregation_labels)s) (cortex_ring_members{name="parquet-converter", state="Unhealthy"}) > 0
+ ||| % $._config,
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ message: |||
+ The Cortex Parquet Converter ring in %(alert_aggregation_variables)s has {{ $value }} unhealthy members.
+ ||| % $._config,
+ },
+ },
+ {
+ // Alert if most of the blocks that the parquet queryable finds are TSDB blocks,
+ // which are read from the store-gateways in place of the parquet files.
+ alert: 'CortexParquetQueryableMostlyTSDB',
+ 'for': '2h',
+ expr: |||
+ (
+ sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total{type="tsdb"}[1h]))
+ /
+ sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total[1h]))
+ ) > 0.7
+ ||| % $._config,
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ message: |||
+ More than 70%% of the blocks queried through the parquet queryable in %(alert_aggregation_variables)s are TSDB blocks.
+ ||| % $._config,
+ },
+ },
+ {
+ // Alert if most of the blocks of a tenant are neither parquet blocks nor due for
+ // conversion. The converter only takes blocks wider than the first compactor block
+ // range, so this usually means that the compactor cannot compact the tenant past
+ // the first range and the parquet converter has nothing to convert.
+ alert: 'CortexParquetCompactionStuckAtFirstRange',
+ 'for': '6h',
+ expr: |||
+ (
+ (
+ max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count)
+ - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_blocks_count)
+ - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count)
+ )
+ / max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count)
+ ) > 0.8
+ and
+ max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count) > 500
+ ||| % $._config,
+ labels: {
+ severity: 'critical',
+ },
+ annotations: {
+ message: |||
+ {{ $value | humanizePercentage }} of the blocks of tenant {{ $labels.user }} in %(alert_aggregation_variables)s are not compacted past the first block range, so the Cortex Parquet Converter has nothing to convert.
+ ||| % $._config,
+ },
+ },
+ ],
+ },
+ ],
+}
diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet
index 2f62070..fc9010d 100644
--- a/cortex-mixin/config.libsonnet
+++ b/cortex-mixin/config.libsonnet
@@ -25,6 +25,7 @@
store_gateway: '(store-gateway|cortex$)',
gateway: '(gateway|cortex-gw|cortex-gw-internal)',
compactor: 'compactor.*', // Match also custom compactor deployments.
+ parquet_converter: 'parquet-converter.*', // Match also custom parquet-converter deployments.
alertmanager: 'alertmanager',
},
@@ -43,6 +44,7 @@
// Name selectors for different application instances, using the "per_instance_label".
instance_names: {
compactor: 'compactor.*',
+ parquet_converter: 'parquet-converter.*',
alertmanager: 'alertmanager.*',
},
diff --git a/cortex-mixin/dashboards.libsonnet b/cortex-mixin/dashboards.libsonnet
index e4b68c4..5190033 100644
--- a/cortex-mixin/dashboards.libsonnet
+++ b/cortex-mixin/dashboards.libsonnet
@@ -11,6 +11,8 @@
(import 'dashboards/rollout-progress.libsonnet') +
(import 'dashboards/compactor.libsonnet') +
(import 'dashboards/compactor-resources.libsonnet') +
+ (import 'dashboards/parquet.libsonnet') +
+ (import 'dashboards/parquet-converter-resources.libsonnet') +
(import 'dashboards/object-store.libsonnet') +
(if !$._config.resources_dashboards_enabled then {} else
diff --git a/cortex-mixin/dashboards/parquet-converter-resources.libsonnet b/cortex-mixin/dashboards/parquet-converter-resources.libsonnet
new file mode 100644
index 0000000..0fde9a3
--- /dev/null
+++ b/cortex-mixin/dashboards/parquet-converter-resources.libsonnet
@@ -0,0 +1,49 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+ 'cortex-parquet-converter-resources.json':
+ ($.dashboard('Cortex / Parquet Converter Resources') + { uid: 'c603a418fc4b1f77e6f13386b24f64f3' })
+ .addClusterSelectorTemplates()
+ .addRow(
+ $.row('CPU and Memory')
+ .addPanel(
+ $.containerCPUUsagePanel('CPU', 'parquet-converter'),
+ )
+ .addPanel(
+ $.containerMemoryWorkingSetPanel('Memory (workingset)', 'parquet-converter'),
+ )
+ .addPanel(
+ $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.parquet_converter),
+ )
+ )
+ .addRow(
+ $.row('Network')
+ .addPanel(
+ $.containerNetworkReceiveBytesPanel($._config.instance_names.parquet_converter),
+ )
+ .addPanel(
+ $.containerNetworkTransmitBytesPanel($._config.instance_names.parquet_converter),
+ )
+ )
+ .addRow(
+ $.row('Disk')
+ .addPanel(
+ $.containerDiskWritesPanel('Disk Writes', 'parquet-converter'),
+ )
+ .addPanel(
+ $.containerDiskReadsPanel('Disk Reads', 'parquet-converter'),
+ )
+ .addPanel(
+ $.containerDiskSpaceUtilization('Disk Space Utilization', 'parquet-converter'),
+ )
+ ) + {
+ templating+: {
+ list: [
+ // Do not allow to include all clusters/namespaces otherwise this dashboard
+ // risks to explode because it shows resources per pod.
+ l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {})
+ for l in super.list
+ ],
+ },
+ },
+}
diff --git a/cortex-mixin/dashboards/parquet.libsonnet b/cortex-mixin/dashboards/parquet.libsonnet
new file mode 100644
index 0000000..69bac77
--- /dev/null
+++ b/cortex-mixin/dashboards/parquet.libsonnet
@@ -0,0 +1,135 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+ // Shows the parquet pipeline end to end: the parquet converter turns TSDB blocks into
+ // parquet files, the compactor reports how many blocks are converted, and the queriers
+ // and rulers read the parquet files through the parquet queryable.
+ local queryableRows(title, jobName) = [
+ $.row(title)
+ .addPanel(
+ $.timeseriesPanel('Blocks queried / sec', unit='ops') +
+ $.queryPanel('sum by(type) (rate(cortex_parquet_queryable_blocks_queried_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{type}}') +
+ $.stack +
+ $.panelDescription(
+ 'Blocks queried / sec',
+ |||
+ Rate of blocks found to query, by the format that served them.
+ tsdb blocks have no parquet files yet and are read from the store-gateways.
+ |||
+ ),
+ )
+ .addPanel(
+ $.timeseriesPanel('Operations / sec', unit='ops') +
+ $.queryPanel('sum by(type, method) (rate(cortex_parquet_queryable_operations_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{method}} {{type}}') +
+ $.stack +
+ $.panelDescription(
+ 'Operations / sec',
+ |||
+ Rate of Select, LabelNames and LabelValues operations, by the blocks they read.
+ parquet reads only parquet blocks, tsdb only TSDB blocks, and mixed both.
+ |||
+ ),
+ )
+ .addPanel(
+ $.timeseriesPanel('Shard cache hit ratio', unit='percentunit') +
+ $.queryPanel(
+ |||
+ sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval]))
+ /
+ (
+ sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval]))
+ +
+ sum(rate(cortex_parquet_cache_misses_total{%s}[$__rate_interval]))
+ )
+ ||| % [$.jobMatcher(jobName), $.jobMatcher(jobName), $.jobMatcher(jobName)],
+ 'hit ratio'
+ ) +
+ { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
+ )
+ .addPanel(
+ $.timeseriesPanel('Shard cache items') +
+ $.queryPanel('sum by(%s) (cortex_parquet_cache_item_count{%s})' % [$._config.per_instance_label, $.jobMatcher(jobName)], '{{%s}}' % $._config.per_instance_label),
+ ),
+ ],
+
+ 'cortex-parquet.json':
+ ($.dashboard('Cortex / Parquet') + { uid: 'dfef97d2b024dfd26ee9968cc0a095b9' })
+ .addClusterSelectorTemplates()
+ .addRow(
+ $.row('Converter')
+ .addPanel(
+ $.successFailurePanel(
+ 'Block conversions / sec',
+ 'sum(rate(cortex_parquet_converter_blocks_converted_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter),
+ 'sum(rate(cortex_parquet_converter_block_convert_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter),
+ ) +
+ { yaxes: $.yaxes('ops') } +
+ $.panelDescription(
+ 'Block conversions / sec',
+ |||
+ Rate of TSDB blocks converted to parquet, and of conversions that failed.
+ |||
+ ),
+ )
+ .addPanel(
+ $.timeseriesPanel('Tenants owned') +
+ $.queryPanel('sum by(%s) (cortex_parquet_converter_users_owned{%s})' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.parquet_converter)], '{{%s}}' % $._config.per_instance_label) +
+ $.panelDescription(
+ 'Tenants owned',
+ |||
+ Number of tenants that each parquet converter instance owns in the ring.
+ |||
+ ),
+ )
+ )
+ .addRow(
+ $.row('')
+ .addPanel(
+ $.timeseriesPanel('Conversion delay', unit='m') +
+ $.latencyPanel('cortex_parquet_converter_convert_block_delay_minutes', '{%s}' % $.jobMatcher($._config.job_names.parquet_converter), multiplier='1') +
+ $.panelDescription(
+ 'Conversion delay',
+ |||
+ Time from the upload of a TSDB block to object storage until its parquet files are written.
+ |||
+ ),
+ )
+ .addPanel(
+ $.timeseriesPanel('Latest block conversion duration', unit='s') +
+ $.queryPanel('topk(10, max by(user) (cortex_parquet_converter_convert_block_duration_seconds{%s}))' % $.jobMatcher($._config.job_names.parquet_converter), '{{user}}') +
+ $.panelDescription(
+ 'Latest block conversion duration',
+ |||
+ Time that the latest block conversion took, for the 10 tenants with the slowest one.
+ |||
+ ),
+ )
+ )
+ .addRow(
+ $.row('Bucket')
+ .addPanel(
+ $.timeseriesPanel('Tenants with largest number of unconverted blocks') +
+ $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_unconverted_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') +
+ $.panelDescription(
+ 'Tenants with largest number of unconverted blocks',
+ |||
+ The 10 tenants with the most blocks that are due for conversion but have no parquet files yet.
+ The compactor reports this value, so it only updates when the compactor cleans the tenant's bucket.
+ |||
+ ),
+ )
+ .addPanel(
+ $.timeseriesPanel('Tenants with largest number of parquet blocks') +
+ $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') +
+ $.panelDescription(
+ 'Tenants with largest number of parquet blocks',
+ |||
+ The 10 tenants with the most parquet blocks in the bucket.
+ |||
+ ),
+ )
+ )
+ .addRows(queryableRows('Querier', $._config.job_names.querier))
+ .addRows(queryableRows('Ruler', $._config.job_names.ruler))
+ .addRows($.getObjectStoreRows('Object Store', 'parquet-converter')),
+}
diff --git a/cortex-mixin/docs/playbooks.md b/cortex-mixin/docs/playbooks.md
index 3958687..9e7aa7a 100644
--- a/cortex-mixin/docs/playbooks.md
+++ b/cortex-mixin/docs/playbooks.md
@@ -462,6 +462,86 @@ How to **investigate**:
- Safely manually delete the block from the bucket if was a partial delete or an upload failed by a compactor
- Further investigate if was an upload failed by an ingester but not later retried (ingesters are expected to retry uploads until succeed)
+### CortexParquetConverterFailingConversions
+
+This alert fires when the parquet converter fails to convert blocks of a tenant and converts none of that tenant's blocks for 1 hour.
+
+How it **works**:
+- The parquet converter scans its tenants on every `-parquet-converter.conversion-interval` (default 1m). For each block that is wider than the first `-compactor.block-ranges` range and has no valid converter mark, it downloads the block, writes the parquet files and then writes a converter mark.
+- `cortex_parquet_converter_block_convert_failures_total` counts every failed conversion, except cancellations and permission errors.
+- A block that fails is tried again on the next scan, so one bad block can make the failure counter increase on every scan.
+
+How to **investigate**:
+- Look for errors in the parquet converter logs for the tenant in the alert.
+- Check the disk use of the parquet converter. It downloads the whole TSDB block to its local disk before it converts it, so a block that is larger than the free disk fails every time. The `Cortex / Parquet Converter Resources` dashboard shows the disk use.
+- Check the object store errors for `component="parquet-converter"` on the `Cortex / Parquet` dashboard.
+- If one block fails every time, for example because it is corrupted, see [`CortexCompactorHasNotSuccessfullyRunCompaction`](#cortexcompactorhasnotsuccessfullyruncompaction) for how to skip a corrupted block.
+
+### CortexParquetConverterNotConverting
+
+This alert fires when a tenant has blocks that are due for parquet conversion, but the parquet converter has converted none of that tenant's blocks for 3 hours.
+
+How it **works**:
+- The compactor sets `cortex_bucket_parquet_unconverted_blocks_count` when it cleans the tenant's bucket, every `-compactor.cleanup-interval` (default 15m). It counts the blocks that are wider than the first block range and have no parquet files. Blocks that are marked for deletion are included.
+- Each tenant is owned by a parquet converter instance in the parquet converter ring. If no healthy instance owns the tenant, its blocks are not converted.
+
+How to **investigate**:
+- Check that the parquet converter is running and that `cortex_parquet_converter_users_owned` is more than 0. If it is 0, check the parquet converter ring (`/parquet-converter/ring`) and [`CortexParquetConverterRingUnhealthy`](#cortexparquetconverterringunhealthy).
+- Check that parquet conversion is enabled for the tenant (`parquet_converter_enabled` in the limits).
+- Check the parquet converter logs and [`CortexParquetConverterFailingConversions`](#cortexparquetconverterfailingconversions).
+- One scan processes the tenants one after another. A tenant with many large blocks can make one scan take hours, so other tenants wait. Check `cortex_parquet_converter_convert_block_duration_seconds` for slow tenants.
+- If the blocks are marked for deletion, the converter does not convert them, and the alert stops when the compactor deletes them.
+
+### CortexParquetConversionDelayHigh
+
+This alert fires when more than 20% of the parquet conversions finish more than 5 hours after the TSDB block was uploaded to the object store.
+
+How it **works**:
+- `cortex_parquet_converter_convert_block_delay_minutes` measures the time from the upload of the block (the `meta.json` modification time) to the end of its conversion. Its top bucket is 300 minutes, so all delays above 5 hours are in the `+Inf` bucket.
+- The delay is only recorded when a conversion succeeds. If the converter stops converting, this alert stops firing. Use [`CortexParquetConverterNotConverting`](#cortexparquetconverternotconverting) for that case.
+- A long delay is expected for blocks that the compactor uploads hours after their time range, for example after a backfill or after compaction was blocked.
+
+How to **investigate**:
+- Check if the compactor is behind. A block can only be converted after the compactor has compacted it past the first block range.
+- Check if the parquet converter is busy with one large tenant, on the `Cortex / Parquet` dashboard.
+- Scale out the parquet converter, or give it more CPU and disk, if it cannot keep up.
+
+### CortexParquetConverterRingUnhealthy
+
+This alert fires when the parquet converter ring has unhealthy members for 15 minutes. The blocks of the tenants that an unhealthy member owns are not converted.
+
+How to **investigate**:
+- Open the parquet converter ring page (`/parquet-converter/ring`) and find the unhealthy members.
+- If a parquet converter pod was deleted without leaving the ring, remove its entry with the "Forget" button.
+- If the pod is running, check its logs for errors with the ring key-value store.
+
+### CortexParquetQueryableMostlyTSDB
+
+This alert fires when more than 70% of the blocks that the parquet queryable finds in the queriers and rulers are TSDB blocks, for 2 hours. TSDB blocks are read from the store-gateways in place of the parquet files.
+
+How it **works**:
+- Blocks that are not wider than the first block range are never converted, so queries for recent data always read TSDB blocks. A query load with many short-range queries can make this alert fire when parquet is working.
+
+How to **investigate**:
+- Check the `Cortex / Parquet` dashboard for tenants with many unconverted blocks.
+- Check [`CortexParquetConverterNotConverting`](#cortexparquetconverternotconverting) and [`CortexParquetCompactionStuckAtFirstRange`](#cortexparquetcompactionstuckatfirstrange).
+- Check the time range of the queries. If most queries read only recent data, the TSDB blocks are expected.
+
+### CortexParquetCompactionStuckAtFirstRange
+
+This alert fires when more than 80% of the blocks of a tenant with more than 500 blocks are neither parquet blocks nor due for conversion, for 6 hours.
+
+How it **works**:
+- The parquet converter only converts blocks that are wider than the first `-compactor.block-ranges` range. The blocks that the ingesters upload are not wider than that range, so they are converted only after the compactor compacts them.
+- If the compactor cannot compact a tenant, its blocks stay at the first range. Then `cortex_bucket_parquet_unconverted_blocks_count` is 0 and the parquet alerts do not fire, but almost nothing is converted.
+- The alert can also fire just after parquet conversion is enabled for a tenant with a long retention, until the compactor and the converter catch up.
+
+How to **investigate**:
+- Check `cortex_compactor_remaining_planned_compactions` for the tenant. A value that stays above 0 means the compactor cannot finish the planned compactions.
+- Check [`CortexCompactorHasNotSuccessfullyRunCompaction`](#cortexcompactorhasnotsuccessfullyruncompaction) and the compactor logs for the tenant.
+- Check the disk use of the compactors. The compactor downloads all source blocks of a compaction job to its local disk. For a tenant with many ingesters, one job at the first range can need hundreds of GiB. A failed job keeps its downloaded blocks on disk until a compaction run for any tenant on that compactor succeeds, so failures can fill the disk.
+- Check the memory use of the compactors for out-of-memory kills.
+
### CortexQueriesIncorrect
_TODO: this playbook has not been written yet._