diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cb776b..2011200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## master * [CHANGE] Use k8s-libsonnet 1.36. k8s-libsonnet removed 1.30, which broke the README instructions and `make test-readme` * [ENHANCEMENT] Add bigger tenants and configure default compactor tenant shards +* [ENHANCEMENT] Add `Cortex / Parquet` and `Cortex / Parquet Converter Resources` dashboards +* [ENHANCEMENT] Add parquet converter and parquet queryable alerts ## 1.17.1 / 2024-10-23 * [CHANGE] Use cortex v1.17.1 diff --git a/cortex-mixin/alerts.libsonnet b/cortex-mixin/alerts.libsonnet index 4dc1f85..40f11ee 100644 --- a/cortex-mixin/alerts.libsonnet +++ b/cortex-mixin/alerts.libsonnet @@ -6,7 +6,8 @@ (if std.member($._config.storage_engine, 'blocks') then (import 'alerts/blocks.libsonnet') + - (import 'alerts/compactor.libsonnet') + (import 'alerts/compactor.libsonnet') + + (import 'alerts/parquet.libsonnet') else {}) + { _config:: $._config + $._group_config }, diff --git a/cortex-mixin/alerts/parquet.libsonnet b/cortex-mixin/alerts/parquet.libsonnet new file mode 100644 index 0000000..ec8a537 --- /dev/null +++ b/cortex-mixin/alerts/parquet.libsonnet @@ -0,0 +1,136 @@ +{ + groups+: [ + { + name: 'cortex_parquet_alerts', + rules: [ + { + // Alert if the parquet converter fails to convert the blocks of a tenant and + // converts none of them. + alert: 'CortexParquetConverterFailingConversions', + 'for': '1h', + expr: ||| + sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_block_convert_failures_total[1h])) > 0 + unless + sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[1h])) > 0 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + Cortex Parquet Converter in %(alert_aggregation_variables)s fails to convert blocks of tenant {{ $labels.user }} and converted none in the last hour. + ||| % $._config, + }, + }, + { + // Alert if a tenant has blocks that are due for conversion, but the parquet + // converter has not converted any of its blocks in the last 3h. + alert: 'CortexParquetConverterNotConverting', + 'for': '3h', + expr: ||| + max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count) > 0 + unless + sum by(%(alert_aggregation_labels)s, user) (increase(cortex_parquet_converter_blocks_converted_total[3h])) > 0 + ||| % $._config, + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + Tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} blocks that are due for parquet conversion, and the Cortex Parquet Converter converted none in the last 3 hours. + ||| % $._config, + }, + }, + { + // Alert if more than 20% of the conversions finish more than 5h (the top bucket + // of the histogram) after the TSDB block was uploaded. + alert: 'CortexParquetConversionDelayHigh', + 'for': '1h', + expr: ||| + ( + 1 - ( + sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_bucket{le="300"}[3h])) + / + sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_converter_convert_block_delay_minutes_count[3h])) + ) + ) > 0.2 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + More than 20%% of the parquet conversions in %(alert_aggregation_variables)s finish more than 5 hours after the TSDB block was uploaded. + ||| % $._config, + }, + }, + { + // Alert if the parquet converter ring has unhealthy members. The blocks of the + // tenants that an unhealthy member owns are not converted. + alert: 'CortexParquetConverterRingUnhealthy', + 'for': '15m', + expr: ||| + max by(%(alert_aggregation_labels)s) (cortex_ring_members{name="parquet-converter", state="Unhealthy"}) > 0 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + The Cortex Parquet Converter ring in %(alert_aggregation_variables)s has {{ $value }} unhealthy members. + ||| % $._config, + }, + }, + { + // Alert if most of the blocks that the parquet queryable finds are TSDB blocks, + // which are read from the store-gateways in place of the parquet files. + alert: 'CortexParquetQueryableMostlyTSDB', + 'for': '2h', + expr: ||| + ( + sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total{type="tsdb"}[1h])) + / + sum by(%(alert_aggregation_labels)s) (rate(cortex_parquet_queryable_blocks_queried_total[1h])) + ) > 0.7 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + More than 70%% of the blocks queried through the parquet queryable in %(alert_aggregation_variables)s are TSDB blocks. + ||| % $._config, + }, + }, + { + // Alert if most of the blocks of a tenant are neither parquet blocks nor due for + // conversion. The converter only takes blocks wider than the first compactor block + // range, so this usually means that the compactor cannot compact the tenant past + // the first range and the parquet converter has nothing to convert. + alert: 'CortexParquetCompactionStuckAtFirstRange', + 'for': '6h', + expr: ||| + ( + ( + max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count) + - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_blocks_count) + - max by(%(alert_aggregation_labels)s, user) (cortex_bucket_parquet_unconverted_blocks_count) + ) + / max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count) + ) > 0.8 + and + max by(%(alert_aggregation_labels)s, user) (cortex_bucket_blocks_count) > 500 + ||| % $._config, + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + {{ $value | humanizePercentage }} of the blocks of tenant {{ $labels.user }} in %(alert_aggregation_variables)s are not compacted past the first block range, so the Cortex Parquet Converter has nothing to convert. + ||| % $._config, + }, + }, + ], + }, + ], +} diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index 2f62070..fc9010d 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -25,6 +25,7 @@ store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. + parquet_converter: 'parquet-converter.*', // Match also custom parquet-converter deployments. alertmanager: 'alertmanager', }, @@ -43,6 +44,7 @@ // Name selectors for different application instances, using the "per_instance_label". instance_names: { compactor: 'compactor.*', + parquet_converter: 'parquet-converter.*', alertmanager: 'alertmanager.*', }, diff --git a/cortex-mixin/dashboards.libsonnet b/cortex-mixin/dashboards.libsonnet index e4b68c4..5190033 100644 --- a/cortex-mixin/dashboards.libsonnet +++ b/cortex-mixin/dashboards.libsonnet @@ -11,6 +11,8 @@ (import 'dashboards/rollout-progress.libsonnet') + (import 'dashboards/compactor.libsonnet') + (import 'dashboards/compactor-resources.libsonnet') + + (import 'dashboards/parquet.libsonnet') + + (import 'dashboards/parquet-converter-resources.libsonnet') + (import 'dashboards/object-store.libsonnet') + (if !$._config.resources_dashboards_enabled then {} else diff --git a/cortex-mixin/dashboards/parquet-converter-resources.libsonnet b/cortex-mixin/dashboards/parquet-converter-resources.libsonnet new file mode 100644 index 0000000..0fde9a3 --- /dev/null +++ b/cortex-mixin/dashboards/parquet-converter-resources.libsonnet @@ -0,0 +1,49 @@ +local utils = import 'mixin-utils/utils.libsonnet'; + +(import 'dashboard-utils.libsonnet') { + 'cortex-parquet-converter-resources.json': + ($.dashboard('Cortex / Parquet Converter Resources') + { uid: 'c603a418fc4b1f77e6f13386b24f64f3' }) + .addClusterSelectorTemplates() + .addRow( + $.row('CPU and Memory') + .addPanel( + $.containerCPUUsagePanel('CPU', 'parquet-converter'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'parquet-converter'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.parquet_converter), + ) + ) + .addRow( + $.row('Network') + .addPanel( + $.containerNetworkReceiveBytesPanel($._config.instance_names.parquet_converter), + ) + .addPanel( + $.containerNetworkTransmitBytesPanel($._config.instance_names.parquet_converter), + ) + ) + .addRow( + $.row('Disk') + .addPanel( + $.containerDiskWritesPanel('Disk Writes', 'parquet-converter'), + ) + .addPanel( + $.containerDiskReadsPanel('Disk Reads', 'parquet-converter'), + ) + .addPanel( + $.containerDiskSpaceUtilization('Disk Space Utilization', 'parquet-converter'), + ) + ) + { + templating+: { + list: [ + // Do not allow to include all clusters/namespaces otherwise this dashboard + // risks to explode because it shows resources per pod. + l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {}) + for l in super.list + ], + }, + }, +} diff --git a/cortex-mixin/dashboards/parquet.libsonnet b/cortex-mixin/dashboards/parquet.libsonnet new file mode 100644 index 0000000..69bac77 --- /dev/null +++ b/cortex-mixin/dashboards/parquet.libsonnet @@ -0,0 +1,135 @@ +local utils = import 'mixin-utils/utils.libsonnet'; + +(import 'dashboard-utils.libsonnet') { + // Shows the parquet pipeline end to end: the parquet converter turns TSDB blocks into + // parquet files, the compactor reports how many blocks are converted, and the queriers + // and rulers read the parquet files through the parquet queryable. + local queryableRows(title, jobName) = [ + $.row(title) + .addPanel( + $.timeseriesPanel('Blocks queried / sec', unit='ops') + + $.queryPanel('sum by(type) (rate(cortex_parquet_queryable_blocks_queried_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{type}}') + + $.stack + + $.panelDescription( + 'Blocks queried / sec', + ||| + Rate of blocks found to query, by the format that served them. + tsdb blocks have no parquet files yet and are read from the store-gateways. + ||| + ), + ) + .addPanel( + $.timeseriesPanel('Operations / sec', unit='ops') + + $.queryPanel('sum by(type, method) (rate(cortex_parquet_queryable_operations_total{%s}[$__rate_interval]))' % $.jobMatcher(jobName), '{{method}} {{type}}') + + $.stack + + $.panelDescription( + 'Operations / sec', + ||| + Rate of Select, LabelNames and LabelValues operations, by the blocks they read. + parquet reads only parquet blocks, tsdb only TSDB blocks, and mixed both. + ||| + ), + ) + .addPanel( + $.timeseriesPanel('Shard cache hit ratio', unit='percentunit') + + $.queryPanel( + ||| + sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval])) + / + ( + sum(rate(cortex_parquet_cache_hits_total{%s}[$__rate_interval])) + + + sum(rate(cortex_parquet_cache_misses_total{%s}[$__rate_interval])) + ) + ||| % [$.jobMatcher(jobName), $.jobMatcher(jobName), $.jobMatcher(jobName)], + 'hit ratio' + ) + + { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) }, + ) + .addPanel( + $.timeseriesPanel('Shard cache items') + + $.queryPanel('sum by(%s) (cortex_parquet_cache_item_count{%s})' % [$._config.per_instance_label, $.jobMatcher(jobName)], '{{%s}}' % $._config.per_instance_label), + ), + ], + + 'cortex-parquet.json': + ($.dashboard('Cortex / Parquet') + { uid: 'dfef97d2b024dfd26ee9968cc0a095b9' }) + .addClusterSelectorTemplates() + .addRow( + $.row('Converter') + .addPanel( + $.successFailurePanel( + 'Block conversions / sec', + 'sum(rate(cortex_parquet_converter_blocks_converted_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter), + 'sum(rate(cortex_parquet_converter_block_convert_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.parquet_converter), + ) + + { yaxes: $.yaxes('ops') } + + $.panelDescription( + 'Block conversions / sec', + ||| + Rate of TSDB blocks converted to parquet, and of conversions that failed. + ||| + ), + ) + .addPanel( + $.timeseriesPanel('Tenants owned') + + $.queryPanel('sum by(%s) (cortex_parquet_converter_users_owned{%s})' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.parquet_converter)], '{{%s}}' % $._config.per_instance_label) + + $.panelDescription( + 'Tenants owned', + ||| + Number of tenants that each parquet converter instance owns in the ring. + ||| + ), + ) + ) + .addRow( + $.row('') + .addPanel( + $.timeseriesPanel('Conversion delay', unit='m') + + $.latencyPanel('cortex_parquet_converter_convert_block_delay_minutes', '{%s}' % $.jobMatcher($._config.job_names.parquet_converter), multiplier='1') + + $.panelDescription( + 'Conversion delay', + ||| + Time from the upload of a TSDB block to object storage until its parquet files are written. + ||| + ), + ) + .addPanel( + $.timeseriesPanel('Latest block conversion duration', unit='s') + + $.queryPanel('topk(10, max by(user) (cortex_parquet_converter_convert_block_duration_seconds{%s}))' % $.jobMatcher($._config.job_names.parquet_converter), '{{user}}') + + $.panelDescription( + 'Latest block conversion duration', + ||| + Time that the latest block conversion took, for the 10 tenants with the slowest one. + ||| + ), + ) + ) + .addRow( + $.row('Bucket') + .addPanel( + $.timeseriesPanel('Tenants with largest number of unconverted blocks') + + $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_unconverted_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') + + $.panelDescription( + 'Tenants with largest number of unconverted blocks', + ||| + The 10 tenants with the most blocks that are due for conversion but have no parquet files yet. + The compactor reports this value, so it only updates when the compactor cleans the tenant's bucket. + ||| + ), + ) + .addPanel( + $.timeseriesPanel('Tenants with largest number of parquet blocks') + + $.queryPanel('topk(10, max by(user) (cortex_bucket_parquet_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') + + $.panelDescription( + 'Tenants with largest number of parquet blocks', + ||| + The 10 tenants with the most parquet blocks in the bucket. + ||| + ), + ) + ) + .addRows(queryableRows('Querier', $._config.job_names.querier)) + .addRows(queryableRows('Ruler', $._config.job_names.ruler)) + .addRows($.getObjectStoreRows('Object Store', 'parquet-converter')), +} diff --git a/cortex-mixin/docs/playbooks.md b/cortex-mixin/docs/playbooks.md index 3958687..9e7aa7a 100644 --- a/cortex-mixin/docs/playbooks.md +++ b/cortex-mixin/docs/playbooks.md @@ -462,6 +462,86 @@ How to **investigate**: - Safely manually delete the block from the bucket if was a partial delete or an upload failed by a compactor - Further investigate if was an upload failed by an ingester but not later retried (ingesters are expected to retry uploads until succeed) +### CortexParquetConverterFailingConversions + +This alert fires when the parquet converter fails to convert blocks of a tenant and converts none of that tenant's blocks for 1 hour. + +How it **works**: +- The parquet converter scans its tenants on every `-parquet-converter.conversion-interval` (default 1m). For each block that is wider than the first `-compactor.block-ranges` range and has no valid converter mark, it downloads the block, writes the parquet files and then writes a converter mark. +- `cortex_parquet_converter_block_convert_failures_total` counts every failed conversion, except cancellations and permission errors. +- A block that fails is tried again on the next scan, so one bad block can make the failure counter increase on every scan. + +How to **investigate**: +- Look for errors in the parquet converter logs for the tenant in the alert. +- Check the disk use of the parquet converter. It downloads the whole TSDB block to its local disk before it converts it, so a block that is larger than the free disk fails every time. The `Cortex / Parquet Converter Resources` dashboard shows the disk use. +- Check the object store errors for `component="parquet-converter"` on the `Cortex / Parquet` dashboard. +- If one block fails every time, for example because it is corrupted, see [`CortexCompactorHasNotSuccessfullyRunCompaction`](#cortexcompactorhasnotsuccessfullyruncompaction) for how to skip a corrupted block. + +### CortexParquetConverterNotConverting + +This alert fires when a tenant has blocks that are due for parquet conversion, but the parquet converter has converted none of that tenant's blocks for 3 hours. + +How it **works**: +- The compactor sets `cortex_bucket_parquet_unconverted_blocks_count` when it cleans the tenant's bucket, every `-compactor.cleanup-interval` (default 15m). It counts the blocks that are wider than the first block range and have no parquet files. Blocks that are marked for deletion are included. +- Each tenant is owned by a parquet converter instance in the parquet converter ring. If no healthy instance owns the tenant, its blocks are not converted. + +How to **investigate**: +- Check that the parquet converter is running and that `cortex_parquet_converter_users_owned` is more than 0. If it is 0, check the parquet converter ring (`/parquet-converter/ring`) and [`CortexParquetConverterRingUnhealthy`](#cortexparquetconverterringunhealthy). +- Check that parquet conversion is enabled for the tenant (`parquet_converter_enabled` in the limits). +- Check the parquet converter logs and [`CortexParquetConverterFailingConversions`](#cortexparquetconverterfailingconversions). +- One scan processes the tenants one after another. A tenant with many large blocks can make one scan take hours, so other tenants wait. Check `cortex_parquet_converter_convert_block_duration_seconds` for slow tenants. +- If the blocks are marked for deletion, the converter does not convert them, and the alert stops when the compactor deletes them. + +### CortexParquetConversionDelayHigh + +This alert fires when more than 20% of the parquet conversions finish more than 5 hours after the TSDB block was uploaded to the object store. + +How it **works**: +- `cortex_parquet_converter_convert_block_delay_minutes` measures the time from the upload of the block (the `meta.json` modification time) to the end of its conversion. Its top bucket is 300 minutes, so all delays above 5 hours are in the `+Inf` bucket. +- The delay is only recorded when a conversion succeeds. If the converter stops converting, this alert stops firing. Use [`CortexParquetConverterNotConverting`](#cortexparquetconverternotconverting) for that case. +- A long delay is expected for blocks that the compactor uploads hours after their time range, for example after a backfill or after compaction was blocked. + +How to **investigate**: +- Check if the compactor is behind. A block can only be converted after the compactor has compacted it past the first block range. +- Check if the parquet converter is busy with one large tenant, on the `Cortex / Parquet` dashboard. +- Scale out the parquet converter, or give it more CPU and disk, if it cannot keep up. + +### CortexParquetConverterRingUnhealthy + +This alert fires when the parquet converter ring has unhealthy members for 15 minutes. The blocks of the tenants that an unhealthy member owns are not converted. + +How to **investigate**: +- Open the parquet converter ring page (`/parquet-converter/ring`) and find the unhealthy members. +- If a parquet converter pod was deleted without leaving the ring, remove its entry with the "Forget" button. +- If the pod is running, check its logs for errors with the ring key-value store. + +### CortexParquetQueryableMostlyTSDB + +This alert fires when more than 70% of the blocks that the parquet queryable finds in the queriers and rulers are TSDB blocks, for 2 hours. TSDB blocks are read from the store-gateways in place of the parquet files. + +How it **works**: +- Blocks that are not wider than the first block range are never converted, so queries for recent data always read TSDB blocks. A query load with many short-range queries can make this alert fire when parquet is working. + +How to **investigate**: +- Check the `Cortex / Parquet` dashboard for tenants with many unconverted blocks. +- Check [`CortexParquetConverterNotConverting`](#cortexparquetconverternotconverting) and [`CortexParquetCompactionStuckAtFirstRange`](#cortexparquetcompactionstuckatfirstrange). +- Check the time range of the queries. If most queries read only recent data, the TSDB blocks are expected. + +### CortexParquetCompactionStuckAtFirstRange + +This alert fires when more than 80% of the blocks of a tenant with more than 500 blocks are neither parquet blocks nor due for conversion, for 6 hours. + +How it **works**: +- The parquet converter only converts blocks that are wider than the first `-compactor.block-ranges` range. The blocks that the ingesters upload are not wider than that range, so they are converted only after the compactor compacts them. +- If the compactor cannot compact a tenant, its blocks stay at the first range. Then `cortex_bucket_parquet_unconverted_blocks_count` is 0 and the parquet alerts do not fire, but almost nothing is converted. +- The alert can also fire just after parquet conversion is enabled for a tenant with a long retention, until the compactor and the converter catch up. + +How to **investigate**: +- Check `cortex_compactor_remaining_planned_compactions` for the tenant. A value that stays above 0 means the compactor cannot finish the planned compactions. +- Check [`CortexCompactorHasNotSuccessfullyRunCompaction`](#cortexcompactorhasnotsuccessfullyruncompaction) and the compactor logs for the tenant. +- Check the disk use of the compactors. The compactor downloads all source blocks of a compaction job to its local disk. For a tenant with many ingesters, one job at the first range can need hundreds of GiB. A failed job keeps its downloaded blocks on disk until a compaction run for any tenant on that compactor succeeds, so failures can fill the disk. +- Check the memory use of the compactors for out-of-memory kills. + ### CortexQueriesIncorrect _TODO: this playbook has not been written yet._