From f74a1088df733f4a8418334bdd8bd7dbfa342c62 Mon Sep 17 00:00:00 2001 From: David Hadley Date: Mon, 5 Oct 2026 15:46:24 +0100 Subject: [PATCH 1/2] feat(monitoring): implement vCluster syncer grafana dashboard --- .../vcluster-syncer-dashboard.json | 1330 +++++++++++++++++ 1 file changed, 1330 insertions(+) create mode 100644 charts/monitoring/grafana-dashboards/vcluster-syncer-dashboard.json diff --git a/charts/monitoring/grafana-dashboards/vcluster-syncer-dashboard.json b/charts/monitoring/grafana-dashboards/vcluster-syncer-dashboard.json new file mode 100644 index 000000000..2e8740fd5 --- /dev/null +++ b/charts/monitoring/grafana-dashboards/vcluster-syncer-dashboard.json @@ -0,0 +1,1330 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus / Thanos", + "description": "Select the Grafana data source that queries Thanos.", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Host API requests / second", + "description": "All syncer REST requests to non-loopback API endpoints.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\"}[$__rate_interval]))", + "legendFormat": "requests/s", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Host API failures in selected range", + "description": "HTTP 4xx/5xx and transport errors to non-loopback endpoints.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\",code=~\"4..|5..|\"}[$__range]))", + "legendFormat": "failures", + "range": false, + "instant": true, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Pod workqueue retries in selected range", + "description": "Delayed/rate-limited requeues for the pod workqueue; not every increment necessarily represents an error.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_workqueue_retries_total{name=\"pod\"}[$__range]))", + "legendFormat": "pod retries", + "range": false, + "instant": true, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Pod reconcile errors in selected range", + "description": "Pod controller reconciliation errors.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_controller_runtime_reconcile_errors_total{controller=\"pod\"}[$__range]))", + "legendFormat": "pod errors", + "range": false, + "instant": true, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 6, + "type": "row", + "title": "Host API pressure", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 6, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Host API request rate by method", + "description": "Non-loopback endpoints are treated as host-cluster APIs.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 7, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Host API request rate by status", + "description": "Includes successful and unsuccessful responses.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 7, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Host API failure rate by code and method", + "description": "Watch especially POST 403, 429, 5xx, and .", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (code, method) (rate(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\",code=~\"4..|5..|\"}[$__rate_interval]))", + "legendFormat": "{{method}} {{code}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Mutating host API request rate", + "description": "Mutating traffic is most directly related to sync attempts.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\",method=~\"POST|PUT|PATCH|DELETE\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "row", + "title": "Retries and reconciliation", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "Workqueue retries per minute", + "description": "Delayed/rate-limited requeues by workqueue.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "60 * sum by (name) (rate(vcluster_syncer_workqueue_retries_total[$__rate_interval]))", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Workqueue additions per minute", + "description": "All additions by workqueue.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "60 * sum by (name) (rate(vcluster_syncer_workqueue_adds_total[$__rate_interval]))", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Retry share of queue additions", + "description": "Diagnostic ratio. It is not an exact percentage of failed objects.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * sum by (name) (rate(vcluster_syncer_workqueue_retries_total[$__rate_interval])) / clamp_min(sum by (name) (rate(vcluster_syncer_workqueue_adds_total[$__rate_interval])), 0.000001)", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Reconcile rate by controller and result", + "description": "Reconciliation outcomes by controller.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller, result) (rate(vcluster_syncer_controller_runtime_reconcile_total[$__rate_interval]))", + "legendFormat": "{{controller}} / {{result}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "row", + "title": "Pod failure correlation", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 17, + "type": "timeseries", + "title": "Pod retries, reconcile errors, and host POST 403", + "description": "Correlation panel for persistent policy rejection. Values are increases over Grafana's calculated rate interval.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 24, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_workqueue_retries_total{name=\"pod\"}[$__rate_interval]))", + "legendFormat": "pod queue retries", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_controller_runtime_reconcile_errors_total{controller=\"pod\"}[$__rate_interval]))", + "legendFormat": "pod reconcile errors", + "range": true, + "instant": false, + "refId": "B", + "format": "time_series" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\",method=\"POST\",code=\"403\"}[$__rate_interval]))", + "legendFormat": "host POST 403", + "range": true, + "instant": false, + "refId": "C", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "row", + "title": "Queue health and worker saturation", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 50, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 19, + "type": "timeseries", + "title": "Workqueue depth", + "description": "Current queued items by workqueue.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 51, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (name) (vcluster_syncer_workqueue_depth)", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Queue wait duration p95", + "description": "95th percentile time in queue.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 51, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (name, le) (rate(vcluster_syncer_workqueue_queue_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "timeseries", + "title": "Work duration p95", + "description": "95th percentile processing duration.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 59, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (name, le) (rate(vcluster_syncer_workqueue_work_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 22, + "type": "timeseries", + "title": "Worker utilisation", + "description": "Active workers as a percentage of configured maximum.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 59, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * max by (controller) (vcluster_syncer_controller_runtime_active_workers) / clamp_min(max by (controller) (vcluster_syncer_controller_runtime_max_concurrent_reconciles), 1)", + "legendFormat": "{{controller}}", + "range": true, + "instant": false, + "refId": "A", + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 23, + "type": "row", + "title": "Failure history", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 67, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 24, + "type": "table", + "title": "Host API failures over selected range", + "description": "Cumulative failures for the selected dashboard time range.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 68, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (code, method) (increase(vcluster_syncer_rest_client_requests_total{host=~\"$host_api\",host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\",code=~\"4..|5..|\"}[$__range]))", + "legendFormat": "{{method}} {{code}}", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + }, + { + "id": 25, + "type": "table", + "title": "Top workqueues by retries over selected range", + "description": "Top delayed/rate-limited workqueues in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 68, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum by (name) (increase(vcluster_syncer_workqueue_retries_total[$__range])))", + "legendFormat": "{{name}}", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": {} + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["vcluster"], + "templating": { + "list": [ + { + "name": "host_api", + "label": "Observed non-loopback API hosts", + "type": "query", + "hide": 0, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(vcluster_syncer_rest_client_requests_total{host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\"}, host)", + "query": { + "query": "label_values(vcluster_syncer_rest_client_requests_total{host!~\"^(127[.]0[.]0[.]1|localhost|[[]?::1[]]?)(:[0-9]+)?$\"}, host)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "vCluster Syncer", + "uid": "vcluster-syncer-0001", + "version": 1, + "weekStart": "" +} From ef338b3ec976507e468bd95f5cdcea94cfdcbb4c Mon Sep 17 00:00:00 2001 From: David Hadley Date: Thu, 15 Oct 2026 08:23:37 +0100 Subject: [PATCH 2/2] chore(charts): increment monitoring chart version --- charts/monitoring/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/monitoring/Chart.yaml b/charts/monitoring/Chart.yaml index d4eb9436e..bca0d59c1 100644 --- a/charts/monitoring/Chart.yaml +++ b/charts/monitoring/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: monitoring description: A monitoring stack for the workflows deployment type: application -version: 0.1.41 +version: 0.1.42 dependencies: - name: grafana repository: https://grafana-community.github.io/helm-charts