From 556160e47f5bb66bc10866dc6ca84883c98d2d44 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 15 Oct 2026 13:43:32 +0200 Subject: [PATCH 1/2] feat: Support Vector 0.58.0 --- CHANGELOG.md | 3 ++- .../properties/product_logging/test-vector.sh | 24 ++++++++++++------- .../product_logging/vector-test.yaml | 6 +++++ .../properties/product_logging/vector.yaml | 13 +++++++--- .../01-install-hdfs-vector-aggregator.yaml | 2 +- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27a7e250..df690090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ All notable changes to this project will be documented in this file. inputs and role-specific containers, and a shared role-builder then builds the Services, ConfigMap and StatefulSet from them ([#833]). - Bump stackable-operator to 0.119.0 ([#835]). -- test: Bump vector-aggregator to 0.58.0 ([#843]). +- Support Vector 0.58.0 ([#843], [#847]). - test: Update OPA to 1.20.2 in integration tests ([#845]). - Bump stackable-operator to 0.121.0 and remove `serde_yaml` ([#846]). - All pods now set `securityContext.seccompProfile.type` to `RuntimeDefault` ([#846]). @@ -82,6 +82,7 @@ All notable changes to this project will be documented in this file. [#843]: https://github.com/stackabletech/hdfs-operator/pull/843 [#845]: https://github.com/stackabletech/hdfs-operator/pull/845 [#846]: https://github.com/stackabletech/hdfs-operator/pull/846 +[#847]: https://github.com/stackabletech/hdfs-operator/pull/847 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh index 1e87f18c..992b8c9c 100755 --- a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh +++ b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh @@ -1,11 +1,19 @@ #!/usr/bin/env sh -DATA_DIR=/stackable/log/_vector-state \ -LOG_DIR=/stackable/log \ -NAMESPACE=default \ -CLUSTER_NAME=test-hdfs \ -ROLE_NAME=namenode \ -ROLE_GROUP_NAME=default \ -VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ -VECTOR_FILE_LOG_LEVEL=info \ +set -eu + +export DATA_DIR=/stackable/log/_vector-state +export LOG_DIR=/stackable/log +export NAMESPACE=default +export CLUSTER_NAME=test-hdfs +export ROLE_NAME=namenode +export ROLE_GROUP_NAME=default +export VECTOR_AGGREGATOR_ADDRESS=http://vector-aggregator:6000 +export VECTOR_FILE_LOG_LEVEL=info + +# Required because vector.yaml contains ${...} placeholders +export VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true + +vector validate --no-environment vector.yaml + vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index b4bf7670..379f4cc3 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -17,6 +17,7 @@ tests: timestamp: 2025-10-02T09:27:28.582Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -47,6 +48,7 @@ tests: timestamp: 2025-10-02T09:27:28.582Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -81,6 +83,7 @@ tests: timestamp: 2025-10-02T09:27:29.473487331Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -116,6 +119,7 @@ tests: timestamp: 2025-10-02T09:27:29.473487331Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -153,6 +157,7 @@ tests: version: 0.49.0 outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -181,6 +186,7 @@ tests: level: INFO outputs: - extract_from: filtered_logs_vector + expected_event_count: 1 conditions: - type: vrl source: | diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index deb19f6d..1d4e04b0 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -119,7 +119,9 @@ transforms: # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - - processed_files_* + - processed_files_stdout + - processed_files_stderr + - processed_files_log4j type: remap source: | del(.source_type) @@ -157,7 +159,8 @@ transforms: # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -171,4 +174,8 @@ sinks: inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} diff --git a/tests/templates/kuttl/logging/01-install-hdfs-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-hdfs-vector-aggregator.yaml index 5c85ef9b..d142a949 100644 --- a/tests/templates/kuttl/logging/01-install-hdfs-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-hdfs-vector-aggregator.yaml @@ -14,4 +14,4 @@ kind: ConfigMap metadata: name: hdfs-vector-aggregator-discovery data: - ADDRESS: hdfs-vector-aggregator:6123 + ADDRESS: http://hdfs-vector-aggregator:6123 From c0c8e69fd9d47e619120573eeeb1c943040a4410 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 15 Oct 2026 16:13:04 +0200 Subject: [PATCH 2/2] fix: Fix the smoke test --- tests/templates/kuttl/smoke/30-assert.yaml.j2 | 2 +- .../31_configmap_hdfs-datanode-default.yaml.j2 | 13 ++++++++++--- .../31_configmap_hdfs-journalnode-default.yaml.j2 | 13 ++++++++++--- .../31_configmap_hdfs-namenode-default.yaml.j2 | 13 ++++++++++--- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/templates/kuttl/smoke/30-assert.yaml.j2 b/tests/templates/kuttl/smoke/30-assert.yaml.j2 index 475dd22d..6a5d8414 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-assert.yaml.j2 @@ -411,7 +411,7 @@ spec: - |- mkdir --parents /stackable/log/_vector-state # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file - vector & vector_pid=$! + /stackable/bin/vector-with-env-var-interpolation & vector_pid=$! if [ ! -f "/stackable/log/_vector/shutdown" ]; then mkdir -p /stackable/log/_vector inotifywait -qq --event create /stackable/log/_vector; diff --git a/tests/templates/kuttl/smoke/31_configmap_hdfs-datanode-default.yaml.j2 b/tests/templates/kuttl/smoke/31_configmap_hdfs-datanode-default.yaml.j2 index a9bda13c..d49f336a 100644 --- a/tests/templates/kuttl/smoke/31_configmap_hdfs-datanode-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/31_configmap_hdfs-datanode-default.yaml.j2 @@ -295,7 +295,9 @@ vector.yaml: | # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - - processed_files_* + - processed_files_stdout + - processed_files_stderr + - processed_files_log4j type: remap source: | del(.source_type) @@ -333,7 +335,8 @@ vector.yaml: | # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -347,7 +350,11 @@ vector.yaml: | inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} {% endif -%} wait-for-namenodes.log4j.properties: |+ log4j.rootLogger=INFO, CONSOLE, FILE diff --git a/tests/templates/kuttl/smoke/31_configmap_hdfs-journalnode-default.yaml.j2 b/tests/templates/kuttl/smoke/31_configmap_hdfs-journalnode-default.yaml.j2 index f5fd3fb0..e0f91d57 100644 --- a/tests/templates/kuttl/smoke/31_configmap_hdfs-journalnode-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/31_configmap_hdfs-journalnode-default.yaml.j2 @@ -288,7 +288,9 @@ vector.yaml: | # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - - processed_files_* + - processed_files_stdout + - processed_files_stderr + - processed_files_log4j type: remap source: | del(.source_type) @@ -326,7 +328,8 @@ vector.yaml: | # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -340,5 +343,9 @@ vector.yaml: | inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} {%- endif -%} diff --git a/tests/templates/kuttl/smoke/31_configmap_hdfs-namenode-default.yaml.j2 b/tests/templates/kuttl/smoke/31_configmap_hdfs-namenode-default.yaml.j2 index 3d76bd2a..af8d7a0f 100644 --- a/tests/templates/kuttl/smoke/31_configmap_hdfs-namenode-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/31_configmap_hdfs-namenode-default.yaml.j2 @@ -321,7 +321,9 @@ vector.yaml: | # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - - processed_files_* + - processed_files_stdout + - processed_files_stderr + - processed_files_log4j type: remap source: | del(.source_type) @@ -359,7 +361,8 @@ vector.yaml: | # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -373,7 +376,11 @@ vector.yaml: | inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} {% endif -%} zkfc.log4j.properties: |+ log4j.rootLogger=INFO, CONSOLE, FILE