diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eef84f4..7c8c5166 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ All notable changes to this project will be documented in this file. - Bump stackable-operator to 0.119.0 ([#1029]). - Bump stackable-operator to 0.121.0 and remove `serde_yaml` ([#1043]). - All pods now set `securityContext.seccompProfile.type` to `RuntimeDefault` ([#1043]). -- test: Bump vector-aggregator to 0.58.0 ([#1039]). +- Support Vector 0.58.0 ([#1039], [#1044]). - test: Update OPA to 1.20.2 in integration tests and docs example ([#1042]). ### Fixed @@ -91,6 +91,7 @@ All notable changes to this project will be documented in this file. [#1039]: https://github.com/stackabletech/kafka-operator/pull/1039 [#1042]: https://github.com/stackabletech/kafka-operator/pull/1042 [#1043]: https://github.com/stackabletech/kafka-operator/pull/1043 +[#1044]: https://github.com/stackabletech/kafka-operator/pull/1044 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh index d7535667..5497b07a 100755 --- a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh +++ b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh @@ -1,11 +1,19 @@ #!/usr/bin/env sh -DATA_DIR=/stackable/log/_vector-state \ -LOG_DIR=/stackable/log \ -NAMESPACE=default \ -CLUSTER_NAME=kafka \ -ROLE_NAME=broker \ -ROLE_GROUP_NAME=default \ -VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ -VECTOR_FILE_LOG_LEVEL=info \ +set -eu + +export DATA_DIR=/stackable/log/_vector-state +export LOG_DIR=/stackable/log +export NAMESPACE=default +export CLUSTER_NAME=kafka +export ROLE_NAME=broker +export ROLE_GROUP_NAME=default +export VECTOR_AGGREGATOR_ADDRESS=http://vector-aggregator:6000 +export VECTOR_FILE_LOG_LEVEL=info + +# Required because vector.yaml contains ${...} placeholders +export VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true + +vector validate --no-environment vector.yaml + vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index c62a3807..5af20808 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -21,6 +21,7 @@ tests: timestamp: 2025-10-02T09:27:29.473487331Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -56,6 +57,7 @@ tests: timestamp: 2025-10-02T09:27:29.473487331Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -93,6 +95,7 @@ tests: version: 0.49.0 outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -121,6 +124,7 @@ tests: level: INFO outputs: - extract_from: filtered_logs_vector + expected_event_count: 1 conditions: - type: vrl source: | diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 4f545bc8..80bf691c 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -228,7 +228,8 @@ transforms: extended_logs_files: inputs: - - processed_files_* + - processed_files_log4j + - processed_files_log4j2 type: remap source: | del(.source_type) @@ -264,7 +265,8 @@ transforms: extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -273,8 +275,13 @@ transforms: .roleGroup = "${ROLE_GROUP_NAME}" sinks: + # Forward the logs to the Vector aggregator aggregator: inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} diff --git a/tests/templates/kuttl/logging/01-install-kafka-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-kafka-vector-aggregator.yaml index 1b08761c..f479b6d1 100644 --- a/tests/templates/kuttl/logging/01-install-kafka-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-kafka-vector-aggregator.yaml @@ -14,4 +14,4 @@ kind: ConfigMap metadata: name: kafka-vector-aggregator-discovery data: - ADDRESS: kafka-vector-aggregator:6123 + ADDRESS: http://kafka-vector-aggregator:6123 diff --git a/tests/templates/kuttl/opa/01-install-vector-aggregator-discovery-configmap.yaml b/tests/templates/kuttl/opa/01-install-vector-aggregator-discovery-configmap.yaml index 197db165..91752a60 100644 --- a/tests/templates/kuttl/opa/01-install-vector-aggregator-discovery-configmap.yaml +++ b/tests/templates/kuttl/opa/01-install-vector-aggregator-discovery-configmap.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install vector-aggregator vector --namespace $NAMESPACE - --version 0.45.0 + --version 0.58.0 `# app version 0.58.0` --repo https://helm.vector.dev --values vector-aggregator-values.yaml --- @@ -14,4 +14,4 @@ kind: ConfigMap metadata: name: vector-aggregator-discovery data: - ADDRESS: vector-aggregator:6123 + ADDRESS: http://vector-aggregator:6123 diff --git a/tests/templates/kuttl/smoke-kraft/01-install-kafka-vector-aggregator.yaml b/tests/templates/kuttl/smoke-kraft/01-install-kafka-vector-aggregator.yaml index 6c7b01cc..fa2c6841 100644 --- a/tests/templates/kuttl/smoke-kraft/01-install-kafka-vector-aggregator.yaml +++ b/tests/templates/kuttl/smoke-kraft/01-install-kafka-vector-aggregator.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install kafka-vector-aggregator vector --namespace $NAMESPACE - --version 0.43.0 + --version 0.58.0 `# app version 0.58.0` --repo https://helm.vector.dev --values kafka-vector-aggregator-values.yaml --- @@ -14,4 +14,4 @@ kind: ConfigMap metadata: name: vector-aggregator-discovery data: - ADDRESS: kafka-vector-aggregator:6123 + ADDRESS: http://kafka-vector-aggregator:6123 diff --git a/tests/templates/kuttl/smoke/34-assert.yaml.j2 b/tests/templates/kuttl/smoke/34-assert.yaml.j2 index 607cb0b0..8950837a 100644 --- a/tests/templates/kuttl/smoke/34-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/34-assert.yaml.j2 @@ -330,7 +330,8 @@ commands: extended_logs_files: inputs: - - processed_files_* + - processed_files_log4j + - processed_files_log4j2 type: remap source: | del(.source_type) @@ -366,7 +367,8 @@ commands: extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -375,11 +377,16 @@ commands: .roleGroup = "${ROLE_GROUP_NAME}" sinks: + # Forward the logs to the Vector aggregator aggregator: inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} {% endif %} YAMLEOF )