From 9d415a21f55b0dcf38b180926bb0b450013d95ba Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 20 Nov 2026 10:49:27 +0800 Subject: [PATCH 1/2] Search OTLP logs by their own searchable tags and severity, and keep each LAL rule's layer to itself OTLP logs: - Add `receiver-otel.otlpLogSearchableTags` (`SW_OTEL_LOG_SEARCHABLE_TAG_KEYS`, default `level,severity,ai_route_type`), the tag keys indexed as searchable on OTLP logs in place of core's `searchableLogsTags`, which the native logs keep. `ai_route_type` moves from the core default to it. The log query accepts a key of either list. Both lists are split and trimmed the same way. - Write a record's `severity_number` as the `severity` tag (its enum name) and the `level` tag (its range: TRACE, DEBUG, INFO, WARN, ERROR, FATAL). - Keep a record, and the logs of a resource, whose attributes repeat a key, with its last value. - Write OTLP logs' autocomplete tags under the new tag type `OTLP_LOG`. `queryLogTagAutocompleteKeys` and `queryLogTagAutocompleteValues` take an optional `type` (query-protocol #182); without it, both sources are read in one storage query. `ITagAutoCompleteQueryDAO` takes a list of tag type names. LAL: - Each rule a log runs gets its own copy of the log's metadata, so a `layer` statement in one rule no longer changes the layer of the rules that run after it. - A `layer` statement that evaluates to an empty, null or unregistered layer drops the log with a warning instead of failing in the traffic sink. - A `layer: auto` rule with a `sink` but no `layer` statement is skipped at boot and refused as a runtime rule. - Warn when no `fallback: true` rule is loaded, or a runtime change removes the last one. - Attribute a runtime LAL rule to `lal/.yaml`, not `lal/lal/.yaml`. --- docs/en/api/query-protocol.md | 8 +- docs/en/changes/changes.md | 8 +- docs/en/concepts-and-designs/lal.md | 15 +- .../backend-envoy-ai-gateway-monitoring.md | 3 +- .../setup/backend/configuration-vocabulary.md | 5 +- docs/en/setup/backend/log-otlp.md | 32 +++ oap-server/analyzer/log-analyzer/CLAUDE.md | 24 +- .../analyzer/v2/compiler/LALBlockCodegen.java | 9 +- .../analyzer/v2/compiler/LALScriptModel.java | 38 +++ .../log/analyzer/v2/dsl/ExecutionContext.java | 34 +++ .../v2/dsl/spec/filter/FilterSpec.java | 19 +- .../analyzer/v2/provider/log/LogAnalyzer.java | 6 +- .../log/listener/LogFilterListener.java | 70 ++++- .../log/listener/TrafficSinkListener.java | 5 +- .../v2/compiler/LALSinkLayerTest.java | 248 +++++++++++++++++ .../provider/log/LogAnalyzerRoutingTest.java | 18 -- ...gFilterListenerAutoLayerStatementTest.java | 92 +++++++ .../LogFilterListenerPerRuleMetadataTest.java | 136 ++++++++++ .../runtimerule/apply/LalFileApplier.java | 9 +- .../apply/LalFileApplierSourceTest.java | 69 +++++ .../analysis/manual/searchtag/TagType.java | 6 +- .../oap/server/core/config/ConfigService.java | 48 ++++ .../query/TagAutoCompleteQueryService.java | 47 +++- .../oap/server/core/source/LogBuilder.java | 25 +- .../oap/server/core/source/LogMetadata.java | 17 +- .../query/ITagAutoCompleteQueryDAO.java | 22 +- .../config/ConfigServiceSplitTagKeysTest.java | 39 +++ ...agAutoCompleteQueryServiceLogTypeTest.java | 105 ++++++++ .../source/LogBuilderSearchableTagsTest.java | 142 ++++++++++ .../query/logql/handler/LogQLApiHandler.java | 5 +- .../oap/query/graphql/resolver/LogQuery.java | 15 +- .../src/main/resources/query-protocol | 2 +- .../otel/OtelMetricReceiverConfig.java | 16 ++ .../otel/OtelMetricReceiverProvider.java | 8 + .../otel/otlp/OpenTelemetryLogHandler.java | 89 ++++++- .../otlp/OpenTelemetryLogHandlerTagsTest.java | 251 ++++++++++++++++++ .../src/main/resources/application.yml | 8 +- .../BanyanDBTagAutocompleteQueryDAO.java | 10 +- .../banyandb/BanyanDBIndexModeQueryIT.java | 8 +- .../query/TagAutoCompleteQueryDAO.java | 9 +- .../query/EsReadIndexSelectionTest.java | 4 +- .../query/MetadataQueryITBase.java | 18 +- .../jdbc/common/dao/JDBCLogQueryDAO.java | 5 +- .../dao/JDBCTagAutoCompleteQueryDAO.java | 25 +- .../common/dao/JDBCMetadataQueryITBase.java | 8 +- ...BCTagAutoCompleteQueryDAOTagTypesTest.java | 67 +++++ .../dao/JDBCTagAutoCompleteQueryDAOTest.java | 13 +- .../cases/storage/expected/config-dump.yml | 3 +- 48 files changed, 1732 insertions(+), 131 deletions(-) create mode 100644 oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALSinkLayerTest.java create mode 100644 oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerAutoLayerStatementTest.java create mode 100644 oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerPerRuleMetadataTest.java create mode 100644 oap-server/server-admin/runtime-rule/src/test/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplierSourceTest.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/ConfigServiceSplitTagKeysTest.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryServiceLogTypeTest.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/source/LogBuilderSearchableTagsTest.java create mode 100644 oap-server/server-receiver-plugin/otel-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandlerTagsTest.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java diff --git a/docs/en/api/query-protocol.md b/docs/en/api/query-protocol.md index 5bae3c4ba0a9..bc192368cc9a 100644 --- a/docs/en/api/query-protocol.md +++ b/docs/en/api/query-protocol.md @@ -150,12 +150,16 @@ extend type Query { # Test the logs and get the results of the LAL output. test(requests: LogTestRequest!): LogTestResponse! # Read the list of searchable keys - queryLogTagAutocompleteKeys(duration: Duration!):[String!] + queryLogTagAutocompleteKeys(duration: Duration!, type: TagType):[String!] # Search the available value options of the given key. - queryLogTagAutocompleteValues(tagKey: String! , duration: Duration!):[String!] + queryLogTagAutocompleteValues(tagKey: String! , duration: Duration!, type: TagType):[String!] } ``` +The optional `type` of the tag autocomplete queries narrows them to one log source: `OTLP_LOG` for the logs received +through OTLP, whose searchable tags `receiver-otel.otlpLogSearchableTags` sets, or `LOG` for the logs of every other +source, whose searchable tags core's `searchableLogsTags` sets. Without a `type`, the tags of both are returned. + Log implementations vary between different database options. Some search engines like ElasticSearch and OpenSearch can support full log text fuzzy queries, while others do not due to considerations related to performance impact and end user experience. diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index c3986411454e..db0c560f670f 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -49,7 +49,7 @@ * Fix JDBC trace profiling queries failing when a task has no profiled segments: the empty segment-ID list now returns an empty result without generating an invalid SQL `IN ()` clause. * Add the `query-server` module, so the native query APIs can have their own port, apart from the ports agents report to. Set `SW_QUERY_SERVER_REST_PORT` to a port greater than 0, and GraphQL (`/graphql`), the AI agent conversation view and files (`/ai-agent/conversations/*`), and `/status/config/ttl` move from the core REST port (default `12800`) to it, with their own host, context path and TLS settings (`SW_QUERY_SERVER_REST_*`). Point the UI's `queryUrl`, `swctl --base-url` and other GraphQL clients at the new port. With the default `0`, nothing changes. Together with `receiver-sharing-server`, no agent or query traffic is left on the core ports. * LAL: save the logs of the OpenTelemetry services that set `service.layer=OTLP_GENERAL` through the new `otlp-general` rule in `lal/default.yaml`; no rule matched that layer, so these logs were dropped. A LAL rule declaring an unregistered or missing layer (names are case-sensitive) is now skipped with a warning at boot and rejected when applied as a runtime rule, instead of loading silently under `UNDEFINED` and never running. -* LAL: a log without a layer that no `layer: auto` rule claims runs the new `fallback: true` auto rules, instead of the rules declared for `GENERAL`, and is dropped when none claims it either, so where such a log is filed is decided by a rule, not hard-coded. The new `default-fallback` rule in `lal/default.yaml` files it under `GENERAL`, as before; edit its extractor to file it under another layer. A fallback rule must set the layer, as any auto rule. A custom `layer: GENERAL` rule written for the logs without a layer must declare `layer: auto` and `fallback: true` and set `layer "GENERAL"` in its extractor; it then runs next to `default-fallback`, as it ran next to `default` before. +* LAL: a log without a layer that no `layer: auto` rule claims runs the new `fallback: true` auto rules, instead of the rules declared for `GENERAL`, and is dropped when none claims it either, so where such a log is filed is decided by a rule, not hard-coded. The new `default-fallback` rule in `lal/default.yaml` files it under `GENERAL`, as before; edit its extractor to file it under another layer. A fallback rule must set the layer, as any auto rule. A custom `layer: GENERAL` rule written for the logs without a layer must declare `layer: auto` and `fallback: true` and set `layer "GENERAL"` in its extractor; it then runs next to `default-fallback`, as it ran next to `default` before. The OAP warns at boot when no `fallback: true` rule is loaded, and when a runtime rule change removes the last one, since the logs without a layer are then dropped with only a debug log each. * LAL: handle the mini-program error logs with the `miniprogram-errors-wechat` and `miniprogram-errors-alipay` rules in `lal/miniprogram.yaml`, declared for the `WECHAT_MINI_PROGRAM` and `ALIPAY_MINI_PROGRAM` layers, replacing the `layer: auto` rule `miniprogram-errors`. The mini-program SDK v0.5.0 sets `service.layer`, which routes its error logs to these rules; it is now the minimum supported SDK version, since an older SDK's error logs carry no layer and are filed under `GENERAL` without the `error_count` metric. * Fix bugs of the JDBC storage, `mysql` (MySQL and MariaDB) and `postgresql`. Many showed only when a query's time range crossed midnight, as this storage keeps records and metrics in one table per day: - The trace list came out of order when its time range crossed midnight, and its pages skipped or repeated traces. @@ -82,6 +82,12 @@ * Elasticsearch: fix the alarms of a page after the 10th losing their recovery time. * Fix a trace profiling task being refused as overlapping a task of the service that starts later in the minute the new task ends. * Elasticsearch: write a row into the daily index of its time bucket's day, whatever the bucket's precision. Async-profiler, pprof and continuous-profiling tasks, which carry a second-precision bucket, went into an index per hour that the TTL never deleted, and could not be written at all with `dayStep` above 1; the TTL now deletes the hour-named indices (`*-yyyyMMddHH`) earlier versions left, with their day. The copy of an eBPF network profiling task that keeps it alive carried time bucket 0, so it went into an index dated 0 that the next TTL run deleted; it now carries the minute it is written. BanyanDB: a task list now includes the tasks of the range's last time bucket. +* Keep the severity of an OTLP log, which the OTLP log receiver dropped before: a record's `severity_number` is now written as the `severity` tag, named by its enum constant (`SEVERITY_NUMBER_INFO`, `SEVERITY_NUMBER_ERROR2`, ...), and as the `level` tag, named by its range (`TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`), so `SEVERITY_NUMBER_ERROR2` is `ERROR`. A record whose number is unset gets neither tag, and a record attribute named `severity` or `level` is kept instead. +* Add `receiver-otel.otlpLogSearchableTags` (`SW_OTEL_LOG_SEARCHABLE_TAG_KEYS`), the tag keys indexed as searchable on OTLP logs, in place of core's `searchableLogsTags`, which the SkyWalking native logs keep. Its default, `level,severity,ai_route_type`, keeps `level` and `ai_route_type` and adds `severity`. `http.status_code`, which OTLP logs could be searched by before, is not in it: to search OTLP logs by HTTP status, list it, or `http.response.status_code` as the OpenTelemetry semantic conventions name the attribute. `ai_route_type`, the tag the Envoy AI Gateway LAL rules add to its OTLP access logs, moves from the `searchableLogsTags` default to this one, so `searchableLogsTags` defaults to `level,http.status_code` again. A deployment that customized `searchableLogsTags` (`SW_SEARCHABLE_LOGS_TAG_KEYS`) to search OTLP logs sets those keys in `otlpLogSearchableTags` instead: OTLP logs read `searchableLogsTags` only when `otlpLogSearchableTags` is empty, as it is in an `application.yml` without the setting. The log query accepts a key of either list. Both lists are trimmed the same way: a `searchableLogsTags` key written after a space or an empty entry, such as `level, http.status_code`, used to be kept with the space, so it matched no tag; it now names `http.status_code`. +* LAL: every rule a log runs, auto or declared for a layer, saves the log under the layer the log arrived with or one its own `layer` statement decided. The rules shared the log's metadata, so a `layer` statement in one rule changed the layer of the rules that ran after it on the same log, depending on the order the rules ran in, and an invalid layer from one rule made the next one fail; each rule now runs on its own copy of the metadata. An auto rule with a `sink` but no `layer` statement, which dropped every log it saved, is now skipped with a warning at boot and refused as a runtime rule; one without a `sink` needs no `layer` statement. In any LAL rule, a `layer` statement that evaluates to an empty, null or unregistered layer now drops the log with a warning when it is saved, where the traffic sink threw for it and lost the log with only a stack trace. +* The OTLP log receiver keeps a record whose attributes repeat a key, and the logs of a resource whose attributes repeat one, with the key's last value, instead of losing the record, or every log of the export request, to a duplicate-key error. +* Fix a LAL rule applied through the runtime-rule API being attributed to `lal/lal/.yaml`, the catalog prefix applied twice, in its error messages and its generated class; it is now `lal/.yaml`, as the same file loaded at boot. +* Log tag autocomplete keeps the tags of the logs received through OTLP apart from those of every other source, under the new tag type `OTLP_LOG`. `queryLogTagAutocompleteKeys` and `queryLogTagAutocompleteValues` take an optional `type` (`LOG` or `OTLP_LOG`) to read one source; without it, as the UI and LogQL query, they return both merged, within the same limit. The OTLP logs' tags written before the upgrade stay under `LOG`, so a query with `type: OTLP_LOG` misses them until they expire. Both are read in one storage query: `ITagAutoCompleteQueryDAO` takes a list of tag types, so a storage plugin outside this repository that implements it has to follow. #### UI diff --git a/docs/en/concepts-and-designs/lal.md b/docs/en/concepts-and-designs/lal.md index 736fd6129f42..433d46df1e9c 100644 --- a/docs/en/concepts-and-designs/lal.md +++ b/docs/en/concepts-and-designs/lal.md @@ -130,6 +130,18 @@ In `layer: auto` mode, `abort` and `drop` have distinct meanings: If an auto rule claims the log (doesn't abort) but doesn't set a layer, the log is warned and dropped at persistence time. +Every rule a log runs, auto or declared for a layer, starts from the layer the log arrived with, and only its own +`layer` statement changes it: a `layer` statement in one rule never changes the layer another rule saves the log +under, whatever order the rules run in. + +In any rule, auto or declared for a layer, a `layer` statement whose value is empty, null (for example +`layer parsed.kind as String` on a log without `kind`) or not a registered layer name drops the log with a +warning when it is saved. A rule without a `layer` statement saves the log under the layer it arrived with. + +An auto rule with a `sink` must therefore have a `layer` statement, on some branch of its script. One without +it is skipped with a warning at boot, and refused when applied as a runtime rule. An auto rule without a `sink`, +for example one that only extracts metrics, never saves a log and needs no `layer` statement. + All the auto rules run on such a log, in no defined order, and each one that claims it saves a copy of its own, so an auto rule should claim only the logs it recognizes and `abort {}` on the others. @@ -137,7 +149,8 @@ A rule declaring `fallback: true` (allowed only with `layer: auto`) runs after t none of them claimed the log. It is the catch-all for the logs without a layer: `default-fallback` in `lal/default.yaml` files them under `GENERAL`. A fallback rule follows the same rules as any auto rule: it must set the layer, and `abort {}` leaves the log unclaimed, which drops it. Auto and fallback rule names -share one name space. +share one name space. When no `fallback: true` rule is loaded, the OAP warns at boot, and when a runtime rule +change removes the last one, since the logs without a layer that no auto rule claims are then dropped. ```yaml rules: diff --git a/docs/en/setup/backend/backend-envoy-ai-gateway-monitoring.md b/docs/en/setup/backend/backend-envoy-ai-gateway-monitoring.md index ad42eb21a2df..4a4127f5f60c 100644 --- a/docs/en/setup/backend/backend-envoy-ai-gateway-monitoring.md +++ b/docs/en/setup/backend/backend-envoy-ai-gateway-monitoring.md @@ -141,7 +141,8 @@ All MCP service-level metrics are also available per instance with `meter_envoy_ ### Access Log Sampling Access logs are tagged with `ai_route_type` (`llm` or `mcp`) for filtering in the log query UI. -The `ai_route_type` tag is searchable by default. +The `ai_route_type` tag is searchable by default, through `receiver-otel.otlpLogSearchableTags` +(`SW_OTEL_LOG_SEARCHABLE_TAG_KEYS`), since the access logs arrive over OTLP. **LLM route logs:** - **Error responses** (HTTP status >= 400) — always persisted. diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 8d0736296c4c..aaddfd1d32d4 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -38,7 +38,7 @@ It divided into several modules, each of which has its own settings. The followi | - | - | instanceNameMaxLength | Maximum length limit of service instance names. The maximum length of service + instance names should be less than 200. | SW_INSTANCE_NAME_MAX_LENGTH | 70 | | - | - | endpointNameMaxLength | Maximum length limit of endpoint names. The maximum length of service + endpoint names should be less than 240. | SW_ENDPOINT_NAME_MAX_LENGTH | 150 | | - | - | searchableTracesTags | Defines a set of span tag keys which are searchable through GraphQL. Multiple values are separated by commas. The max length of key=value should be less than 256 or will be dropped. | SW_SEARCHABLE_TAG_KEYS | http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker | -| - | - | searchableLogsTags | Defines a set of log tag keys which are searchable through GraphQL. Multiple values are separated by commas. The max length of key=value should be less than 256 or will be dropped. | SW_SEARCHABLE_LOGS_TAG_KEYS | level,http.status_code,ai_route_type | +| - | - | searchableLogsTags | Defines a set of log tag keys which are searchable through GraphQL. Multiple values are separated by commas. The max length of key=value should be less than 256 or will be dropped. | SW_SEARCHABLE_LOGS_TAG_KEYS | level,http.status_code | | - | - | searchableAlarmTags | Defines a set of alarm tag keys which are searchable through GraphQL. Multiple values are separated by commas. The max length of key=value should be less than 256 or will be dropped. | SW_SEARCHABLE_ALARM_TAG_KEYS | level | | - | - | autocompleteTagKeysQueryMaxSize | The max size of tags keys for autocomplete select. | SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE | 100 | | - | - | autocompleteTagValuesQueryMaxSize | The max size of tags values for autocomplete select. | SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE | 100 | @@ -218,7 +218,7 @@ It divided into several modules, each of which has its own settings. The followi | - | - | gRPCSslCertChainPath | File path of gRPC SSL cert chain. | SW_ALS_GRPC_SSL_CERT_CHAIN_PATH | - | | - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_ALS_GRPC_MAX_CONCURRENT_CALL | - | | - | - | maxMessageSize | Sets the maximum message size allowed to be received on the server. Empty means 4 MiB. | SW_ALS_GRPC_MAX_MESSAGE_SIZE | 4M(based on Netty) | -| receiver-otel | default | A receiver for analyzing metrics data from OpenTelemetry. | - | - | | +| receiver-otel | default | A receiver for OpenTelemetry (OTLP) metrics, traces and logs. | - | - | | | - | - | enabledHandlers | Enabled handlers for otel. | SW_OTEL_RECEIVER_ENABLED_HANDLERS | - | | - | - | enabledOtelMetricsRules | Enabled metric rules for OTLP handler. | SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES | - | | - | - | otlpTraceStorage | Where the `otlp-traces` handler stores spans. `otlp` stores the native OTLP span in `otlp_span`, served by the TraceQL `/otlp` datasource; `zipkin` converts to Zipkin v2 and hands them to receiver-zipkin. | SW_OTEL_TRACE_STORAGE | otlp | @@ -226,6 +226,7 @@ It divided into several modules, each of which has its own settings. The followi | - | - | otlpTraceSampleRate | `otlp` mode only. Head sampling by trace id, precision 1/10000. 10000 keeps every trace. | SW_OTEL_TRACE_SAMPLE_RATE | 10000 | | - | - | otlpTraceMaxSpansPerSecond | `otlp` mode only. Spans per second accepted before dropping. 0 means no limit. | SW_OTEL_TRACE_MAX_SPANS_PER_SECOND | 0 | | - | - | otlpServiceGroupKeys | Resource attribute keys, comma separated, in order of preference: the first one a resource carries groups its service as `value::service.name` in the trace analysis, the stored spans and the OTLP logs; the resource attributes stay as sent. See [Service group](otlp-trace.md#service-group). Empty groups nothing. | SW_OTEL_RECEIVER_OTLP_SERVICE_GROUP | | +| - | - | otlpLogSearchableTags | `otlp-logs` only. Tag keys, comma separated, indexed as searchable on OTLP logs, in place of core's `searchableLogsTags`, which the SkyWalking native logs keep: a log record attribute, the `severity` and `level` tags the receiver adds from the record's severity number, or a tag a LAL rule adds. Empty uses core's `searchableLogsTags`. The max length of key=value should be less than 256 or will be dropped. | SW_OTEL_LOG_SEARCHABLE_TAG_KEYS | level,severity,ai_route_type | | receiver-zipkin | default | A receiver for Zipkin traces. | - | - | | | - | - | sampleRate | The sample rate precision is 1/10000, should be between 0 and 10000 | SW_ZIPKIN_SAMPLE_RATE | 10000 | | - | - | maxSpansPerSecond | The maximum spans to be collected per second. 0 means no limit. Spans exceeding this threshold will be dropped. | SW_ZIPKIN_MAX_SPANS_PER_SECOND | 0 | diff --git a/docs/en/setup/backend/log-otlp.md b/docs/en/setup/backend/log-otlp.md index 5b3c68e87170..243a43f4e8b6 100644 --- a/docs/en/setup/backend/log-otlp.md +++ b/docs/en/setup/backend/log-otlp.md @@ -56,6 +56,38 @@ Note, that these attributes should be set manually through OpenTelemetry SDK or The OTLP log record body is delivered to [LAL](../../concepts-and-designs/lal.md) as a plain-text body, even when the string content is JSON-shaped. A LAL rule can still parse such content with the `json {}` parser — when the JSON body is empty and a text body is present, `json {}` tries the text as JSON. On a successful parse the log is normalized to a JSON body, so it is stored and rendered as JSON. See [`json`](../../concepts-and-designs/lal.md#json) for details. +A log record's attributes become the log's tags. Its `severity_number` adds two more: + +- `severity`: the `SeverityNumber` enum constant, such as `SEVERITY_NUMBER_INFO`, `SEVERITY_NUMBER_INFO2` or + `SEVERITY_NUMBER_ERROR`. +- `level`: the name of the number's range, `TRACE` (1-4), `DEBUG` (5-8), `INFO` (9-12), `WARN` (13-16), `ERROR` (17-20) + or `FATAL` (21-24), so `SEVERITY_NUMBER_INFO2` is `INFO` and `SEVERITY_NUMBER_ERROR3` is `ERROR`. + +The number is used rather than the free-form `severity_text`, which differs between SDKs (`INFO`, `info`, +`Information`). A record with no severity number, such as an Envoy access log, gets neither tag, and a record attribute +named `severity` or `level` is kept instead. + +### Searchable tags + +The tags OTLP logs can be searched by, and that tag autocomplete offers, are set by `otlpLogSearchableTags`, apart from +core's `searchableLogsTags`, which the logs of the SkyWalking native protocols keep: + +```yaml +receiver-otel: + default: + otlpLogSearchableTags: ${SW_OTEL_LOG_SEARCHABLE_TAG_KEYS:level,severity,ai_route_type} +``` + +- A key is a log record attribute, such as `http.response.status_code`, the `severity` or `level` tag, or a tag a LAL rule + adds, such as the Envoy AI Gateway rule's `ai_route_type`. +- Resource attributes are not stored with a log. To search by one, copy it to a tag in a LAL rule, for example + `tag 'deployment.environment.name': sourceAttribute("deployment.environment.name")`, and list that tag. +- List low-cardinality keys only: every distinct value of a searchable tag is stored for tag autocomplete. +- A tag whose `key=value` is longer than 256 characters is not made searchable, though it is still stored with the log. +- An empty list makes OTLP logs use core's `searchableLogsTags`. + +The log query accepts a key of either list, since all logs are stored together. + A log record written inside a span carries that span's `trace_id` and `span_id`. Both are stored with the log, as lowercase hex: the trace ID as the log's trace ID, and the span ID as its OTLP span ID — not the `spanId`, which is a SkyWalking span's number within its segment. So the log can be queried by the trace that wrote it, through diff --git a/oap-server/analyzer/log-analyzer/CLAUDE.md b/oap-server/analyzer/log-analyzer/CLAUDE.md index 5c1befa8ac83..0661986628fd 100644 --- a/oap-server/analyzer/log-analyzer/CLAUDE.md +++ b/oap-server/analyzer/log-analyzer/CLAUDE.md @@ -197,8 +197,8 @@ routes an empty-layer log in up to two passes, stopping at the first pass in whi (`LogAnalysisListener.claimed()`: did not abort): 1. The regular auto rules. -2. The `fallback: true` auto rules. Before this pass the metadata layer is reset to the value received, - since the rules share one `LogMetadata` and an aborted auto rule may have set a layer on it. +2. The `fallback: true` auto rules. They see the layer the log arrived with, whatever layer an aborted auto rule + set: each rule runs on its own copy of the `LogMetadata` (see below). Both groups live in one immutable `Factory.AutoRules` snapshot, replaced whole on every runtime change, so a rule whose `fallback` flag flips is never in neither (or both). `LogAnalyzer` takes both passes' listeners @@ -210,9 +210,25 @@ Auto and fallback rules share one name space and rule key (`ruleKey(null, name)` `find` / `contains` / `remove` / suspend / runtime-rule keys treat them alike; `addOrReplace` moves a rule between the two groups when its `fallback` flag changes. `fallback: true` on a non-auto rule fails compile. +Each rule a log runs gets its own copy of the log's `LogMetadata` (`LogFilterListener.parse`: +`metadata.toBuilder().build()`), since the generated `layer` statement (`h.ctx().layer(...)`) writes the metadata +layer, which the output builder and the sink listeners read: one rule's `layer` statement must not decide the layer +of the other rules the log runs. Every rule starts from the layer the log arrived with. The copy is shallow — +`sourceAttributes` and `traceContext` are shared, which holds as long as rules only read them. + The `autoLayerMode` flag on `ExecutionContext` (set for both passes) triggers a layer check in -`FilterSpec.doSink()`: if the extractor didn't set a layer, the log is warned and dropped. Because of it, -`TrafficSinkListener` never sees an empty layer and has no `GENERAL` fallback. +`FilterSpec.doSink()`: if the rule's extractor didn't set a layer, the log is warned and dropped. The check +reads `ExecutionContext.layer()`, the value the rule's own `layer` statement recorded on its context. + +For every rule, auto or not, `doSink()` also drops (with a warning) a log whose rule ran a `layer` statement that +evaluated to an empty, null or unregistered name. `ExecutionContext.isLayerDecided()` tells that the statement ran +— `layer()` alone is null both when it did not run and when it evaluated to null. A rule without one keeps the +layer the log arrived with, which `LogAnalyzer` checked. So `TrafficSinkListener`'s `Layer.valueOf` only sees +registered layers and has no `GENERAL` fallback. + +`Factory.savesWithoutLayer` refuses an auto rule with a `sink` but no `layer` statement on any branch +(`LALScriptModel.hasSink()` / `assignsLayer()`): `loadStaticRules` skips it with a warning, `compile` rejects it as +a runtime rule. An auto rule without a `sink` never saves a log and needs no `layer` statement. A static rule whose `layer` is neither `auto` nor a registered layer name (case-sensitive) is skipped with a warning at boot (`Factory.loadStaticRules`); a runtime rule declaring one is rejected (`resolveLayer`). diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java index 8fc2959aea20..1be0d3f8e654 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java @@ -288,11 +288,12 @@ private static void generateFieldToOutput( } sb.append(");\n"); - // LAYER is also used by FilterSpec.doSink() (layer:auto check) and by downstream routing - // that reads from LogMetadata. Propagate the LAL-decided layer to metadata so the output - // builder's initFromMetadata() and other metadata consumers see the same value. + // LAYER is also read by FilterSpec.doSink() (layer:auto check, per rule) and by downstream routing + // that reads from LogMetadata. ExecutionContext.layer() records it on the rule's context and writes it + // to the rule's own copy of the metadata, so the output builder's initFromMetadata() and the sink + // listeners see it too. if (field.getFieldType() == LALScriptModel.FieldType.LAYER) { - sb.append(" h.ctx().metadata().setLayer("); + sb.append(" h.ctx().layer("); LALValueCodegen.generateCastedValueAccess(sb, field.getValue(), "String", genCtx); sb.append(");\n"); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALScriptModel.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALScriptModel.java index e0dfe266a1a6..f11102b5ce0c 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALScriptModel.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALScriptModel.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import lombok.Getter; /** @@ -43,6 +44,43 @@ public LALScriptModel(final List statements) { this.statements = Collections.unmodifiableList(statements); } + /** + * @return whether the script has a {@code sink} block on any branch, so it may save a log + */ + public boolean hasSink() { + return anyStatement(statements, stmt -> stmt instanceof SinkBlock); + } + + /** + * @return whether the script has a {@code layer} statement on any branch + */ + public boolean assignsLayer() { + return anyStatement(statements, stmt -> stmt instanceof FieldAssignment + && ((FieldAssignment) stmt).getFieldType() == FieldType.LAYER); + } + + /** + * Whether a statement, or one nested in an {@code extractor} or {@code if} block, matches. + */ + private static boolean anyStatement(final List statements, final Predicate match) { + for (final Object stmt : statements) { + if (match.test(stmt)) { + return true; + } + if (stmt instanceof ExtractorBlock + && anyStatement(((ExtractorBlock) stmt).getStatements(), match)) { + return true; + } + if (stmt instanceof IfBlock) { + final IfBlock ifBlock = (IfBlock) stmt; + if (anyStatement(ifBlock.getThenBranch(), match) || anyStatement(ifBlock.getElseBranch(), match)) { + return true; + } + } + } + return false; + } + // ==================== Filter statements ==================== public interface FilterStatement { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java index 2cf19eefabcc..4ec9701d7065 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java @@ -58,6 +58,8 @@ public class ExecutionContext { public static final String KEY_AUTO_LAYER = "auto_layer"; public static final String KEY_OUTPUT = "output"; public static final String KEY_DROP_REASON = "drop_reason"; + public static final String KEY_LAYER = "layer"; + public static final String KEY_LAYER_DECIDED = "layer_decided"; private final Map properties = new HashMap<>(); @@ -114,6 +116,8 @@ public ExecutionContext init(final LogMetadata metadata, final Object input) { setProperty(KEY_DRY_RUN, false); setProperty(KEY_OUTPUT, null); setProperty(KEY_DROP_REASON, null); + setProperty(KEY_LAYER, null); + setProperty(KEY_LAYER_DECIDED, false); return this; } @@ -212,6 +216,36 @@ public boolean isDryRun() { return (boolean) getProperty(KEY_DRY_RUN); } + /** + * The LAL {@code layer} statement: record the layer this rule decided, and write it to this rule's + * {@link LogMetadata}, which the output builder and the sink listeners read. Each rule a log runs has its own + * copy of the metadata (LogFilterListener#parse), so the layer decides this rule's alone. + * + * @param layer the layer name the rule's expression evaluated to + * @return this context + */ + public ExecutionContext layer(final String layer) { + setProperty(KEY_LAYER, layer); + setProperty(KEY_LAYER_DECIDED, true); + metadata().setLayer(layer); + return this; + } + + /** + * @return the layer this rule's {@code layer} statement decided, as evaluated; null when the rule has not run + * one, or when its expression evaluated to null ({@link #isLayerDecided()} tells the two apart) + */ + public String layer() { + return (String) getProperty(KEY_LAYER); + } + + /** + * @return whether this rule ran a {@code layer} statement for this log + */ + public boolean isLayerDecided() { + return Boolean.TRUE.equals(getProperty(KEY_LAYER_DECIDED)); + } + public ExecutionContext autoLayerMode(final boolean autoLayer) { setProperty(KEY_AUTO_LAYER, autoLayer); return this; diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java index 55bbe54f25ed..6206c4e03f92 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java @@ -39,6 +39,7 @@ import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogSinkListenerFactory; import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.RecordSinkListener; import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.TrafficSinkListener; +import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.source.LogMetadata; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleStartException; @@ -287,9 +288,10 @@ private void doSink(final ExecutionContext ctx) { return; } - // In auto-layer mode, the script must set the layer. If not set, drop the log. + // In auto-layer mode, the rule must set the layer. If not set, drop the log. Read from the rule's own + // decision rather than its metadata, which is only what the rule decided as long as nothing else wrote it. if (ctx.isAutoLayerMode()) { - final String layer = metadata.getLayer(); + final String layer = ctx.layer(); if (layer == null || layer.isEmpty()) { LOGGER.warn("Auto-layer LAL rule did not set layer for service={}, dropping log", metadata.getService()); @@ -298,6 +300,19 @@ private void doSink(final ExecutionContext ctx) { } } + // A layer the rule's own `layer` statement decided must be registered: the sink listeners turn the log's layer + // into a Layer (TrafficSinkListener's Layer.valueOf), which throws for a null, empty or unregistered name. + // A rule without one leaves the layer the log arrived with, which LogAnalyzer checked. + if (ctx.isLayerDecided()) { + final String ruleLayer = ctx.layer(); + if (ruleLayer == null || ruleLayer.isEmpty() || Layer.nameOf(ruleLayer) == Layer.UNDEFINED) { + LOGGER.warn("LAL rule set the unregistered layer '{}' for service={}, dropping log", + ruleLayer, metadata.getService()); + ctx.drop(); + return; + } + } + sinkListenerFactories.stream() .map(LogSinkListenerFactory::create) .forEach(it -> it.parse(metadata, input, ctx).build()); diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java index 5985829aed20..5f7e7be25659 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java @@ -75,13 +75,11 @@ public void doAnalysis(LogMetadata metadata, final Object input) { .stream() .map(LogAnalysisListenerFactory::createAuto) .collect(Collectors.toList()); - final String arrivedLayer = metadata.getLayer(); + // Each rule runs on its own copy of the metadata (LogFilterListener#parse), so the fallback rules start + // from the layer the log arrived with, whatever layer an auto rule set before it aborted. if (analyzeUntilClaimed(metadata, input, passes, AutoListeners::getAuto)) { return; } - // The rules share this metadata, and an auto rule may have set a layer before it aborted - // (LALBlockCodegen writes `layer` into it); the fallback rules start from the layer as received. - metadata.setLayer(arrivedLayer); if (analyzeUntilClaimed(metadata, input, passes, AutoListeners::getFallback)) { return; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java index a000c8b3e340..211beaae9c55 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java @@ -31,6 +31,8 @@ import lombok.extern.slf4j.Slf4j; import java.util.HashMap; +import org.apache.skywalking.oap.log.analyzer.v2.compiler.LALScriptModel; +import org.apache.skywalking.oap.log.analyzer.v2.compiler.LALScriptParser; import org.apache.skywalking.oap.log.analyzer.v2.dsl.DSL; import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; import org.apache.skywalking.oap.log.analyzer.v2.dsl.debug.LalStaticBindingHook; @@ -135,7 +137,9 @@ public LogAnalysisListener parse(final LogMetadata metadata, if (effectiveInputType != null && !effectiveInputType.isInstance(input)) { continue; } - final ExecutionContext ctx = new ExecutionContext().init(metadata, input); + // Each rule gets its own copy: a rule's `layer` statement writes the metadata, and must not decide the + // layer of the other rules the log runs, which each start from the layer it arrived with. + final ExecutionContext ctx = new ExecutionContext().init(metadata.toBuilder().build(), input); ctx.setSourceTypeProvider(sourceTypeProvider); if (autoMode) { ctx.autoLayerMode(true); @@ -190,6 +194,14 @@ public static class Factory implements LogAnalysisListenerFactory, Service { private static final String AUTO_LAYER_PREFIX = "|"; + /** + * LogAnalyzer drops a log without a layer that no rule claims, logging it at debug only, since that is per + * log; this warns once instead, when the rules are loaded or changed. + */ + private static final String NO_FALLBACK_RULE = "No LAL rule declares `fallback: true`, so a log without a " + + "layer that no `layer: auto` rule claims is dropped. lal/default.yaml ships the `default-fallback` " + + "rule, which saves such logs under GENERAL; keep `default` in lalFiles, or declare a fallback rule."; + /** Serializes runtime mutations (addOrReplace / remove / suspend / resume). Startup * writes are single-threaded. */ private final Object writeLock = new Object(); @@ -259,6 +271,13 @@ public void loadStaticRules() throws Exception { c.getName(), c.getSourcePath(), c.getLayer()); continue; } + final String noLayer = savesWithoutLayer(c); + if (noLayer != null) { + // Such a rule dropped every log it saved; skip it rather than fail the boot. A runtime rule is + // rejected instead, by compile. + log.warn("{}; the rule is skipped", noLayer); + continue; + } final CompiledLAL compiled = compile(c); if (compiled.layer == null) { if (initAutoDsls.containsKey(c.getName()) || initFallbackDsls.containsKey(c.getName())) { @@ -281,6 +300,9 @@ public void loadStaticRules() throws Exception { // Publish: readers from now on see the startup-complete registry. this.dsls = initDsls; this.autoRules = new AutoRules(initAutoDsls, initFallbackDsls); + if (initFallbackDsls.isEmpty()) { + log.warn(NO_FALLBACK_RULE); + } } /** @@ -308,6 +330,10 @@ public CompiledLAL compile(final LALConfig c, "LAL rule " + c.getName() + " in " + c.getSourcePath() + " declares fallback: true, which only a layer: auto rule may"); } + final String noLayer = savesWithoutLayer(c); + if (noLayer != null) { + throw new ModuleStartException(noLayer); + } final LALSourceTypeProvider spiProvider = isAuto ? null : spiProviders.get(layer); final Class resolvedInputType = resolveInputType(c, spiProvider); final Class resolvedOutputType = resolveOutputType(c, spiProvider); @@ -322,6 +348,32 @@ public CompiledLAL compile(final LALConfig c, return new CompiledLAL(layer, c.getName(), dsl, c.isFallback()); } + /** + * A {@code layer: auto} rule decides the layer of the logs it saves, and FilterSpec#doSink drops a log its + * rule saved without one. So an auto rule with a {@code sink} must have a {@code layer} statement; one + * without a {@code sink} never saves a log (a rule that only extracts metrics) and needs none. + * + * @return why the rule is refused, or null when it is not an auto rule, has a {@code layer} statement, or + * has no {@code sink}. A script that does not parse is left to the compiler to report. + */ + static String savesWithoutLayer(final LALConfig c) { + if (!LALConfig.LAYER_AUTO.equalsIgnoreCase(c.getLayer())) { + return null; + } + final LALScriptModel model; + try { + model = LALScriptParser.parse(c.getDsl()); + } catch (final RuntimeException e) { + // Whatever the parser throws, compile reports it with the rule's name and file. + return null; + } + if (!model.hasSink() || model.assignsLayer()) { + return null; + } + return "LAL rule " + c.getName() + " in " + c.getSourcePath() + " is a layer: auto rule with a sink but " + + "no layer statement, so every log it saved would be dropped"; + } + /** * {@link Layer#nameOf} answers UNDEFINED for an unregistered name, which no log is routed to, so a * misspelled or missing layer would leave its rule silently never running. @@ -356,7 +408,7 @@ public void addOrReplace(final CompiledLAL compiled) { nextAuto.remove(compiled.ruleName); nextFallback.remove(compiled.ruleName); (compiled.fallback ? nextFallback : nextAuto).put(compiled.ruleName, compiled.dsl); - autoRules = new AutoRules(nextAuto, nextFallback); + publishAutoRules(new AutoRules(nextAuto, nextFallback)); } else { final Map> next = new HashMap<>(dsls); final Map layerMap = @@ -389,6 +441,18 @@ public DSL find(final Layer layer, final String ruleName) { return layerMap == null ? null : layerMap.get(ruleName); } + /** + * Replace the auto rules, warning when the change leaves no fallback rule, so a runtime change that removes + * the last one (or turns it into a plain auto rule) does not start dropping the logs without a layer unseen. + */ + private void publishAutoRules(final AutoRules next) { + final boolean hadFallback = !autoRules.fallback.isEmpty(); + autoRules = next; + if (hadFallback && next.fallback.isEmpty()) { + log.warn(NO_FALLBACK_RULE); + } + } + /** Runtime remove. No-op when the key isn't present. */ public void remove(final Layer layer, final String ruleName) { synchronized (writeLock) { @@ -401,7 +465,7 @@ public void remove(final Layer layer, final String ruleName) { final Map nextFallback = new HashMap<>(current.fallback); nextAuto.remove(ruleName); nextFallback.remove(ruleName); - autoRules = new AutoRules(nextAuto, nextFallback); + publishAutoRules(new AutoRules(nextAuto, nextFallback)); } else { final Map layerMap = dsls.get(layer); if (layerMap == null || !layerMap.containsKey(ruleName)) { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java index 2bb08473a0aa..6f9510b5c1f6 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java @@ -65,8 +65,9 @@ public void build() { public LogSinkListener parse(final LogMetadata metadata, final Object input, final ExecutionContext ctx) { - // Never empty here: a log without a layer is saved only by an auto rule, which must set one - // (FilterSpec#doSink drops the log otherwise). + // A registered layer here: this rule's copy of the metadata holds the layer the log arrived with, which + // LogAnalyzer checked, or one the rule's own `layer` statement decided, which FilterSpec#doSink checked (and + // an auto rule must decide one). final Layer layer = Layer.valueOf(metadata.getLayer()); final long timeBucket = TimeBucket.getTimeBucket(System.currentTimeMillis(), DownSampling.Minute); // to service traffic diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALSinkLayerTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALSinkLayerTest.java new file mode 100644 index 000000000000..fd5ee610ccc6 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALSinkLayerTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.v2.compiler; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.skywalking.apm.network.logging.v3.JSONLog; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.LalExpression; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.spec.filter.FilterSpec; +import org.apache.skywalking.oap.log.analyzer.v2.module.LogAnalyzerModule; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleProvider; +import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogSinkListener; +import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogSinkListenerFactory; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.source.LogBuilder; +import org.apache.skywalking.oap.server.core.source.LogMetadata; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The layer a rule saves a log under, as FilterSpec#doSink checks it. An auto rule saves a log only under a layer it + * decided itself, read from its own decision, not from its {@link LogMetadata}: these tests give two rules one + * metadata, which LogFilterListener no longer does (each rule gets its own copy), so the check holds even if a rule's + * metadata held a layer it did not decide. And any rule saves a log only under a registered layer, which the sink + * listeners turn into a {@code Layer}. + */ +class LALSinkLayerTest { + private static final String SETS_LAYER = "filter { extractor { layer \"IOS\" } sink {} }"; + private static final String SETS_NO_LAYER = "filter { extractor { } sink {} }"; + /** + * A computed layer: the body's {@code kind}, null when the body has none. + */ + private static final String KIND_LAYER = "filter { json {} extractor { layer parsed.kind as String } sink {} }"; + + /** + * The layer each saved log was saved under, in order. + */ + private final List saved = new ArrayList<>(); + private FilterSpec filterSpec; + private LogMetadata metadata; + + @BeforeEach + void setUp() throws Exception { + filterSpec = filterSpec(); + metadata = LogMetadata.builder().service("svc").layer("").timestamp(1L).build(); + } + + @Test + void aRuleThatDecidedALayerSavesTheLog() throws Exception { + final ExecutionContext ctx = run(SETS_LAYER); + + assertEquals("IOS", ctx.layer()); + assertEquals("IOS", metadata.getLayer(), "the layer statement writes the rule's metadata"); + assertEquals(List.of("IOS"), saved); + } + + @Test + void aRuleWithoutALayerIsNotSavedUnderAnotherRulesLayer() throws Exception { + run(SETS_LAYER); + final ExecutionContext noLayer = run(SETS_NO_LAYER); + + assertNull(noLayer.layer()); + assertFalse(noLayer.shouldSave(), "dropped, though the metadata holds the other rule's IOS"); + assertEquals(List.of("IOS"), saved, "saved once, by the rule that decided the layer"); + } + + @Test + void aRuleWithoutALayerIsDroppedWhicheverRuleRunsFirst() throws Exception { + final ExecutionContext noLayer = run(SETS_NO_LAYER); + run(SETS_LAYER); + + assertFalse(noLayer.shouldSave()); + assertEquals(List.of("IOS"), saved); + } + + @Test + void anEmptyLayerIsNoLayer() throws Exception { + final ExecutionContext ctx = run("filter { extractor { layer sourceAttribute(\"absent\") } sink {} }"); + + assertTrue(ctx.layer() == null || ctx.layer().isEmpty()); + assertFalse(ctx.shouldSave()); + assertTrue(saved.isEmpty()); + } + + @Test + void aLayerRoutedRuleThatSetsAnEmptyLayerIsDropped() throws Exception { + metadata.setLayer("GENERAL"); + final ExecutionContext ctx = runRouted( + "filter { extractor { layer sourceAttribute(\"absent\") } sink {} }", "{}"); + + assertFalse(ctx.shouldSave(), "dropped, not thrown by Layer.valueOf in the sink"); + assertTrue(saved.isEmpty()); + } + + @Test + void aLayerRoutedRuleThatSetsANullLayerIsDropped() throws Exception { + metadata.setLayer("GENERAL"); + final ExecutionContext ctx = runRouted(KIND_LAYER, "{}"); + + assertTrue(ctx.isLayerDecided(), "the statement ran, though its value is null"); + assertNull(ctx.layer()); + assertNull(metadata.getLayer()); + assertFalse(ctx.shouldSave()); + assertTrue(saved.isEmpty()); + } + + @Test + void aRuleThatSetsAnUnregisteredLayerIsDropped() throws Exception { + metadata.setLayer("GENERAL"); + final ExecutionContext routed = runRouted(KIND_LAYER, "{\"kind\":\"FOO\"}"); + metadata.setLayer(""); + final ExecutionContext auto = run(KIND_LAYER, "{\"kind\":\"IOSS\"}"); + + assertFalse(routed.shouldSave()); + assertFalse(auto.shouldSave(), "a non-empty layer passes the auto check, but is not registered"); + assertTrue(saved.isEmpty()); + } + + @Test + void aRuleThatSetsARegisteredComputedLayerSavesTheLog() throws Exception { + metadata.setLayer("GENERAL"); + runRouted(KIND_LAYER, "{\"kind\":\"IOS\"}"); + + assertEquals(List.of("IOS"), saved); + } + + @Test + void aLayerRoutedRuleWithoutALayerStatementSavesUnderTheArrivedLayer() throws Exception { + metadata.setLayer("GENERAL"); + final ExecutionContext ctx = runRouted("filter { sink {} }", "{}"); + + assertFalse(ctx.isLayerDecided()); + assertEquals(List.of("GENERAL"), saved); + } + + private ExecutionContext run(final String dsl) throws Exception { + return run(dsl, "{}"); + } + + /** + * Run in auto mode, as a rule of the auto or the fallback pass. + */ + private ExecutionContext run(final String dsl, final String jsonBody) throws Exception { + return execute(dsl, jsonBody, true); + } + + /** + * Run as a rule declared for the log's layer. + */ + private ExecutionContext runRouted(final String dsl, final String jsonBody) throws Exception { + return execute(dsl, jsonBody, false); + } + + private ExecutionContext execute(final String dsl, final String jsonBody, + final boolean autoMode) throws Exception { + final LALClassGenerator generator = new LALClassGenerator(); + generator.setOutputType(LogBuilder.class); + final LalExpression expr = generator.compile(dsl); + final LogData.Builder input = LogData.newBuilder() + .setService("svc") + .setBody(LogDataBody.newBuilder() + .setJson(JSONLog.newBuilder().setJson(jsonBody))); + final ExecutionContext ctx = new ExecutionContext().init(metadata, input); + ctx.autoLayerMode(autoMode); + expr.execute(filterSpec, ctx); + return ctx; + } + + private FilterSpec filterSpec() throws Exception { + final ModuleManager manager = mock(ModuleManager.class); + setField(manager, "isInPrepareStage", false); + when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class)); + final ModuleProviderHolder logHolder = mock(ModuleProviderHolder.class); + final LogAnalyzerModuleProvider logProvider = mock(LogAnalyzerModuleProvider.class); + when(logProvider.getMetricConverts()).thenReturn(Collections.emptyList()); + when(logHolder.provider()).thenReturn(logProvider); + when(manager.find(LogAnalyzerModule.NAME)).thenReturn(logHolder); + final ModuleProviderHolder coreHolder = mock(ModuleProviderHolder.class); + when(coreHolder.provider()).thenReturn(mock(ModuleServiceHolder.class)); + when(manager.find(CoreModule.NAME)).thenReturn(coreHolder); + + final FilterSpec spec = new FilterSpec(manager, new LogAnalyzerModuleConfig()); + final LogSinkListenerFactory recording = () -> new LogSinkListener() { + private String layer; + + @Override + public LogSinkListener parse(final LogMetadata metadata, final Object input, final ExecutionContext ctx) { + layer = metadata.getLayer(); + return this; + } + + @Override + public void build() { + saved.add(layer); + } + }; + setField(spec, "sinkListenerFactories", List.of(recording)); + return spec; + } + + private static void setField(final Object target, final String name, final Object value) throws Exception { + Class clazz = target.getClass(); + while (clazz != null) { + try { + final Field field = clazz.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + return; + } catch (final NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(name); + } +} diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerRoutingTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerRoutingTest.java index 0a2696281d8e..9daa68e6ea28 100644 --- a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerRoutingTest.java +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerRoutingTest.java @@ -26,11 +26,8 @@ import org.apache.skywalking.oap.server.core.source.LogMetadata; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -87,21 +84,6 @@ void fallbackRulesRunWhenNoAutoRuleClaimedTheLog() { verify(general, never()).build(); } - @Test - void fallbackRulesDoNotSeeALayerAnAbortedAutoRuleSet() { - // An auto rule sets a layer on the shared metadata, then aborts (claimed() stays false). - doAnswer(inv -> { - ((LogMetadata) inv.getArgument(0)).setLayer("IOS"); - return auto; - }).when(auto).parse(any(), any()); - - analyzer.doAnalysis(log(""), null); - - final ArgumentCaptor seen = ArgumentCaptor.forClass(LogMetadata.class); - verify(fallback).parse(seen.capture(), any()); - assertEquals("", seen.getValue().getLayer()); - } - @Test void logNoRuleClaimedIsNotRoutedToGeneral() { analyzer.doAnalysis(log(""), null); diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerAutoLayerStatementTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerAutoLayerStatementTest.java new file mode 100644 index 000000000000..b1806831a9d5 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerAutoLayerStatementTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; + +import org.apache.skywalking.oap.log.analyzer.v2.provider.LALConfig; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * A {@code layer: auto} rule that can save a log must decide its layer: one with a {@code sink} but no + * {@code layer} statement is skipped at boot and rejected as a runtime rule. + */ +class LogFilterListenerAutoLayerStatementTest { + + private static LALConfig rule(final String layer, final String dsl) { + final LALConfig c = new LALConfig(); + c.setName("r"); + c.setSourcePath("lal/test.yaml"); + c.setLayer(layer); + c.setDsl(dsl); + return c; + } + + @Test + void anAutoRuleThatSavesWithoutALayerIsRefused() { + final String reason = LogFilterListener.Factory.savesWithoutLayer( + rule("auto", "filter { extractor { service \"svc\" } sink {} }")); + + assertNotNull(reason); + assertTrue(reason.contains("lal/test.yaml") && reason.contains("layer statement"), reason); + } + + @Test + void aLayerStatementOnAnyBranchIsEnough() { + assertNull(LogFilterListener.Factory.savesWithoutLayer( + rule("auto", "filter { extractor { layer \"IOS\" } sink {} }"))); + assertNull(LogFilterListener.Factory.savesWithoutLayer(rule("auto", + "filter { if (tag(\"a\") == \"1\") { extractor { layer \"IOS\" } } else { abort {} } sink {} }"))); + assertNull(LogFilterListener.Factory.savesWithoutLayer(rule("auto", + "filter { extractor { if (tag(\"a\") == \"1\") { layer \"IOS\" } } sink {} }"))); + } + + @Test + void anAutoRuleWithoutASinkNeedsNoLayer() { + // It never saves a log, e.g. one that only extracts metrics. + assertNull(LogFilterListener.Factory.savesWithoutLayer(rule("auto", "filter { extractor { service \"svc\" } }"))); + } + + @Test + void aLayeredRuleNeedsNoLayerStatement() { + assertNull(LogFilterListener.Factory.savesWithoutLayer(rule("GENERAL", "filter { sink {} }"))); + } + + @Test + void aScriptThatDoesNotParseIsLeftToTheCompiler() { + assertNull(LogFilterListener.Factory.savesWithoutLayer(rule("auto", "filter { sink {"))); + } + + @Test + void aRuntimeAutoRuleThatSavesWithoutALayerIsRejected() throws Exception { + final LogFilterListener.Factory factory = + new LogFilterListener.Factory(mock(ModuleManager.class), mock(LogAnalyzerModuleConfig.class)); + + final ModuleStartException e = assertThrows( + ModuleStartException.class, () -> factory.compile(rule("auto", "filter { sink {} }"))); + assertTrue(e.getMessage().contains("layer statement"), e.getMessage()); + } +} diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerPerRuleMetadataTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerPerRuleMetadataTest.java new file mode 100644 index 000000000000..559aa407ccea --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListenerPerRuleMetadataTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.DSL; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; +import org.apache.skywalking.oap.server.core.source.LogMetadata; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +/** + * Each rule a log runs starts from the layer the log arrived with: a rule's {@code layer} statement writes only that + * rule's copy of the metadata, never the layer another rule saves the log under. + */ +class LogFilterListenerPerRuleMetadataTest { + private final List seen = new ArrayList<>(); + + @Test + void aValidLayerOfOneRuleIsNotSeenByTheNext() { + final LogMetadata metadata = log("GENERAL"); + + run(metadata, false, decides("MESH"), records()); + + assertEquals(List.of("GENERAL"), seen); + assertEquals("GENERAL", metadata.getLayer(), "the log's own metadata is not written by a rule"); + } + + @Test + void anInvalidLayerOfOneRuleIsNotSeenByTheNext() { + run(log("GENERAL"), false, decides("bogus"), records(), decides(null), records(), decides(""), records()); + + assertEquals(List.of("GENERAL", "GENERAL", "GENERAL"), seen); + } + + @Test + void theRuleThatDecidedALayerSeesIt() { + run(log("GENERAL"), false, decidesAndRecords("MESH"), records()); + + assertEquals(List.of("MESH", "GENERAL"), seen); + } + + @Test + void anAutoRulesLayerIsNotSeenByAnotherAutoRule() { + run(log(""), true, decides("IOS"), records()); + + assertEquals(List.of(""), seen, "the other auto rule must still decide a layer of its own"); + } + + @Test + void theFallbackRulesDoNotSeeALayerAnAbortedAutoRuleSet() { + final LogMetadata metadata = log(""); + final DSL setsAndAborts = mock(DSL.class); + doAnswer(inv -> { + ((ExecutionContext) inv.getArgument(0)).layer("IOS").abort(); + return null; + }).when(setsAndAborts).evaluate(any()); + + // LogAnalyzer runs the auto pass and, as no rule claimed the log, the fallback pass on the same metadata. + final LogFilterListener auto = run(metadata, true, setsAndAborts); + run(metadata, true, records()); + + assertFalse(auto.claimed()); + assertEquals(List.of(""), seen, "the fallback rule starts from the layer the log arrived with"); + assertEquals("", metadata.getLayer()); + } + + private LogFilterListener run(final LogMetadata metadata, final boolean autoMode, final DSL... rules) { + final LogFilterListener listener = new LogFilterListener(Arrays.asList(rules), autoMode, null); + listener.parse(metadata, null); + listener.build(); + return listener; + } + + private static LogMetadata log(final String layer) { + return LogMetadata.builder().service("svc").layer(layer).timestamp(1L).build(); + } + + /** + * A rule whose {@code layer} statement runs, as the generated code calls it. + */ + private static DSL decides(final String layer) { + final DSL rule = mock(DSL.class); + doAnswer(inv -> { + ((ExecutionContext) inv.getArgument(0)).layer(layer); + return null; + }).when(rule).evaluate(any()); + return rule; + } + + private DSL decidesAndRecords(final String layer) { + final DSL rule = mock(DSL.class); + doAnswer(inv -> { + final ExecutionContext ctx = inv.getArgument(0); + ctx.layer(layer); + seen.add(ctx.metadata().getLayer()); + return null; + }).when(rule).evaluate(any()); + return rule; + } + + /** + * A rule without a {@code layer} statement, recording the layer its sink would save the log under. + */ + private DSL records() { + final DSL rule = mock(DSL.class); + doAnswer(inv -> { + seen.add(((ExecutionContext) inv.getArgument(0)).metadata().getLayer()); + return null; + }).when(rule).evaluate(any()); + return rule; + } +} diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplier.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplier.java index ab7d68cf9e3a..6d0fba0ff1e2 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplier.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplier.java @@ -175,10 +175,11 @@ public Applied apply(final String yamlContent, final String sourceName, // during phase 2, the caller's rollback list is accurate. final List compiled = new ArrayList<>(rules.size()); for (final LALConfig c : rules) { - // Shared with the boot loader, not restated: runtime rule names carry no extension, - // and the sourceName it derives keys the dsl-debugging registry, so a hot update must - // land on the same key as its disk-loaded twin rather than beside it. - LALConfigs.stampSource(c, sourceName); + // Stamped through the boot loader's stampSource, so a runtime rule carries the same source + // coordinates as its disk-loaded twin. It takes the bare rule name: sourceName is + // "catalog/name", and stampSource adds the catalog itself. The dsl-debugging registry key is + // built separately, from the bare rule name (see stampSource). + LALConfigs.stampSource(c, ruleNameStr); try { compiled.add(factory.compile(c, pool, ruleLoader)); } catch (final Throwable t) { diff --git a/oap-server/server-admin/runtime-rule/src/test/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplierSourceTest.java b/oap-server/server-admin/runtime-rule/src/test/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplierSourceTest.java new file mode 100644 index 000000000000..de81a56120f9 --- /dev/null +++ b/oap-server/server-admin/runtime-rule/src/test/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/LalFileApplierSourceTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.runtimerule.apply; + +import java.util.ArrayList; +import java.util.List; +import javassist.ClassPool; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LALConfig; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LALConfigs; +import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogFilterListener; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** + * A runtime rule is stamped with the same source coordinates as the same file loaded at boot: the applier's + * {@code catalog/name} source must not leave the catalog in the name, or the attribution reads {@code lal/lal/...}. + */ +class LalFileApplierSourceTest { + private static final String YAML = + "rules:\n" + + " - name: default\n" + + " layer: GENERAL\n" + + " dsl: |\n" + + " filter {\n" + + " sink {\n" + + " }\n" + + " }\n"; + + @Test + void aRuntimeRuleIsStampedAsItsBootLoadedTwin() throws Exception { + final List compiled = new ArrayList<>(); + final LogFilterListener.Factory factory = Mockito.mock(LogFilterListener.Factory.class); + when(factory.compile(any(LALConfig.class), any(ClassPool.class), any(ClassLoader.class))) + .thenAnswer(inv -> { + final LALConfig c = inv.getArgument(0); + compiled.add(c); + return new LogFilterListener.Factory.CompiledLAL(Layer.nameOf(c.getLayer()), c.getName(), null); + }); + + new LalFileApplier(factory).apply(YAML, "lal/default", "h"); + + final LALConfig bootLoaded = new LALConfig(); + LALConfigs.stampSource(bootLoaded, "default.yaml"); + assertEquals("lal/default.yaml", compiled.get(0).getSourcePath()); + assertEquals(bootLoaded.getSourcePath(), compiled.get(0).getSourcePath()); + assertEquals(bootLoaded.getSourceName(), compiled.get(0).getSourceName()); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagType.java index f946aa4dc60f..ad4699c138ad 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagType.java @@ -23,5 +23,9 @@ public enum TagType { LOG, ZIPKIN, ALARM, - OTLP + OTLP, + /** + * The tags of the logs received through OTLP, apart from {@link #LOG}, the logs of every other source. + */ + OTLP_LOG } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/ConfigService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/ConfigService.java index b8d74513d4af..51146327303d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/ConfigService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/ConfigService.java @@ -18,7 +18,15 @@ package org.apache.skywalking.oap.server.core.config; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import lombok.Getter; +import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleProvider; import org.apache.skywalking.oap.server.library.module.Service; @@ -33,6 +41,7 @@ public class ConfigService implements Service { private final int metricsDataTTL; private final int recordDataTTL; private final int persistentPeriod; + private final Set searchableLogsTagKeys = ConcurrentHashMap.newKeySet(); public ConfigService(CoreModuleConfig moduleConfig, ModuleProvider provider) { this.gRPCHost = moduleConfig.getGRPCHost(); @@ -43,9 +52,48 @@ public ConfigService(CoreModuleConfig moduleConfig, ModuleProvider provider) { moduleConfig.setSearchableTracesTagsWatcher(this.searchableTracesTags); this.searchableLogsTags = moduleConfig.getSearchableLogsTags(); + this.searchableLogsTagKeys.addAll(splitTagKeys(this.searchableLogsTags)); this.searchableAlarmTags = moduleConfig.getSearchableAlarmTags(); this.metricsDataTTL = moduleConfig.getMetricsDataTTL(); this.recordDataTTL = moduleConfig.getRecordDataTTL(); this.persistentPeriod = moduleConfig.getPersistentPeriod(); } + + /** + * The log tag keys searchable on any log: {@link #searchableLogsTags}, which the logs of the SkyWalking native + * protocols index, plus the keys a receiver with its own list registered through + * {@link #addSearchableLogsTagKeys}. All logs share one table, so a log query accepts a key of either. + * + * @return a live, read-only view + */ + public Set getSearchableLogsTagKeys() { + return Collections.unmodifiableSet(searchableLogsTagKeys); + } + + /** + * Split a comma-separated tag key list, trimming each key and dropping empty ones, so {@code "level, http.status_code"} + * and {@code "level,http.status_code"} name the same keys. + * + * @param tagKeys a comma-separated list, or null + * @return the keys in order + */ + public static List splitTagKeys(final String tagKeys) { + if (tagKeys == null) { + return Collections.emptyList(); + } + return Arrays.stream(tagKeys.split(Const.COMMA)) + .map(String::trim) + .filter(key -> !key.isEmpty()) + .collect(Collectors.toList()); + } + + /** + * Register the tag keys a receiver indexes on its own logs in place of {@link #searchableLogsTags}, so the log + * query accepts them. + * + * @param keys the receiver's searchable log tag keys + */ + public void addSearchableLogsTagKeys(final Collection keys) { + searchableLogsTagKeys.addAll(keys); + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java index ba0901388387..0466deba176a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java @@ -19,7 +19,10 @@ package org.apache.skywalking.oap.server.core.query; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; @@ -47,13 +50,53 @@ private ITagAutoCompleteQueryDAO getTagAutoCompleteQueryDAO() { public Set queryTagAutocompleteKeys(final TagType tagType, final Duration duration) throws IOException { - return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys(tagType, config.getAutocompleteTagKeysQueryMaxSize(), duration); + return queryTagAutocompleteKeys(List.of(tagType), duration); } public Set queryTagAutocompleteValues(final TagType tagType, final String tagKey, final Duration duration) throws IOException { + return queryTagAutocompleteValues(List.of(tagType), tagKey, duration); + } + + /** + * @param type the log tag type a query asks for, or null for none + * @return the tag types to read: the one asked for, or those of every log source + */ + public static List logTagTypes(final TagType type) { + return type == null ? List.of(TagType.LOG, TagType.OTLP_LOG) : List.of(type); + } + + /** + * The keys of any of the tag types, read in one storage query within one limit. + */ + public Set queryTagAutocompleteKeys(final List tagTypes, + final Duration duration) throws IOException { + if (tagTypes.isEmpty()) { + // No tag type, no tag; the DAO takes a non-empty list, as a storage cannot express an empty IN. + return Collections.emptySet(); + } + return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys( + namesOf(tagTypes), config.getAutocompleteTagKeysQueryMaxSize(), duration); + } + + /** + * The values of a key under any of the tag types, read in one storage query within one limit. + */ + public Set queryTagAutocompleteValues(final List tagTypes, + final String tagKey, + final Duration duration) throws IOException { + if (tagTypes.isEmpty()) { + return Collections.emptySet(); + } return getTagAutoCompleteQueryDAO().queryTagAutocompleteValues( - tagType, tagKey, config.getAutocompleteTagValuesQueryMaxSize(), duration); + namesOf(tagTypes), tagKey, config.getAutocompleteTagValuesQueryMaxSize(), duration); + } + + /** + * The names the storage keeps the tag types by. + */ + private static List namesOf(final List tagTypes) { + return tagTypes.stream().map(TagType::name).collect(Collectors.toList()); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java index 38cf7f455d1b..f745b3fe5df7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java @@ -21,7 +21,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -35,7 +34,6 @@ import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.apm.network.logging.v3.LogDataBody; import org.apache.skywalking.apm.network.logging.v3.LogTags; -import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; @@ -81,6 +79,15 @@ public class LogBuilder implements LALOutputBuilder { private int spanId = -1; private final List lalTags = new ArrayList<>(); + /** + * The searchable tag keys the log's receiver gave, in place of {@link #SEARCHABLE_TAG_KEYS}; null when it gave + * none. + */ + private Collection searchableTagKeys; + /** + * The autocomplete type the log's searchable tags are written under, from its receiver. + */ + private TagType tagType = TagType.LOG; public void setSpanId(final String spanId) { if (spanId != null) { @@ -127,8 +134,7 @@ protected void ensureInitialized(final ModuleManager moduleManager) { final ConfigService configService = moduleManager.find(CoreModule.NAME) .provider() .getService(ConfigService.class); - SEARCHABLE_TAG_KEYS = Arrays.asList( - configService.getSearchableLogsTags().split(Const.COMMA)); + SEARCHABLE_TAG_KEYS = ConfigService.splitTagKeys(configService.getSearchableLogsTags()); INITIALIZED = true; } } @@ -168,6 +174,8 @@ protected void initFromMetadata(final LogMetadata metadata) { if (this.timestamp == 0) { this.timestamp = metadata.getTimestamp(); } + this.searchableTagKeys = metadata.getSearchableTagKeys(); + this.tagType = metadata.getTagType(); } @Override @@ -260,18 +268,19 @@ private byte[] buildMergedTagsRawData() { private Collection collectSearchableTags() { final HashSet result = new HashSet<>(); - if (SEARCHABLE_TAG_KEYS != null) { + final Collection keys = searchableTagKeys != null ? searchableTagKeys : SEARCHABLE_TAG_KEYS; + if (keys != null) { // Tags from original LogData if (logData != null) { logData.getTags().getDataList().forEach(kv -> { - if (SEARCHABLE_TAG_KEYS.contains(kv.getKey())) { + if (keys.contains(kv.getKey())) { addSearchableTag(result, kv.getKey(), kv.getValue()); } }); } // Tags added by LAL extractor for (final String[] kv : lalTags) { - if (SEARCHABLE_TAG_KEYS.contains(kv[0])) { + if (keys.contains(kv[0])) { addSearchableTag(result, kv[0], kv[1]); } } @@ -408,7 +417,7 @@ private void addAutocompleteTags(final SourceReceiver sourceReceiver, final TagAutocomplete tagAutocomplete = new TagAutocomplete(); tagAutocomplete.setTagKey(tag.getKey()); tagAutocomplete.setTagValue(tag.getValue()); - tagAutocomplete.setTagType(TagType.LOG); + tagAutocomplete.setTagType(tagType); tagAutocomplete.setTimeBucket(TimeBucket.getMinuteTimeBucket(log.getTimestamp())); sourceReceiver.receive(tagAutocomplete); }); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogMetadata.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogMetadata.java index a6aab1155a25..8eddbd2a9c01 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogMetadata.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogMetadata.java @@ -17,10 +17,12 @@ package org.apache.skywalking.oap.server.core.source; +import java.util.Collection; import java.util.Collections; import java.util.Map; import lombok.Builder; import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; /** * Uniform metadata carrier for log analysis. Receivers build this from their @@ -29,7 +31,7 @@ *

For building from {@code LogData} protobuf, see {@link LogMetadataUtils}. */ @Data -@Builder +@Builder(toBuilder = true) public class LogMetadata { private String service; private String serviceInstance; @@ -47,6 +49,19 @@ public class LogMetadata { @Builder.Default private Map sourceAttributes = Collections.emptyMap(); + /** + * The tag keys indexed as searchable on this log, or null for core's {@code searchableLogsTags}. A receiver with + * its own list, such as {@code receiver-otel.otlpLogSearchableTags}, sets it. + */ + private Collection searchableTagKeys; + + /** + * The autocomplete type the log's tags are written under: {@link TagType#OTLP_LOG} for the logs received through + * OTLP, {@link TagType#LOG} for every other source. + */ + @Builder.Default + private TagType tagType = TagType.LOG; + @Data @Builder public static class TraceContext { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java index 2f3d7c4283f4..764d65a417c9 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java @@ -19,17 +19,35 @@ package org.apache.skywalking.oap.server.core.storage.query; import java.io.IOException; +import java.util.List; import java.util.Set; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.library.module.Service; public interface ITagAutoCompleteQueryDAO extends Service { - Set queryTagAutocompleteKeys(final TagType tagType, + /** + * @param tagTypes the names of the tag types to read ({@link TagType#name()}), the tags of any of them; + * not empty + * @param limit the most keys returned, across all the tag types + * @param duration the time range + * @return the distinct tag keys + * @throws IOException when the storage cannot be read + */ + Set queryTagAutocompleteKeys(final List tagTypes, final int limit, final Duration duration) throws IOException; - Set queryTagAutocompleteValues(final TagType tagType, + /** + * @param tagTypes the names of the tag types to read ({@link TagType#name()}), the tags of any of them; + * not empty + * @param tagKey the key whose values are read + * @param limit the most values returned, across all the tag types + * @param duration the time range + * @return the distinct values of the key + * @throws IOException when the storage cannot be read + */ + Set queryTagAutocompleteValues(final List tagTypes, final String tagKey, final int limit, final Duration duration) throws IOException; diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/ConfigServiceSplitTagKeysTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/ConfigServiceSplitTagKeysTest.java new file mode 100644 index 000000000000..3659a886445b --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/ConfigServiceSplitTagKeysTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config; + +import java.util.List; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ConfigServiceSplitTagKeysTest { + @Test + void keysAreTrimmedAndEmptyOnesDropped() { + assertEquals(List.of("level", "http.status_code"), ConfigService.splitTagKeys(" level , http.status_code,,")); + assertEquals(List.of("level", "http.status_code"), ConfigService.splitTagKeys("level,http.status_code")); + } + + @Test + void anEmptyOrMissingListHasNoKeys() { + assertEquals(List.of(), ConfigService.splitTagKeys("")); + assertEquals(List.of(), ConfigService.splitTagKeys(" , ")); + assertEquals(List.of(), ConfigService.splitTagKeys(null)); + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryServiceLogTypeTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryServiceLogTypeTest.java new file mode 100644 index 000000000000..2bfe6a7da715 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryServiceLogTypeTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import java.util.List; +import java.util.Set; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The log tag autocomplete reads one log source, or both in one storage query, passing the storage the tag types' + * names. + */ +class TagAutoCompleteQueryServiceLogTypeTest { + private final Duration duration = new Duration(); + private ITagAutoCompleteQueryDAO dao; + private TagAutoCompleteQueryService service; + + @BeforeEach + void setUp() throws Exception { + dao = mock(ITagAutoCompleteQueryDAO.class); + when(dao.queryTagAutocompleteKeys(anyList(), anyInt(), any())).thenReturn(Set.of()); + when(dao.queryTagAutocompleteValues(anyList(), anyString(), anyInt(), any())).thenReturn(Set.of()); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(services.getService(ITagAutoCompleteQueryDAO.class)).thenReturn(dao); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + when(provider.provider()).thenReturn(services); + final ModuleManager manager = mock(ModuleManager.class); + when(manager.find(StorageModule.NAME)).thenReturn(provider); + final CoreModuleConfig config = new CoreModuleConfig(); + config.setAutocompleteTagKeysQueryMaxSize(20); + config.setAutocompleteTagValuesQueryMaxSize(30); + service = new TagAutoCompleteQueryService(manager, config); + } + + @Test + void withoutATypeBothLogSourcesAreReadInOneQuery() throws Exception { + assertEquals(List.of(TagType.LOG, TagType.OTLP_LOG), TagAutoCompleteQueryService.logTagTypes(null)); + + service.queryTagAutocompleteKeys(TagAutoCompleteQueryService.logTagTypes(null), duration); + service.queryTagAutocompleteValues(TagAutoCompleteQueryService.logTagTypes(null), "level", duration); + + verify(dao, times(1)).queryTagAutocompleteKeys(List.of("LOG", "OTLP_LOG"), 20, duration); + verify(dao, times(1)).queryTagAutocompleteValues(List.of("LOG", "OTLP_LOG"), "level", 30, duration); + } + + @Test + void aTypeReadsThatSourceOnly() throws Exception { + service.queryTagAutocompleteKeys(TagAutoCompleteQueryService.logTagTypes(TagType.OTLP_LOG), duration); + + verify(dao).queryTagAutocompleteKeys(eq(List.of("OTLP_LOG")), eq(20), any()); + } + + @Test + void noTagTypeReadsNothing() throws Exception { + assertEquals(Set.of(), service.queryTagAutocompleteKeys(List.of(), duration)); + assertEquals(Set.of(), service.queryTagAutocompleteValues(List.of(), "level", duration)); + + verify(dao, never()).queryTagAutocompleteKeys(anyList(), anyInt(), any()); + verify(dao, never()).queryTagAutocompleteValues(anyList(), anyString(), anyInt(), any()); + } + + @Test + void aSingleTypeQueryIsAListOfOne() throws Exception { + service.queryTagAutocompleteValues(TagType.TRACE, "http.method", duration); + + verify(dao).queryTagAutocompleteValues(List.of("TRACE"), "http.method", 30, duration); + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/source/LogBuilderSearchableTagsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/source/LogBuilderSearchableTagsTest.java new file mode 100644 index 000000000000..85d08b75d1ad --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/source/LogBuilderSearchableTagsTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A log indexes the searchable tag keys its receiver gave, and core's {@code searchableLogsTags} when it gave none. + */ +class LogBuilderSearchableTagsTest { + /** + * Written with a space and a trailing comma, which the keys are trimmed of. + */ + private static final String CORE_KEYS = "level, http.status_code,"; + + private static ModuleManager MANAGER; + + @BeforeAll + static void setUp() { + final ConfigService configService = mock(ConfigService.class); + when(configService.getSearchableLogsTags()).thenReturn(CORE_KEYS); + final NamingControl namingControl = mock(NamingControl.class); + when(namingControl.formatServiceName(anyString())).thenAnswer(inv -> inv.getArgument(0)); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(services.getService(ConfigService.class)).thenReturn(configService); + when(services.getService(NamingControl.class)).thenReturn(namingControl); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + when(provider.provider()).thenReturn(services); + MANAGER = mock(ModuleManager.class); + when(MANAGER.find(CoreModule.NAME)).thenReturn(provider); + } + + @Test + void theReceiversKeysReplaceCoresList() { + final LogMetadata metadata = metadata(); + metadata.setSearchableTagKeys(Set.of("severity", "http.response.status_code")); + + final Set searchable = searchableTagsOf(metadata); + + assertEquals(Set.of("severity=SEVERITY_NUMBER_ERROR", "http.response.status_code=502"), searchable, + "level and http.status_code are core's keys, which these logs do not index"); + } + + @Test + void aLogWithoutReceiverKeysIndexesCoresList() { + final Set searchable = searchableTagsOf(metadata()); + + assertEquals(Set.of("level=ERROR", "http.status_code=500"), searchable); + } + + @Test + void anEmptyReceiverListIndexesNothing() { + final LogMetadata metadata = metadata(); + metadata.setSearchableTagKeys(Set.of()); + + assertTrue(searchableTagsOf(metadata).isEmpty()); + } + + @Test + void theAutocompleteIsWrittenUnderTheReceiversTagType() { + final LogMetadata otlp = metadata(); + otlp.setTagType(TagType.OTLP_LOG); + + assertEquals(Set.of("OTLP_LOG"), autocompleteTypesOf(otlp)); + assertEquals(Set.of("LOG"), autocompleteTypesOf(metadata()), "a log whose receiver gave no type"); + } + + private static Set autocompleteTypesOf(final LogMetadata metadata) { + final SourceReceiver receiver = mock(SourceReceiver.class); + final LogBuilder builder = new LogBuilder(); + builder.init(metadata, LogData.newBuilder().setService("checkout") + .setTags(LogTags.newBuilder().addData(tag("level", "ERROR"))).build(), MANAGER); + builder.complete(receiver); + + final ArgumentCaptor sources = ArgumentCaptor.forClass(ISource.class); + verify(receiver, atLeastOnce()).receive(sources.capture()); + return sources.getAllValues().stream() + .filter(TagAutocomplete.class::isInstance) + .map(source -> ((TagAutocomplete) source).getTagType().name()) + .collect(Collectors.toSet()); + } + + private static LogMetadata metadata() { + return LogMetadata.builder().service("checkout").timestamp(1_700_000_000_000L).build(); + } + + private static Set searchableTagsOf(final LogMetadata metadata) { + final LogData input = LogData.newBuilder() + .setService("checkout") + .setTags(LogTags.newBuilder() + .addData(tag("level", "ERROR")) + .addData(tag("http.status_code", "500")) + .addData(tag("severity", "SEVERITY_NUMBER_ERROR")) + .addData(tag("http.response.status_code", "502"))) + .build(); + final LogBuilder builder = new LogBuilder(); + builder.init(metadata, input, MANAGER); + return builder.toLog().getTags().stream().map(Object::toString).collect(Collectors.toSet()); + } + + private static KeyStringValuePair tag(final String key, final String value) { + return KeyStringValuePair.newBuilder().setKey(key).setValue(value).build(); + } +} diff --git a/oap-server/server-query-plugin/logql-plugin/src/main/java/org/apache/skywalking/oap/query/logql/handler/LogQLApiHandler.java b/oap-server/server-query-plugin/logql-plugin/src/main/java/org/apache/skywalking/oap/query/logql/handler/LogQLApiHandler.java index 0af32bfcfd8b..56293ee1729f 100644 --- a/oap-server/server-query-plugin/logql-plugin/src/main/java/org/apache/skywalking/oap/query/logql/handler/LogQLApiHandler.java +++ b/oap-server/server-query-plugin/logql-plugin/src/main/java/org/apache/skywalking/oap/query/logql/handler/LogQLApiHandler.java @@ -56,7 +56,6 @@ import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.DurationUtils; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.TagAutoCompleteQueryService; @@ -92,7 +91,7 @@ public HttpResponse labels( labelsQueryRsp.setStatus(ResultStatus.SUCCESS); Duration duration = DurationUtils.timestamp2Duration(nano2Millis(start), nano2Millis(end)); - tagAutoCompleteQueryService.queryTagAutocompleteKeys(TagType.LOG, duration) + tagAutoCompleteQueryService.queryTagAutocompleteKeys(TagAutoCompleteQueryService.logTagTypes(null), duration) .forEach(tag -> labelsQueryRsp.getData().add(tag)); return successResponse(labelsQueryRsp); @@ -108,7 +107,7 @@ public HttpResponse labelValues( response.setStatus(ResultStatus.SUCCESS); Duration duration = DurationUtils.timestamp2Duration(nano2Millis(start), nano2Millis(end)); - tagAutoCompleteQueryService.queryTagAutocompleteValues(TagType.LOG, labelName, duration) + tagAutoCompleteQueryService.queryTagAutocompleteValues(TagAutoCompleteQueryService.logTagTypes(null), labelName, duration) .forEach(value -> response.getData().add(value)); return successResponse(response); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java index eb725bc8a103..0784129da7ac 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.util.Set; import java.util.concurrent.CompletableFuture; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.UnexpectedException; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.TagAutoCompleteQueryService; import org.apache.skywalking.oap.server.core.query.enumeration.Order; @@ -153,11 +153,16 @@ private Logs invokeQueryLogs(LogQueryCondition condition) throws IOException { ); } - public CompletableFuture> queryLogTagAutocompleteKeys(final Duration queryDuration) { - return queryAsync(() -> getTagQueryService().queryTagAutocompleteKeys(TagType.LOG, queryDuration)); + public CompletableFuture> queryLogTagAutocompleteKeys(final Duration queryDuration, + final TagType type) { + return queryAsync( + () -> getTagQueryService().queryTagAutocompleteKeys(TagAutoCompleteQueryService.logTagTypes(type), queryDuration)); } - public CompletableFuture> queryLogTagAutocompleteValues(final String tagKey, final Duration queryDuration) { - return queryAsync(() -> getTagQueryService().queryTagAutocompleteValues(TagType.LOG, tagKey, queryDuration)); + public CompletableFuture> queryLogTagAutocompleteValues(final String tagKey, + final Duration queryDuration, + final TagType type) { + return queryAsync( + () -> getTagQueryService().queryTagAutocompleteValues(TagAutoCompleteQueryService.logTagTypes(type), tagKey, queryDuration)); } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 966fb2cec582..a16e46620eaa 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 966fb2cec582846a4edb1a2f44af440a4803d479 +Subproject commit a16e46620eaac7088f8dd3c2923dff11276cc51b diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverConfig.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverConfig.java index 67de25447275..70117ed75200 100644 --- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverConfig.java +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverConfig.java @@ -23,6 +23,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.otlp.OTLPSpanRecord; import org.apache.skywalking.oap.server.library.module.ModuleConfig; @@ -97,6 +98,21 @@ public List getEnabledHandlers() { return Splitter.on(",").trimResults().omitEmptyStrings().splitToList(Strings.nullToEmpty(enabledHandlers)); } + /** + * The tag keys indexed as searchable on OTLP logs, in place of core's {@code searchableLogsTags}, which the logs of + * the SkyWalking native protocols keep. A key is a record attribute, a tag the receiver adds ({@code severity}, + * {@code level}) or a tag a LAL rule adds. + */ + @Setter + private String otlpLogSearchableTags; + + /** + * Split as core's {@code searchableLogsTags} is, so a key both lists name is the same key in each. + */ + public List getOtlpLogSearchableTags() { + return ConfigService.splitTagKeys(otlpLogSearchableTags); + } + public List getOtlpTraceSearchableTags() { return Splitter.on(",").trimResults().omitEmptyStrings().splitToList(Strings.nullToEmpty(otlpTraceSearchableTags)); } diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverProvider.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverProvider.java index 65870f30a8fd..f56eb97a1b66 100644 --- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverProvider.java +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/OtelMetricReceiverProvider.java @@ -24,8 +24,10 @@ import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; import org.apache.skywalking.oap.meter.analyzer.v2.MalConverterRegistry; import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.trace.analyzer.v2.module.TraceAnalyzerModule; import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.receiver.otel.otlp.OpenTelemetryLogHandler; import org.apache.skywalking.oap.server.receiver.otel.otlp.OpenTelemetryMetricRequestProcessor; import org.apache.skywalking.oap.server.receiver.otel.otlp.OpenTelemetryTraceHandler; import org.apache.skywalking.oap.server.receiver.zipkin.ZipkinReceiverModule; @@ -104,6 +106,12 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { @Override public void start() throws ServiceNotProvidedException, ModuleStartException { metricRequestProcessor.start(); + if (config.getEnabledHandlers().contains(OpenTelemetryLogHandler.TYPE)) { + // OTLP logs index these keys instead of core's searchableLogsTags; the log query, which reads all logs + // from one table, must accept them too. + getManager().find(CoreModule.NAME).provider().getService(ConfigService.class) + .addSearchableLogsTagKeys(config.getOtlpLogSearchableTags()); + } for (Handler h : handlers) { h.init(getManager(), config); h.active(); diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java index c934b12366f2..a053297bf4d9 100644 --- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java @@ -26,6 +26,7 @@ import io.opentelemetry.proto.common.v1.KeyValue; import com.google.protobuf.ByteString; import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.SeverityNumber; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.Getter; @@ -35,6 +36,7 @@ import org.apache.skywalking.apm.network.logging.v3.LogTags; import org.apache.skywalking.apm.network.logging.v3.TextLog; import org.apache.skywalking.apm.network.logging.v3.TraceContext; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.source.LogMetadata; import org.apache.skywalking.oap.server.core.source.LogMetadataUtils; import org.apache.skywalking.oap.log.analyzer.v2.module.LogAnalyzerModule; @@ -55,6 +57,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; @@ -64,6 +67,15 @@ public class OpenTelemetryLogHandler extends LogsServiceGrpc.LogsServiceImplBase implements Handler { + public static final String TYPE = "otlp-logs"; + + /** + * The tags a record's severity number is written under: {@code severity} as its enum name, {@code level} as the + * name of its range. A record attribute of either name is kept instead. + */ + private static final String SEVERITY_TAG = "severity"; + private static final String LEVEL_TAG = "level"; + private ModuleManager manager; private ILogAnalyzerService logAnalyzerService; @@ -79,6 +91,12 @@ public class OpenTelemetryLogHandler */ private boolean zipkinTraceIds; + /** + * {@code receiver-otel.otlpLogSearchableTags}: the tag keys indexed as searchable on these logs; null when the + * list is empty, so the logs keep core's {@code searchableLogsTags}. + */ + private Set searchableTagKeys; + @Getter(lazy = true) private final MetricsCreator metricsCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class); @@ -95,11 +113,13 @@ public void init(ModuleManager manager, OtelMetricReceiverConfig config) { this.manager = manager; this.serviceGroupKeys = config.getOtlpServiceGroupKeys(); this.zipkinTraceIds = !config.isOtlpTraceStorageNative(); + final List keys = config.getOtlpLogSearchableTags(); + this.searchableTagKeys = keys.isEmpty() ? null : Set.copyOf(keys); } @Override public String type() { - return "otlp-logs"; + return TYPE; } @Override @@ -133,7 +153,9 @@ void processExport(ExportLogsServiceRequest request) { final var attributes = resource .getAttributesList().stream() .map(it -> Map.entry(it.getKey(), buildTagValue(it))) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + // A repeated key keeps its last value, as a record's attributes do; toMap would throw and lose every + // log of the request. + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, (first, last) -> last)); // Filed under the grouped name, so the logs belong to the service the trace analysis names. final var service = OTLPServiceGroup.apply(attributes.get("service.name"), attributes, serviceGroupKeys); if (Strings.isNullOrEmpty(service)) { @@ -177,6 +199,8 @@ private void doAnalysisQuietly(String service, String layer, String serviceInsta builder.setTraceContext(TraceContext.newBuilder().setTraceId(traceId)); } final LogMetadata metadata = LogMetadataUtils.fromLogData(builder, resourceAttributes); + metadata.setSearchableTagKeys(searchableTagKeys); + metadata.setTagType(TagType.OTLP_LOG); final String spanId = idOf(logRecord.getSpanId()); if (!spanId.isEmpty()) { // A new context: the one a log without a trace id gets is a shared constant. @@ -222,12 +246,22 @@ private static LogDataBody buildBody(LogRecord logRecord) { } private LogTags buildTags(LogRecord logRecord) { + // OTLP requires attribute keys to be unique, but a sender may repeat one; the last value is kept instead of + // the record being lost to toMap's duplicate-key exception. + final Map tags = logRecord.getAttributesList() + .stream() + .collect(toMap(KeyValue::getKey, this::buildTagValue, + (first, last) -> last)); + final SeverityNumber severity = logRecord.getSeverityNumber(); + if (severity != SeverityNumber.SEVERITY_NUMBER_UNSPECIFIED && severity != SeverityNumber.UNRECOGNIZED) { + tags.putIfAbsent(SEVERITY_TAG, severity.name()); + final String level = levelOf(severity); + if (level != null) { + tags.putIfAbsent(LEVEL_TAG, level); + } + } return LogTags.newBuilder().addAllData( - logRecord - .getAttributesList() - .stream() - .collect(toMap(KeyValue::getKey, this::buildTagValue)) - .entrySet() + tags.entrySet() .stream() .map(it -> KeyStringValuePair .newBuilder() @@ -238,6 +272,47 @@ private LogTags buildTags(LogRecord logRecord) { ).build(); } + /** + * The name of the range a severity number falls in, or null for one outside the six ranges: unset, unrecognized, + * or a number a later protocol version adds, which keeps its {@code severity} tag but gets no level. + */ + static String levelOf(final SeverityNumber severity) { + switch (severity) { + case SEVERITY_NUMBER_TRACE: + case SEVERITY_NUMBER_TRACE2: + case SEVERITY_NUMBER_TRACE3: + case SEVERITY_NUMBER_TRACE4: + return "TRACE"; + case SEVERITY_NUMBER_DEBUG: + case SEVERITY_NUMBER_DEBUG2: + case SEVERITY_NUMBER_DEBUG3: + case SEVERITY_NUMBER_DEBUG4: + return "DEBUG"; + case SEVERITY_NUMBER_INFO: + case SEVERITY_NUMBER_INFO2: + case SEVERITY_NUMBER_INFO3: + case SEVERITY_NUMBER_INFO4: + return "INFO"; + case SEVERITY_NUMBER_WARN: + case SEVERITY_NUMBER_WARN2: + case SEVERITY_NUMBER_WARN3: + case SEVERITY_NUMBER_WARN4: + return "WARN"; + case SEVERITY_NUMBER_ERROR: + case SEVERITY_NUMBER_ERROR2: + case SEVERITY_NUMBER_ERROR3: + case SEVERITY_NUMBER_ERROR4: + return "ERROR"; + case SEVERITY_NUMBER_FATAL: + case SEVERITY_NUMBER_FATAL2: + case SEVERITY_NUMBER_FATAL3: + case SEVERITY_NUMBER_FATAL4: + return "FATAL"; + default: + return null; + } + } + private String buildTagValue(KeyValue it) { final var value = it.getValue(); return value.hasStringValue() ? value.getStringValue() : diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandlerTagsTest.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandlerTagsTest.java new file mode 100644 index 000000000000..3ed91a56cf82 --- /dev/null +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandlerTagsTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.otel.otlp; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.logs.v1.SeverityNumber; +import io.opentelemetry.proto.resource.v1.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.v2.module.LogAnalyzerModule; +import org.apache.skywalking.oap.log.analyzer.v2.provider.log.ILogAnalyzerService; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.source.LogMetadata; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.receiver.otel.OtelMetricReceiverConfig; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The tags an OTLP log record reaches the log analysis with, its attributes and its severity number as the + * {@code severity} and {@code level} tags, and the searchable tag keys the log is indexed by. + */ +class OpenTelemetryLogHandlerTagsTest { + private final List metadata = new ArrayList<>(); + private final List inputs = new ArrayList<>(); + + @Test + void theSeverityNumberIsTaggedByItsEnumName() { + export(LogRecord.newBuilder() + .setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_ERROR) + .setSeverityText("error")); + export(LogRecord.newBuilder() + .setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_INFO2) + .setSeverityText("Notice")); + + assertEquals("SEVERITY_NUMBER_ERROR", tagsOf(0).get("severity"), "the number decides, not the text"); + assertEquals("SEVERITY_NUMBER_INFO2", tagsOf(1).get("severity"), "a finer level keeps its own name"); + } + + @Test + void theLevelIsTheNameOfTheSeverityRange() { + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_TRACE)); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_DEBUG4)); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_INFO).setSeverityText("Information")); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_INFO2).setSeverityText("Notice")); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_WARN3)); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_ERROR2)); + export(LogRecord.newBuilder().setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_FATAL4)); + + final String[] expected = {"TRACE", "DEBUG", "INFO", "INFO", "WARN", "ERROR", "FATAL"}; + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], tagsOf(i).get("level"), tagsOf(i).get("severity")); + } + } + + @Test + void everyDefinedNumberHasTheLevelOfItsRange() { + for (final SeverityNumber severity : SeverityNumber.values()) { + if (severity == SeverityNumber.SEVERITY_NUMBER_UNSPECIFIED || severity == SeverityNumber.UNRECOGNIZED) { + assertNull(OpenTelemetryLogHandler.levelOf(severity), severity.name()); + continue; + } + final String range = severity.name().substring("SEVERITY_NUMBER_".length()).replaceAll("[0-9]$", ""); + assertEquals(range, OpenTelemetryLogHandler.levelOf(severity), severity.name()); + } + } + + @Test + void aRepeatedAttributeKeyKeepsTheLastValue() { + export(LogRecord.newBuilder() + .addAttributes(KeyValue.newBuilder().setKey("url.path") + .setValue(AnyValue.newBuilder().setStringValue("/first"))) + .addAttributes(KeyValue.newBuilder().setKey("url.path") + .setValue(AnyValue.newBuilder().setStringValue("/last")))); + + assertEquals(1, inputs.size(), "the record reaches the analysis instead of failing on the duplicate key"); + assertEquals("/last", tagsOf(0).get("url.path")); + } + + @Test + void aRepeatedResourceAttributeKeyKeepsTheLastValue() { + export(LogRecord.newBuilder(), "", "service.instance.id", "first", "service.instance.id", "last"); + + assertEquals(1, inputs.size(), "the request's logs reach the analysis instead of failing on the duplicate key"); + assertEquals("last", inputs.get(0).getServiceInstance()); + } + + @Test + void aRecordWithoutASeverityNumberGetsNoTag() { + export(LogRecord.newBuilder()); + export(LogRecord.newBuilder().setSeverityText("INFO")); + export(LogRecord.newBuilder().setSeverityNumberValue(99)); + + for (int i = 0; i < inputs.size(); i++) { + assertFalse(tagsOf(i).containsKey("severity"), "record " + i + " has no defined severity number"); + assertFalse(tagsOf(i).containsKey("level"), "record " + i + " has no defined severity number"); + } + } + + @Test + void attributesNamedSeverityOrLevelAreKept() { + export(LogRecord.newBuilder() + .setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_WARN) + .addAttributes(KeyValue.newBuilder().setKey("severity") + .setValue(AnyValue.newBuilder().setStringValue("major"))) + .addAttributes(KeyValue.newBuilder().setKey("url.path") + .setValue(AnyValue.newBuilder().setStringValue("/api/cart")))); + export(LogRecord.newBuilder() + .setSeverityNumber(SeverityNumber.SEVERITY_NUMBER_WARN) + .addAttributes(KeyValue.newBuilder().setKey("level") + .setValue(AnyValue.newBuilder().setStringValue("warning")))); + + assertEquals("major", tagsOf(0).get("severity")); + assertEquals("WARN", tagsOf(0).get("level")); + assertEquals("/api/cart", tagsOf(0).get("url.path")); + assertEquals(3, inputs.get(0).getTags().getDataCount()); + assertEquals("SEVERITY_NUMBER_WARN", tagsOf(1).get("severity")); + assertEquals("warning", tagsOf(1).get("level")); + } + + @Test + void resourceAttributesStayOffTheLog() { + export(LogRecord.newBuilder(), "level,service.version", "service.version", "3.1.0"); + + assertEquals(Set.of(), tagsOf(0).keySet(), "even a resource attribute named in the list is not a tag"); + } + + @Test + void theLogIsIndexedByTheOtlpList() { + export(LogRecord.newBuilder(), " level , severity,http.response.status_code "); + + assertEquals(Set.of("level", "severity", "http.response.status_code"), + Set.copyOf(metadata.get(0).getSearchableTagKeys())); + } + + @Test + void theLogsTagsAreAutocompletedAsOtlpLogs() { + export(LogRecord.newBuilder()); + + assertEquals(TagType.OTLP_LOG, metadata.get(0).getTagType()); + } + + @Test + void anEmptyOtlpListLeavesTheLogToCoresList() { + export(LogRecord.newBuilder(), ""); + + assertNull(metadata.get(0).getSearchableTagKeys(), "null makes LogBuilder use core's searchableLogsTags"); + } + + private Map tagsOf(final int index) { + return inputs.get(index).getTags().getDataList().stream() + .collect(Collectors.toMap(KeyStringValuePair::getKey, KeyStringValuePair::getValue)); + } + + private void export(final LogRecord.Builder record) { + export(record, ""); + } + + /** + * @param resourceAttributes key, value, key, value... added to the resource besides {@code service.name} + */ + private void export(final LogRecord.Builder record, final String searchableTags, + final String... resourceAttributes) { + final OpenTelemetryLogHandler handler = new OpenTelemetryLogHandler(); + final OtelMetricReceiverConfig config = new OtelMetricReceiverConfig(); + config.setOtlpLogSearchableTags(searchableTags); + handler.init(manager(), config); + final Resource.Builder resource = Resource.newBuilder().addAttributes( + KeyValue.newBuilder().setKey("service.name").setValue(AnyValue.newBuilder().setStringValue("checkout"))); + for (int i = 0; i < resourceAttributes.length; i += 2) { + resource.addAttributes(KeyValue.newBuilder().setKey(resourceAttributes[i]) + .setValue(AnyValue.newBuilder().setStringValue(resourceAttributes[i + 1]))); + } + handler.processExport(ExportLogsServiceRequest.newBuilder().addResourceLogs( + ResourceLogs.newBuilder() + .setResource(resource) + .addScopeLogs(ScopeLogs.newBuilder().addLogRecords( + record.setTimeUnixNano(1_700_000_000_000_000_000L) + .setBody(AnyValue.newBuilder().setStringValue("a line"))))) + .build()); + } + + private ModuleManager manager() { + final ILogAnalyzerService analyzer = mock(ILogAnalyzerService.class); + doAnswer(inv -> { + metadata.add(inv.getArgument(0)); + inputs.add(inv.getArgument(1)); + return null; + }).when(analyzer).doAnalysis(any(), any()); + final MetricsCreator metricsCreator = mock(MetricsCreator.class); + when(metricsCreator.createHistogramMetric(anyString(), anyString(), any(), any())) + .thenReturn(new HistogramMetrics() { + @Override + public void observe(final double value) { + } + }); + final ModuleManager manager = mock(ModuleManager.class); + provide(manager, LogAnalyzerModule.NAME, ILogAnalyzerService.class, analyzer); + provide(manager, TelemetryModule.NAME, MetricsCreator.class, metricsCreator); + return manager; + } + + private static void provide(final ModuleManager manager, final String module, + final Class type, final T service) { + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(services.getService(type)).thenReturn(service); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + when(provider.provider()).thenReturn(services); + when(manager.find(module)).thenReturn(provider); + } +} diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 6244f3531f00..0bbcd7f8769d 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -123,7 +123,7 @@ core: searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker} # Define the set of log tag keys, which should be searchable through the GraphQL. # The max length of key=value should be less than 256 or will be dropped. - searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,http.status_code,ai_route_type} + searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,http.status_code} # Define the set of alarm tag keys, which should be searchable through the GraphQL. # The max length of key=value should be less than 256 or will be dropped. searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level} @@ -498,6 +498,12 @@ receiver-otel: # `cart` still reaches `shop::cart`. Use an attribute that holds one low-cardinality value per service, such as # `service.namespace`, `deployment.environment.name` or `k8s.namespace.name`. Empty, the default, groups nothing. otlpServiceGroupKeys: ${SW_OTEL_RECEIVER_OTLP_SERVICE_GROUP:""} + # `otlp-logs` only. Tag keys, comma separated, indexed as searchable on OTLP logs, in place of core's + # `searchableLogsTags`, which the SkyWalking native logs keep. A key is a log record attribute, the `severity` and + # `level` tags the receiver adds from the record's severity number, or a tag a LAL rule adds. List low-cardinality + # keys only: every distinct value is stored for tag autocomplete. Empty uses core's `searchableLogsTags`. + # The max length of key=value should be less than 256 or will be dropped. + otlpLogSearchableTags: ${SW_OTEL_LOG_SEARCHABLE_TAG_KEYS:level,severity,ai_route_type} # `otlp` mode only. Attribute keys offered by tag autocomplete (TraceQL /api/v2/search/tags), each written with its # TraceQL scope, `resource.` or `span.`; an entry without a scope fails the boot. List low-cardinality # attributes only: every distinct value is stored once per day. Every resource and span attribute is indexed for diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java index ed48968a7bf9..6fcbc84abfe0 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java @@ -25,7 +25,6 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; @@ -36,6 +35,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import static java.util.Objects.nonNull; @@ -58,7 +58,7 @@ public BanyanDBTagAutocompleteQueryDAO(BanyanDBStorageClient client) { } @Override - public Set queryTagAutocompleteKeys(TagType tagType, int limit, Duration duration) throws IOException { + public Set queryTagAutocompleteKeys(List tagTypes, int limit, Duration duration) throws IOException { MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetricMetadata(TagAutocompleteData.INDEX_NAME, DownSampling.Minute); long startMinTB = 0; long endMinTB = 0; @@ -78,7 +78,7 @@ public Set queryTagAutocompleteKeys(TagType tagType, int limit, Duration TAGS_KEY, Collections.emptySet(), range, Conditions.create() - .eq(TagAutocompleteData.TAG_TYPE, tagType.name()) + .in(TagAutocompleteData.TAG_TYPE, tagTypes) .groupBy(TagAutocompleteData.TAG_KEY) .limit(limit)); @@ -94,7 +94,7 @@ public Set queryTagAutocompleteKeys(TagType tagType, int limit, Duration } @Override - public Set queryTagAutocompleteValues(TagType tagType, String tagKey, int limit, Duration duration) throws IOException { + public Set queryTagAutocompleteValues(List tagTypes, String tagKey, int limit, Duration duration) throws IOException { MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetricMetadata(TagAutocompleteData.INDEX_NAME, DownSampling.Minute); long startMinTB = 0; long endMinTB = 0; @@ -114,7 +114,7 @@ public Set queryTagAutocompleteValues(TagType tagType, String tagKey, in TAGS_KV, Collections.emptySet(), range, Conditions.create() - .eq(TagAutocompleteData.TAG_TYPE, tagType.name()) + .in(TagAutocompleteData.TAG_TYPE, tagTypes) .eq(TagAutocompleteData.TAG_KEY, tagKey) .groupBy(TagAutocompleteData.TAG_VALUE) .limit(limit)); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexModeQueryIT.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexModeQueryIT.java index d6ac2374655f..31a310fee964 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexModeQueryIT.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexModeQueryIT.java @@ -119,12 +119,12 @@ public static void close() throws IOException { public void autocompletesTagKeysAndValuesOfTheWindow() throws IOException { final BanyanDBTagAutocompleteQueryDAO dao = new BanyanDBTagAutocompleteQueryDAO(SUPPORT.getClient()); assertEquals(Arrays.asList("db.type", "http.method", "status_code"), - new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteKeys(TagType.TRACE, 10, today())))); + new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 10, today())))); assertEquals(Collections.singletonList("level"), - new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteKeys(TagType.LOG, 10, today())))); + new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteKeys(List.of(TagType.LOG.name()), 10, today())))); assertEquals(Arrays.asList("GET", "POST"), - new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteValues(TagType.TRACE, "http.method", 10, today())))); - assertEquals(2, dao.queryTagAutocompleteKeys(TagType.TRACE, 2, today()).size()); + new ArrayList<>(new TreeSet<>(dao.queryTagAutocompleteValues(List.of(TagType.TRACE.name()), "http.method", 10, today())))); + assertEquals(2, dao.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 2, today()).size()); } @Test diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java index f99e7a4f84ec..a7aca429af3d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java @@ -32,7 +32,6 @@ import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; @@ -49,13 +48,13 @@ public TagAutoCompleteQueryDAO(ElasticSearchClient client) { } @Override - public Set queryTagAutocompleteKeys(final TagType tagType, + public Set queryTagAutocompleteKeys(final List tagTypes, final int limit, final Duration duration) throws IOException { // Tags combine records by day and es rolling index by day, need search the whole day // just use the time to locate the physical indexes and ignore time conditions. BoolQueryBuilder query = Query.bool(); - query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); + query.must(Query.terms(TagAutocompleteData.TAG_TYPE, tagTypes)); if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); } @@ -86,11 +85,11 @@ public Set queryTagAutocompleteKeys(final TagType tagType, } @Override - public Set queryTagAutocompleteValues(final TagType tagType, final String tagKey, + public Set queryTagAutocompleteValues(final List tagTypes, final String tagKey, final int limit, final Duration duration) throws IOException { BoolQueryBuilder query = Query.bool().must(Query.term(TagAutocompleteData.TAG_KEY, tagKey)); - query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); + query.must(Query.terms(TagAutocompleteData.TAG_TYPE, tagTypes)); if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java index ba3604106af9..52cdbe6993a7 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java @@ -185,7 +185,7 @@ void queryTagAutocompleteKeys_searchesTheTagIndexRolledByDayStep() throws Except IndexNames.configure(3, 1); answerRangeSearch(); - new TagAutoCompleteQueryDAO(client).queryTagAutocompleteKeys(TagType.TRACE, 10, RANGE); + new TagAutoCompleteQueryDAO(client).queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 10, RANGE); assertThat(rangeSearchIndices()).containsExactly("tag_autocomplete-20200807", "tag_autocomplete-20200810"); } @@ -195,7 +195,7 @@ void queryTagAutocompleteValues_withoutDuration_searchesTheWholeTagIndexFamily() IndexNames.configure(3, 1); answerRangeSearch(); - new TagAutoCompleteQueryDAO(client).queryTagAutocompleteValues(TagType.TRACE, "http.method", 10, null); + new TagAutoCompleteQueryDAO(client).queryTagAutocompleteValues(List.of(TagType.TRACE.name()), "http.method", 10, null); assertThat(rangeSearchIndices()).containsExactly("tag_autocomplete"); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java index b983226a0090..06c221b30cc0 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java @@ -391,13 +391,13 @@ public void readsTheLabelsOfAServiceFromEveryDay() { public void autocompletesTheTagKeysOfTheDaysOfTheWindow() throws Exception { assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method", "http.route", "http.status_code", "rpc.method")), - new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, aroundMidnight))); + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 10, aroundMidnight))); assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method", "http.route", "http.status_code")), - new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, afterMidnight))); + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 10, afterMidnight))); assertEquals(new TreeSet<>(Arrays.asList("http.method", "level")), - new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.LOG, 10, aroundMidnight))); + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(List.of(TagType.LOG.name()), 10, aroundMidnight))); - final Set limited = tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 2, aroundMidnight); + final Set limited = tagDAO.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 2, aroundMidnight); assertEquals(2, limited.size()); assertTrue(Arrays.asList("db.type", "http.method", "http.route", "http.status_code", "rpc.method") .containsAll(limited)); @@ -407,16 +407,16 @@ public void autocompletesTheTagKeysOfTheDaysOfTheWindow() throws Exception { public void autocompletesTheTagValuesOfTheDaysOfTheWindow() throws Exception { assertEquals(new TreeSet<>(Arrays.asList("GET", "POST")), new TreeSet<>(tagDAO.queryTagAutocompleteValues( - TagType.TRACE, "http.method", 10, aroundMidnight))); + List.of(TagType.TRACE.name()), "http.method", 10, aroundMidnight))); assertEquals(new TreeSet<>(Arrays.asList("200", "500")), new TreeSet<>(tagDAO.queryTagAutocompleteValues( - TagType.TRACE, "http.status_code", 10, aroundMidnight))); + List.of(TagType.TRACE.name()), "http.status_code", 10, aroundMidnight))); assertEquals(Collections.singleton("500"), - tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 10, afterMidnight)); + tagDAO.queryTagAutocompleteValues(List.of(TagType.TRACE.name()), "http.status_code", 10, afterMidnight)); assertEquals(Collections.singleton("PUT"), - tagDAO.queryTagAutocompleteValues(TagType.LOG, "http.method", 10, aroundMidnight)); + tagDAO.queryTagAutocompleteValues(List.of(TagType.LOG.name()), "http.method", 10, aroundMidnight)); assertEquals(Collections.emptySet(), - tagDAO.queryTagAutocompleteValues(TagType.TRACE, "old.key", 10, aroundMidnight)); + tagDAO.queryTagAutocompleteValues(List.of(TagType.TRACE.name()), "old.key", 10, aroundMidnight)); } private Set expectedInstances(final String... names) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java index fabbad866f09..609397b58f9b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java @@ -22,7 +22,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord; @@ -47,8 +46,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -94,7 +91,7 @@ public Logs queryLogs(String serviceId, final ConfigService configService = manager.find(CoreModule.NAME) .provider() .getService(ConfigService.class); - searchableTagKeys = new HashSet<>(Arrays.asList(configService.getSearchableLogsTags().split(Const.COMMA))); + searchableTagKeys = configService.getSearchableLogsTagKeys(); } if (tags != null && !searchableTagKeys.containsAll(tags.stream().map(Tag::getKey).collect(toSet()))) { log.warn( diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java index 3b6b3cd26708..244bbc22d171 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java @@ -22,7 +22,6 @@ import lombok.SneakyThrows; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; @@ -44,7 +43,7 @@ public class JDBCTagAutoCompleteQueryDAO implements ITagAutoCompleteQueryDAO { @Override @SneakyThrows - public Set queryTagAutocompleteKeys(final TagType tagType, + public Set queryTagAutocompleteKeys(final List tagTypes, final int limit, final Duration duration) { final var tables = tableHelper.getTablesForRead( @@ -55,7 +54,7 @@ public Set queryTagAutocompleteKeys(final TagType tagType, final var results = new HashSet(); for (String table : tables) { - final var sqlAndParameters = buildSQLForQueryKeys(tagType, limit, duration, table); + final var sqlAndParameters = buildSQLForQueryKeys(tagTypes, limit, duration, table); jdbcClient.executeQuery( sqlAndParameters.sql(), resultSet -> { @@ -69,14 +68,14 @@ public Set queryTagAutocompleteKeys(final TagType tagType, return limit(results, limit); } - protected SQLAndParameters buildSQLForQueryKeys(TagType tagType, int limit, Duration duration, String table) { + protected SQLAndParameters buildSQLForQueryKeys(List tagTypes, int limit, Duration duration, String table) { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(2); sql.append("select distinct ").append(TagAutocompleteData.TAG_KEY).append(" from ") .append(table).append(" where "); sql.append(" 1=1 "); - appendTagAutocompleteCondition(tagType, duration, sql, condition); + appendTagAutocompleteCondition(tagTypes, duration, sql, condition); sql.append(" limit ").append(limit); return new SQLAndParameters(sql.toString(), condition); @@ -84,7 +83,7 @@ protected SQLAndParameters buildSQLForQueryKeys(TagType tagType, int limit, Dura @Override @SneakyThrows - public Set queryTagAutocompleteValues(final TagType tagType, + public Set queryTagAutocompleteValues(final List tagTypes, final String tagKey, final int limit, final Duration duration) { @@ -96,7 +95,7 @@ public Set queryTagAutocompleteValues(final TagType tagType, final var results = new HashSet(); for (String table : tables) { - final var sqlAndParameters = buildSQLForQueryValues(tagType, tagKey, limit, duration, table); + final var sqlAndParameters = buildSQLForQueryValues(tagTypes, tagKey, limit, duration, table); jdbcClient.executeQuery( sqlAndParameters.sql(), resultSet -> { @@ -118,19 +117,19 @@ private static Set limit(final Set results, final int limit) { return results.stream().limit(limit).collect(Collectors.toSet()); } - protected SQLAndParameters buildSQLForQueryValues(TagType tagType, String tagKey, int limit, Duration duration, String table) { + protected SQLAndParameters buildSQLForQueryValues(List tagTypes, String tagKey, int limit, Duration duration, String table) { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(3); sql.append("select * from ").append(table).append(" where "); sql.append(TagAutocompleteData.TAG_KEY).append(" = ?"); condition.add(tagKey); - appendTagAutocompleteCondition(tagType, duration, sql, condition); + appendTagAutocompleteCondition(tagTypes, duration, sql, condition); sql.append(" limit ").append(limit); return new SQLAndParameters(sql.toString(), condition); } - private void appendTagAutocompleteCondition(final TagType tagType, + private void appendTagAutocompleteCondition(final List tagTypes, final Duration duration, final StringBuilder sql, final List condition) { @@ -145,8 +144,10 @@ private void appendTagAutocompleteCondition(final TagType tagType, long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB); sql.append(" and "); - sql.append(TagAutocompleteData.TAG_TYPE).append(" = ?"); - condition.add(tagType.name()); + sql.append(TagAutocompleteData.TAG_TYPE).append(" in (") + .append(tagTypes.stream().map(it -> "?").collect(Collectors.joining(","))) + .append(")"); + condition.addAll(tagTypes); if (startTB > 0) { sql.append(" and "); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java index 8cb37048c536..a08f3ac79e62 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java @@ -439,9 +439,9 @@ public void limitsTheTagKeysOfBothDays() throws Exception { for (final String[] tag : TAGS_TODAY) { keys.add(tag[0]); } - assertEquals(keys, tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 100, aroundMidnight)); + assertEquals(keys, tagDAO.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 100, aroundMidnight)); - final Set limited = tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 5, aroundMidnight); + final Set limited = tagDAO.queryTagAutocompleteKeys(List.of(TagType.TRACE.name()), 5, aroundMidnight); assertEquals(5, limited.size()); assertTrue(keys.containsAll(limited)); } @@ -449,10 +449,10 @@ public void limitsTheTagKeysOfBothDays() throws Exception { @Test public void limitsTheTagValuesOfBothDays() throws Exception { final Set values = new TreeSet<>(Arrays.asList("200", "404", "500", "503")); - assertEquals(values, tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 100, aroundMidnight)); + assertEquals(values, tagDAO.queryTagAutocompleteValues(List.of(TagType.TRACE.name()), "http.status_code", 100, aroundMidnight)); final Set limited = tagDAO.queryTagAutocompleteValues( - TagType.TRACE, "http.status_code", 3, aroundMidnight); + List.of(TagType.TRACE.name()), "http.status_code", 3, aroundMidnight); assertEquals(3, limited.size()); assertTrue(values.containsAll(limited)); } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java new file mode 100644 index 000000000000..521df8b68e45 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * Several tag types are read in one query: the tags of any of them, within one limit. + */ +class JDBCTagAutoCompleteQueryDAOTagTypesTest { + private static final List LOG_SOURCES = List.of(TagType.LOG.name(), TagType.OTLP_LOG.name()); + + private final JDBCTagAutoCompleteQueryDAO dao = + new JDBCTagAutoCompleteQueryDAO(mock(JDBCClient.class), mock(TableHelper.class)); + + @Test + void theKeysOfSeveralTagTypesAreOneQuery() { + final SQLAndParameters result = dao.buildSQLForQueryKeys(LOG_SOURCES, 100, duration(), "t"); + + assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " in (?,?)"); + assertThat(result.sql()).contains("limit 100"); + assertThat(result.parameters()).containsSubsequence(TagType.LOG.name(), TagType.OTLP_LOG.name()); + } + + @Test + void theValuesOfSeveralTagTypesAreOneQuery() { + final SQLAndParameters result = dao.buildSQLForQueryValues(LOG_SOURCES, "level", 100, duration(), "t"); + + assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " in (?,?)"); + assertThat(result.parameters()).containsSubsequence("level", TagType.LOG.name(), TagType.OTLP_LOG.name()); + } + + private static Duration duration() { + final Duration duration = new Duration(); + duration.setStart("2023-01-01 0000"); + duration.setEnd("2023-01-02 0000"); + duration.setStep(Step.MINUTE); + return duration; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTest.java index d7b724d0860b..86b43b810fda 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTest.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import java.util.List; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.enumeration.Step; @@ -65,10 +66,10 @@ private Duration buildDuration() { void buildSQLForQueryKeys_shouldContainDistinctAndTagType() { final Duration duration = buildDuration(); - final SQLAndParameters result = dao.buildSQLForQueryKeys(TagType.TRACE, 100, duration, TABLE); + final SQLAndParameters result = dao.buildSQLForQueryKeys(List.of(TagType.TRACE.name()), 100, duration, TABLE); assertThat(result.sql()).contains("select distinct " + TagAutocompleteData.TAG_KEY); - assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " = ?"); + assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " in (?)"); assertThat(result.parameters()).contains(TagType.TRACE.name()); } @@ -76,7 +77,7 @@ void buildSQLForQueryKeys_shouldContainDistinctAndTagType() { void buildSQLForQueryKeys_shouldContainLimit() { final Duration duration = buildDuration(); - final SQLAndParameters result = dao.buildSQLForQueryKeys(TagType.TRACE, 50, duration, TABLE); + final SQLAndParameters result = dao.buildSQLForQueryKeys(List.of(TagType.TRACE.name()), 50, duration, TABLE); assertThat(result.sql()).contains("limit 50"); } @@ -86,7 +87,7 @@ void buildSQLForQueryValues_shouldContainTagKeyCondition() { final Duration duration = buildDuration(); final SQLAndParameters result = dao.buildSQLForQueryValues( - TagType.TRACE, "http.method", 100, duration, TABLE); + List.of(TagType.TRACE.name()), "http.method", 100, duration, TABLE); assertThat(result.sql()).contains(TagAutocompleteData.TAG_KEY + " = ?"); assertThat(result.parameters()).contains("http.method"); @@ -97,9 +98,9 @@ void buildSQLForQueryValues_shouldContainTagTypeCondition() { final Duration duration = buildDuration(); final SQLAndParameters result = dao.buildSQLForQueryValues( - TagType.LOG, "level", 100, duration, TABLE); + List.of(TagType.LOG.name()), "level", 100, duration, TABLE); - assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " = ?"); + assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " in (?)"); assertThat(result.parameters()).contains(TagType.LOG.name()); } } diff --git a/test/e2e-v2/cases/storage/expected/config-dump.yml b/test/e2e-v2/cases/storage/expected/config-dump.yml index 8c54cccfa274..254174dc7618 100644 --- a/test/e2e-v2/cases/storage/expected/config-dump.yml +++ b/test/e2e-v2/cases/storage/expected/config-dump.yml @@ -116,7 +116,7 @@ "core.default.restSSLKeyPath": "", "core.default.role": "Mixed", "core.default.searchableAlarmTags": "level", - "core.default.searchableLogsTags": "level,http.status_code,ai_route_type", + "core.default.searchableLogsTags": "level,http.status_code", "core.default.searchableTracesTags": "http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker", "core.default.serviceCacheRefreshInterval": "10", "core.default.serviceNameMaxLength": "70", @@ -213,6 +213,7 @@ "receiver-meter.provider": "default", "receiver-otel.default.enabledHandlers": "otlp-traces,otlp-metrics,otlp-logs", "receiver-otel.default.enabledOtelMetricsRules": "apisix,nginx/*,k8s/*,istio-controlplane,vm,process-hostmetrics-linux,mysql/*,postgresql/*,oap,aws-eks/*,windows,process-hostmetrics-windows,aws-s3/*,aws-dynamodb/*,aws-gateway/*,redis/*,elasticsearch/*,rabbitmq/*,mongodb/*,kafka/*,pulsar/*,bookkeeper/*,rocketmq/*,clickhouse/*,activemq/*,kong/*,flink/*,airflow/*,banyandb/*,envoy-ai-gateway/*,ios/*,miniprogram/*,ai-agent/*", + "receiver-otel.default.otlpLogSearchableTags": "level,severity,ai_route_type", "receiver-otel.default.otlpServiceGroupKeys": "", "receiver-otel.default.otlpTraceMaxSpansPerSecond": "0", "receiver-otel.default.otlpTraceSampleRate": "10000", From 6c5eee848357b03f7493f7cd590ccb639d184d61 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 20 Nov 2026 11:41:31 +0800 Subject: [PATCH 2/2] Read the autocomplete values of several tag types without duplicates taking the limit - JDBC: select distinct tag_value, so a value's row under each tag type does not take the limit's places. - Elasticsearch: read the values one query per tag type and merge them within the limit; tag_value has no doc values to aggregate on. - JDBC log IT: stub the searchable log tag keys on the mocked ConfigService. --- .../query/ITagAutoCompleteQueryDAO.java | 6 +- .../query/TagAutoCompleteQueryDAO.java | 18 +++- .../TagAutoCompleteQueryDAOTagTypesTest.java | 96 +++++++++++++++++++ .../dao/JDBCTagAutoCompleteQueryDAO.java | 4 +- .../jdbc/common/dao/JDBCLogQueryITBase.java | 2 + ...BCTagAutoCompleteQueryDAOTagTypesTest.java | 2 + 6 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAOTagTypesTest.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java index 764d65a417c9..5baa9973b429 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java @@ -28,7 +28,8 @@ public interface ITagAutoCompleteQueryDAO extends Service { /** * @param tagTypes the names of the tag types to read ({@link TagType#name()}), the tags of any of them; - * not empty + * not empty. Most of the time a query has only one tag type; only the logs' autocomplete + * without a type reads two (LOG and OTLP_LOG). * @param limit the most keys returned, across all the tag types * @param duration the time range * @return the distinct tag keys @@ -40,7 +41,8 @@ Set queryTagAutocompleteKeys(final List tagTypes, /** * @param tagTypes the names of the tag types to read ({@link TagType#name()}), the tags of any of them; - * not empty + * not empty. Most of the time a query has only one tag type; only the logs' autocomplete + * without a type reads two (LOG and OTLP_LOG). * @param tagKey the key whose values are read * @param limit the most values returned, across all the tag types * @param duration the time range diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java index a7aca429af3d..b38f0b155407 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java @@ -20,10 +20,12 @@ import java.io.IOException; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; @@ -88,8 +90,22 @@ public Set queryTagAutocompleteKeys(final List tagTypes, public Set queryTagAutocompleteValues(final List tagTypes, final String tagKey, final int limit, final Duration duration) throws IOException { + // One query per tag type: the values are read as documents, not aggregated (tag_value has no doc values), + // so in one query a value's document under each type would take the limit's places from other values. + // Most of the time there is only one tag type, so this is one query; only the logs' autocomplete without a + // type (LOG and OTLP_LOG) reads two. + final Set tagValues = new LinkedHashSet<>(); + for (final String tagType : tagTypes) { + tagValues.addAll(queryTagAutocompleteValues(tagType, tagKey, limit, duration)); + } + return tagValues.stream().limit(limit).collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private Set queryTagAutocompleteValues(final String tagType, final String tagKey, + final int limit, + final Duration duration) throws IOException { BoolQueryBuilder query = Query.bool().must(Query.term(TagAutocompleteData.TAG_KEY, tagKey)); - query.must(Query.terms(TagAutocompleteData.TAG_TYPE, tagTypes)); + query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType)); if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAOTagTypesTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAOTagTypesTest.java new file mode 100644 index 000000000000..f3c82348b9b6 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAOTagTypesTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexNames; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The values of several tag types are read one type at a time: in one query, a value's document under each type + * would take the limit's places from other values. + */ +@ExtendWith(MockitoExtension.class) +class TagAutoCompleteQueryDAOTagTypesTest { + private static final List LOG_SOURCES = List.of(TagType.LOG.name(), TagType.OTLP_LOG.name()); + + @Mock + private ElasticSearchClient client; + + @BeforeEach + void setUp() { + IndexNames.configure(3, 1); + } + + @Test + void aValueUnderBothTypesDoesNotHideAnother() throws Exception { + when(client.search(any(String[].class), any(Search.class))) + .thenReturn(response(TagType.LOG, "ERROR"), response(TagType.OTLP_LOG, "ERROR", "WARN")); + + final Set values = new TagAutoCompleteQueryDAO(client).queryTagAutocompleteValues( + LOG_SOURCES, "level", 2, null); + + assertThat(values).containsExactly("ERROR", "WARN"); + verify(client, times(2)).search(any(String[].class), any(Search.class)); + } + + @Test + void theMergedValuesKeepTheLimit() throws Exception { + when(client.search(any(String[].class), any(Search.class))) + .thenReturn(response(TagType.LOG, "ERROR"), response(TagType.OTLP_LOG, "INFO")); + + assertThat(new TagAutoCompleteQueryDAO(client).queryTagAutocompleteValues(LOG_SOURCES, "level", 1, null)) + .containsExactly("ERROR"); + } + + private static SearchResponse response(final TagType tagType, final String... values) { + final List hits = new ArrayList<>(); + for (final String value : values) { + final SearchHit hit = new SearchHit(); + hit.setSource(Map.of( + TagAutocompleteData.TAG_KEY, "level", + TagAutocompleteData.TAG_VALUE, value, + TagAutocompleteData.TAG_TYPE, tagType.name(), + TagAutocompleteData.TIME_BUCKET, 20200810L)); + hits.add(hit); + } + final SearchResponse response = new SearchResponse(); + response.getHits().setHits(hits); + return response; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java index 244bbc22d171..dab9501d47cb 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java @@ -120,7 +120,9 @@ private static Set limit(final Set results, final int limit) { protected SQLAndParameters buildSQLForQueryValues(List tagTypes, String tagKey, int limit, Duration duration, String table) { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(3); - sql.append("select * from ").append(table).append(" where "); + // Distinct before the limit: a value can have a row under each of the tag types read, and those copies must + // not take the limit's places from other values. + sql.append("select distinct ").append(TagAutocompleteData.TAG_VALUE).append(" from ").append(table).append(" where "); sql.append(TagAutocompleteData.TAG_KEY).append(" = ?"); condition.add(tagKey); appendTagAutocompleteCondition(tagTypes, duration, sql, condition); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java index 146a7ee11a80..d9befc9d6f6f 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; @@ -187,6 +188,7 @@ public void writeEventsBrowserErrorsAndLogs() throws Exception { final ConfigService config = support.getModuleManager().find(CoreModule.NAME).provider() .getService(ConfigService.class); when(config.getSearchableLogsTags()).thenReturn(SEARCHABLE_LOG_TAGS); + when(config.getSearchableLogsTagKeys()).thenReturn(Set.copyOf(ConfigService.splitTagKeys(SEARCHABLE_LOG_TAGS))); logDAO = new JDBCLogQueryDAO(support.getClient(), support.getModuleManager(), support.getTableHelper()); } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java index 521df8b68e45..d06d3651a918 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAOTagTypesTest.java @@ -54,6 +54,8 @@ void theValuesOfSeveralTagTypesAreOneQuery() { final SQLAndParameters result = dao.buildSQLForQueryValues(LOG_SOURCES, "level", 100, duration(), "t"); assertThat(result.sql()).contains(TagAutocompleteData.TAG_TYPE + " in (?,?)"); + assertThat(result.sql()).contains("select distinct " + TagAutocompleteData.TAG_VALUE + " from "); + assertThat(result.sql()).endsWith(" limit 100"); assertThat(result.parameters()).containsSubsequence("level", TagType.LOG.name(), TagType.OTLP_LOG.name()); }