From a1d074f8f173807243d90dadec28de6bdee96d9e Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:36:11 +0000 Subject: [PATCH 1/9] [client-v2] Send the User-Agent in the standard client format and add client metadata The User-Agent header is now ' clickhouse-java-v2/ (:; ...; lv:java/; os:)', the format that clickhouse-go, clickhouse-js, clickhouse-connect and clickhouse-rs use. 'jvm:' becomes 'lv:java/', the OS becomes the 'os:' entry, and the trailing 'Apache-HttpClient/' token is removed. Client.Builder#setClientMetadata(Map) (property 'client_metadata', 'key1=value1,key2=value2') adds key/value pairs to the comment group, before 'lv' and 'os'. Entries that would break the format are rejected with ClientMisconfigurationException when the client is built. Implements: https://github.com/ClickHouse/clickhouse-java/issues/3202 Co-Authored-By: Claude Opus 5.5 --- .../com/clickhouse/client/api/Client.java | 25 ++++ .../client/api/ClientConfigProperties.java | 70 ++++++++++ .../api/internal/HttpAPIClientHelper.java | 49 ++----- .../resources/client-v2-version.properties | 3 +- .../clickhouse/client/HttpTransportTests.java | 6 +- .../api/ClientConfigPropertiesTest.java | 55 ++++++++ .../client/api/UserAgentUnitTest.java | 131 ++++++++++++++++++ docs/clickhouse-docs/client.mdx | 19 ++- docs/clickhouse-docs/jdbc.mdx | 5 +- docs/features.md | 3 +- docs/integration-client.md | 26 +++- docs/integration-jdbc.md | 6 +- history/latest/3202.md | 6 + .../com/clickhouse/jdbc/ConnectionTest.java | 41 +++++- 14 files changed, 388 insertions(+), 57 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java create mode 100644 history/latest/3202.md diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 621f88efb..d422983ca 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1144,6 +1144,7 @@ public Builder useHTTPBasicAuth(boolean useBasicAuth) { * Warn: If custom value of User-Agent header is set it will override this value for HTTP transport * Client name is used by server to identify client application when investigating {@code system.query_log}. In case of HTTP * transport this value will be in the {@code system.query_log.http_user_agent} column. Currently only HTTP transport is used. + * Use {@link #setClientMetadata(Map)} to add key/value pairs to the comment group of the {@code User-Agent} header. * * @param clientName - client application display name. * @return same instance of the builder @@ -1153,6 +1154,30 @@ public Builder setClientName(String clientName) { return this; } + /** + * Sets key/value pairs that the client adds to the comment group of the {@code User-Agent} header, in the + * iteration order of {@code metadata}, before its own {@code lv} and {@code os} entries. For example, a client + * name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give + * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; os:)}. + * Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be + * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, a value must not contain any of + * {@code ;()}, and the keys {@code lv} and {@code os} are reserved; otherwise {@link #build()} throws + * {@link ClientMisconfigurationException}. This is a client-level setting. + * + * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata + * @return same instance of the builder + * @see ClientConfigProperties#CLIENT_METADATA + */ + public Builder setClientMetadata(Map metadata) { + if (metadata == null || metadata.isEmpty()) { + this.configuration.remove(ClientConfigProperties.CLIENT_METADATA.getKey()); + } else { + this.configuration.put(ClientConfigProperties.CLIENT_METADATA.getKey(), + ClientConfigProperties.fromKeyValuePairs(metadata)); + } + return this; + } + /** * Sets client options from provided map. Values are copied as is * @param options - map of client options diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index 63173c5a2..888892631 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -257,6 +257,24 @@ public Object parseValue(String value) { .collect(Collectors.toList()); } }, + + /** + * Key/value pairs, written as {@code key1=value1,key2=value2}, that the client adds to the comment group of the + * {@code User-Agent} header. A {@code ,} or {@code \} in a value is escaped with {@code \}. This is a client-level + * setting: it is not read from operation settings. See {@link Client.Builder#setClientMetadata(Map)} for the + * format and the rules for keys and values. + */ + CLIENT_METADATA("client_metadata", Map.class) { + @Override + @SuppressWarnings("unchecked") + public Object parseValue(String value) { + Map metadata = (Map) super.parseValue(value); + if (metadata != null) { + metadata.forEach(ClientConfigProperties::checkClientMetadataEntry); + } + return metadata; + } + }, ; private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); @@ -506,6 +524,58 @@ public static Map toKeyValuePairs(String str) { + private static void checkClientMetadataEntry(String key, String value) { + if (key.equalsIgnoreCase("lv") || key.equalsIgnoreCase("os")) { + throw new ClientMisconfigurationException("Invalid '" + CLIENT_METADATA.getKey() + "' entry '" + key + + "': the key is reserved by the client"); + } + if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) { + throw new ClientMisconfigurationException("Invalid '" + CLIENT_METADATA.getKey() + "' entry '" + key + + "=" + value + "': keys and values must be printable ASCII, a key must not contain spaces or" + + " any of ':;(),=', and a value must not contain any of ';()'"); + } + } + + private static boolean isClientMetadataText(String str, String forbiddenChars) { + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if (ch < 0x20 || ch > 0x7E || forbiddenChars.indexOf(ch) >= 0) { + return false; + } + } + return true; + } + + /** + * Inverse of {@link #toKeyValuePairs(String)}: escapes {@code \}, {@code ,} and {@code =} with {@code \} and + * skips entries with a {@code null} key or value. + */ + static String fromKeyValuePairs(Map map) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + continue; + } + if (sb.length() > 0) { + sb.append(','); + } + appendEscaped(sb, entry.getKey()).append('='); + appendEscaped(sb, entry.getValue()); + } + return sb.toString(); + } + + private static StringBuilder appendEscaped(StringBuilder sb, String str) { + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if (ch == '\\' || ch == ',' || ch == '=') { + sb.append('\\'); + } + sb.append(ch); + } + return sb; + } + public static String mapToString(Map map, Function valueConverter) { StringBuilder sb = new StringBuilder(); for (Map.Entry entry : map.entrySet()) { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 70db8d438..cc5dcc12b 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -105,7 +105,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; @@ -171,7 +170,7 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - this.defaultUserAgent = buildDefaultUserAgent(); + this.defaultUserAgent = buildDefaultUserAgent(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); } /** @@ -1204,47 +1203,17 @@ private void correctUserAgentHeader(HttpRequest request, Map req request.setHeader(HttpHeaders.USER_AGENT, userAgentValue); } - private String buildDefaultUserAgent() { + private static String buildDefaultUserAgent(Map clientMetadata) { StringBuilder userAgent = new StringBuilder(); - userAgent.append(Client.CLIENT_USER_AGENT); - - String clientVersion = Client.clientVersion; - - userAgent.append(clientVersion); - - userAgent.append(" ("); - userAgent.append(System.getProperty("os.name")); - userAgent.append("; "); - userAgent.append("jvm:").append(System.getProperty("java.version")); - userAgent.append("; "); - - userAgent.setLength(userAgent.length() - 2); - userAgent.append(')'); - - try { - String httpClientVersion = this.httpClient.getClass().getPackage().getImplementationVersion(); - if (Objects.equals(this.httpClient.getClass().getPackage().getImplementationTitle(), this.getClass().getPackage().getImplementationTitle())) { - // shaded jar - all packages have same implementation title - httpClientVersion = "unknown"; - try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("client-v2-version.properties")) { - Properties p = new Properties(); - p.load(in); - - String tmp = p.getProperty("apache.http.client.version"); - if (tmp != null && !tmp.isEmpty() && !tmp.equals("${apache.httpclient.version}")) { - httpClientVersion = tmp; - } - } catch (Exception e) { - LOG.debug("Failed to read HTTP client version from client-v2-version.properties", e); - } + userAgent.append(Client.CLIENT_USER_AGENT).append(Client.clientVersion).append(" ("); + if (clientMetadata != null) { + for (Map.Entry entry : clientMetadata.entrySet()) { + userAgent.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); } - userAgent.append(" ") - .append("Apache-HttpClient") - .append('/') - .append(httpClientVersion); - } catch (Exception e) { - LOG.warn("failed to construct http client version string", e); } + userAgent.append("lv:java/").append(System.getProperty("java.version")) + .append("; os:").append(System.getProperty("os.name")) + .append(')'); return userAgent.toString(); } diff --git a/client-v2/src/main/resources/client-v2-version.properties b/client-v2/src/main/resources/client-v2-version.properties index a617d20fb..a54c8f97d 100644 --- a/client-v2/src/main/resources/client-v2-version.properties +++ b/client-v2/src/main/resources/client-v2-version.properties @@ -1,3 +1,2 @@ version=${revision} -build.date=${build_timestamp} -apache.http.client.version=${apache.httpclient.version} \ No newline at end of file +build.date=${build_timestamp} \ No newline at end of file diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index b8b412742..a19eb8ba2 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -1201,9 +1201,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider") public static Object[][] testUserAgentHasCompleteProductName_dataProvider() { return new Object[][] { - { "", Pattern.compile("clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$") }, - { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")}, - { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")}}; + { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)") }, + { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")}, + { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")}}; } @Test(dataProvider = "testClientNameDataProvider") diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index 9c23ca522..d044ff731 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -5,9 +5,11 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -76,4 +78,57 @@ public void testParseConfigMapSanitizesSslCipherSuites() { Assert.assertEquals(parsed.get(ClientConfigProperties.SSL_CIPHER_SUITES.getKey()), Arrays.asList("TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256")); } + + @DataProvider(name = "validClientMetadata") + public static Object[][] validClientMetadata() { + return new Object[][]{ + {"fv=flink/2.0.0,api=table", new String[]{"fv", "flink/2.0.0", "api", "table"}}, + {" api = table , os_version = Mac OS X ", new String[]{"api", "table", "os_version", "Mac OS X"}}, + {"url=http://host:8123/path?x=1", new String[]{"url", "http://host:8123/path?x=1"}}, + {"api=,fv=flink/2.0.0,=x", new String[]{"fv", "flink/2.0.0"}}, + {"", new String[0]}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "validClientMetadata") + public void testClientMetadataParsed(String raw, String[] expectedEntries) { + Map expected = new LinkedHashMap<>(); + for (int i = 0; i < expectedEntries.length; i += 2) { + expected.put(expectedEntries[i], expectedEntries[i + 1]); + } + Map parsed = (Map) ClientConfigProperties.CLIENT_METADATA.parseValue(raw); + Assert.assertEquals(new ArrayList<>(parsed.entrySet()), new ArrayList<>(expected.entrySet())); + } + + @DataProvider(name = "invalidClientMetadata") + public static Object[][] invalidClientMetadata() { + return new Object[][]{ + {"a:b=c"}, {"a;b=c"}, {"a(b=c"}, {"a)b=c"}, {"a\\,b=c"}, {"a\\=b=c"}, {"my key=c"}, + {"k=v;x"}, {"k=v(x"}, {"k=v)x"}, + {"k=a\nb"}, {"k=a\tb"}, {"k=caf\u00e9"}, {"k\u00e9=v"}, + {"lv=java/8"}, {"os=Linux"}, {"OS=Linux"}, + {"fv=flink/2.0.0,api=table; lv:java/8"}, + {",api=table"}, {"fv=flink/2.0.0,,api=table"}, {"fv=flink/2.0.0,api,ver=1"}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "invalidClientMetadata") + public void testInvalidClientMetadataRejected(String raw) { + Assert.expectThrows(ClientMisconfigurationException.class, + () -> ClientConfigProperties.CLIENT_METADATA.parseValue(raw)); + } + + @Test(groups = {"unit"}) + public void testClientMetadataUnsetParsesToNull() { + Assert.assertNull(ClientConfigProperties.CLIENT_METADATA.parseValue(null)); + } + + @Test(groups = {"unit"}) + public void testFromKeyValuePairsRoundTrip() { + Map pairs = new LinkedHashMap<>(); + pairs.put("a=b", "c,d\\e=f"); + pairs.put("x,y", "z"); + Map parsed = ClientConfigProperties.toKeyValuePairs(ClientConfigProperties.fromKeyValuePairs(pairs)); + Assert.assertEquals(new ArrayList<>(parsed.entrySet()), new ArrayList<>(pairs.entrySet())); + } } \ No newline at end of file diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java new file mode 100644 index 000000000..265382a70 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -0,0 +1,131 @@ +package com.clickhouse.client.api; + +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import org.apache.hc.core5.http.HttpHeaders; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +public class UserAgentUnitTest { + + private WireMockServer mockServer; + + @BeforeMethod + public void setUp() { + mockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + mockServer.start(); + mockServer.stubFor(WireMock.post(WireMock.anyUrl()) + .willReturn(WireMock.aResponse().withStatus(200))); + } + + @AfterMethod + public void tearDown() { + if (mockServer != null) { + mockServer.stop(); + } + } + + @Test(dataProvider = "userAgentData") + public void testUserAgentHeader(Consumer configure, String expectedUserAgent) throws Exception { + Client.Builder builder = newClientBuilder(); + configure.accept(builder); + try (Client client = builder.build()) { + assertUserAgent(client, null, expectedUserAgent); + } + } + + @DataProvider(name = "userAgentData") + public static Object[][] userAgentData() { + Map flinkMetadata = new LinkedHashMap<>(); + flinkMetadata.put("fv", "flink/2.0.0"); + flinkMetadata.put("api", "table"); + + Map withEmptyValue = new LinkedHashMap<>(); + withEmptyValue.put("api", ""); + withEmptyValue.put("fv", "flink/2.0.0"); + + return new Object[][]{ + {(Consumer) b -> { }, userAgent(null, null)}, + {(Consumer) b -> b.setClientName("Flink-ClickHouse-Sink/0.2.0") + .setClientMetadata(flinkMetadata), + userAgent("Flink-ClickHouse-Sink/0.2.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), + "api=table, fv=flink/2.0.0"), + userAgent(null, "api:table; fv:flink/2.0.0")}, + {(Consumer) b -> b.setClientMetadata(Collections.singletonMap("build", "a=b,c\\d")), + userAgent(null, "build:a=b,c\\d")}, + {(Consumer) b -> b.setClientMetadata(withEmptyValue), + userAgent(null, "fv:flink/2.0.0")}, + {(Consumer) b -> b.setClientMetadata(flinkMetadata).setClientMetadata(null), + userAgent(null, null)}, + {(Consumer) b -> b.setClientName("my-app/1.0") + .httpHeader(HttpHeaders.USER_AGENT, "custom-agent/2.0") + .setClientMetadata(Collections.singletonMap("api", "table")), + userAgent("custom-agent/2.0", "api:table")}, + }; + } + + @Test + public void testClientMetadataKeptWhenApplicationPartChanges() throws Exception { + try (Client client = newClientBuilder() + .setClientName("my-app/1.0") + .setClientMetadata(Collections.singletonMap("api", "table")) + .build()) { + client.updateClientName("new-app/2.0"); + assertUserAgent(client, null, userAgent("new-app/2.0", "api:table")); + assertUserAgent(client, new QuerySettings().httpHeader(HttpHeaders.USER_AGENT, "op-agent/3.0"), + userAgent("op-agent/3.0", "api:table")); + assertUserAgent(client, (QuerySettings) new QuerySettings() + .setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "fv=flink/2.0.0"), + userAgent("new-app/2.0", "api:table")); + } + } + + @Test + public void testInvalidClientMetadataRejectedOnBuild() { + Client.Builder builder = newClientBuilder() + .setClientMetadata(Collections.singletonMap("api", "table; os:other")); + Assert.expectThrows(ClientMisconfigurationException.class, builder::build); + } + + private Client.Builder newClientBuilder() { + return new Client.Builder() + .addEndpoint("http://localhost:" + mockServer.port()) + .setUsername("default") + .setPassword(""); + } + + private void assertUserAgent(Client client, QuerySettings settings, String expectedUserAgent) throws Exception { + mockServer.resetRequests(); + try (QueryResponse response = client.query("SELECT 1", settings).get(10, TimeUnit.SECONDS)) { + Assert.assertNotNull(response); + } + List requests = mockServer.findAll(WireMock.postRequestedFor(WireMock.anyUrl())); + Assert.assertFalse(requests.isEmpty(), "no request was sent"); + for (LoggedRequest request : requests) { + Assert.assertEquals(request.getHeader(HttpHeaders.USER_AGENT), expectedUserAgent); + } + } + + private static String userAgent(String application, String metadata) { + return (application == null ? "" : application + " ") + + "clickhouse-java-v2/" + Client.clientVersion + + " (" + (metadata == null ? "" : metadata + "; ") + + "lv:java/" + System.getProperty("java.version") + + "; os:" + System.getProperty("os.name") + ")"; + } +} diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index b408ed64a..ccf55862b 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -264,6 +264,7 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | | `setClientName(String clientName)` | `clientName` - application name string | Sets additional information about calling application. Will be passed as `User-Agent` header. | - | `client_name` | +| `setClientMetadata(Map metadata)` | `metadata` - key/value pairs | Adds key/value pairs to the comment group of the `User-Agent` header, in the iteration order of the map, before the `lv` and `os` entries. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -277,19 +278,27 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. There are two fields in a query log that identify application originated a request: `client_name` and `http_user_agent`. Native TCP protocol uses `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. -The field `http_user_agent` is set according to `User-Agent` header common format: `application-name[/version] [(operating-system; architecture; ...)]`. -This set of values is repeated for each layer: application, client library, http client library. What is set by `setClientName` method comes first in the list. +The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:)`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientMetadata` are added to the comment group before `lv` and `os`. For example: ```java showLineNumbers -client.setClientName("my-app-01/1.0"); +Map metadata = new LinkedHashMap<>(); +metadata.put("fv", "flink/2.0.0"); +metadata.put("api", "table"); + +Client client = new Client.Builder() + // ... + .setClientName("my-app-01/1.0") + .setClientMetadata(metadata) + .build(); ``` will result in the following `http_user_agent` value: ``` -my-app-01/1.0 clickhouse-java-v2/0.9.6-SNAPSHOT (Linux; jvm:17.0.17) Apache-HttpClient/5.4.4 +my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux) ``` -Application can set own http header `User-Agent` to identify itself. But part `clickhouse-java-v2/0.9.6-SNAPSHOT` will be appended to the end of the header. +Application can set own http header `User-Agent` to identify itself. But part `clickhouse-java-v2/0.12.0 (...)` will be appended to the end of the header. ### Operation Identification {#operation-identification} diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 0d23aae0f..67e811005 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -146,9 +146,12 @@ conn.setClientInfo(com.clickhouse.jdbc.ClientInfoProperties.APPLICATION_NAME.get Both ways will result in the following `http_user_agent` value in the query log: ``` -my-app-01/1.0 jdbc-v2/0.9.7 clickhouse-java-v2/0.9.6 (Linux; jvm:17.0.17) Apache-HttpClient/5.4.4 +my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux) ``` +To add key/value pairs to the comment group of the header, set the `client_metadata` property. For example, +`client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux)`. + **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. ### Operation Identification {#operation-identification} diff --git a/docs/features.md b/docs/features.md index a9c6e405a..945cc194a 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,6 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise (omitted when both are empty). The header does not name the HTTP transport library. `Client.Builder.setClientMetadata(Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client#updateClientName`). - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: @@ -97,7 +98,7 @@ Compatibility-sensitive traits: - Nested columns: Un-flattened `Nested(f1 T1, ..., fN TN)` columns (tables created with `flatten_nested = 0`) are exposed as JDBC `ARRAY` whose element type is `Tuple(f1 T1, ..., fN TN)`. They can be inserted through `Connection#createArrayOf`/`setArray` or `setObject` (a Java array of tuples) and read back through `getArray`/`getObject`; `java.sql.Array#getResultSet()` iterates the nested rows as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple. An `Array(Nested(...))` column is read as a JDBC `ARRAY` whose elements are the nested values, each an array of tuples (`Object[]` of `Object[]`); its `getResultSet()` returns each element as a `java.sql.Array` equal to a top-level `Nested(...)` value. - Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape. For ClickHouse `26.8+` the variant also carries `MultiPoint`. - MultiPoint type mapping: For ClickHouse `26.8+`, JDBC exposes `MultiPoint` as `ARRAY` with type name `MultiPoint`, returns `double[][]` from `getObject()` and a `java.sql.Array` from `getArray()`, and accepts a nested `Array` (`createArrayOf("Array(Point)", ...)`) for prepared-statement inserts, the same as `Ring` and `LineString`. -- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. +- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. The `client_metadata` connection property is forwarded to the client and adds key/value pairs to the `User-Agent` header (see the `client-v2` `User-Agent` entry). - Wrapper support: Implements standard JDBC `Wrapper` and `unwrap` behavior on major JDBC objects. - Packaging and runtime compatibility: Ships as a JDBC 4.2 driver, depends on `client-v2`, and includes native-image metadata for GraalVM users. diff --git a/docs/integration-client.md b/docs/integration-client.md index 7e0e7342b..be2a11758 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -119,12 +119,34 @@ public void updateClientWorkload(Client client, String workloadOrTenant) { #### How it is observed on the server (`User-Agent` header) -The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version, operating system, JVM version, and underlying HTTP client details: +The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`) and operating system (`os`), in the standard ClickHouse client format: ```text -order-service/1.2.0 clickhouse-java-v2/0.9.6 (Linux; jvm:17.0.2) Apache-HttpClient/5.4.4 +order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux) ``` +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientMetadata`. The pairs are added in the iteration order of the map, before `lv` and `os`: + +```java +Map metadata = new LinkedHashMap<>(); +metadata.put("fv", "flink/2.0.0"); +metadata.put("api", "table"); + +Client client = new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("secret") + .setClientName("Flink-ClickHouse-Sink/0.2.0") + .setClientMetadata(metadata) + .build(); +``` + +```text +Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux) +``` + +The same pairs can be set with the `client_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. + > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting Java Client applications. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index bbdd38d30..f3d8087b0 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -152,12 +152,14 @@ public void executeWorkloadTask(Connection conn, String taskName) throws SQLExce #### How it is observed on the server (`User-Agent` header) -The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. HikariCP), client version, operating system, and JVM version: +The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`) and operating system (`os`): ```text -order-service/1.2.0 ClickHouse-JDBC/0.9.8 clickhouse-java-v2/0.9.8 (Linux; jvm:17.0.2) Apache-HttpClient/5.4.4 +order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux) ``` +To add key/value pairs to the comment group of the header, set the `client_metadata` property (for example, `client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux)`). + > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting JDBC applications. diff --git a/history/latest/3202.md b/history/latest/3202.md new file mode 100644 index 000000000..c642e5dd4 --- /dev/null +++ b/history/latest/3202.md @@ -0,0 +1,6 @@ +- **[client-v2, jdbc-v2]** Breaking change: the `User-Agent` header now uses the standard ClickHouse client format, + ` clickhouse-java-v2/ (lv:java/; os:)`: `jvm:` is now + `lv:java/`, the OS is the `os:` entry, and the trailing `Apache-HttpClient/` is removed. Added + `Client.Builder#setClientMetadata(Map)` (property `client_metadata`) to add key/value pairs to the comment group. + Queries that parse `http_user_agent` in `system.query_log` must use the new format. + (https://github.com/ClickHouse/clickhouse-java/issues/3202) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 782ef8229..7c035796a 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -349,7 +349,7 @@ public void setAndGetClientInfoTest(String clientName) throws Exception { } } - private void executeQueryAndVerifyUserAgent(Statement stmt, String clientName) throws Exception { + private String executeQueryAndVerifyUserAgent(Statement stmt, String clientName) throws Exception { final String testQuery = "SELECT '" + UUID.randomUUID() + "'"; stmt.execute(testQuery); String queryId = ((StatementImpl)stmt).getLastQueryId(); @@ -375,6 +375,45 @@ private void executeQueryAndVerifyUserAgent(Statement stmt, String clientName) t } Assert.assertTrue(userAgent.contains(Client.CLIENT_USER_AGENT), "Expected to contain '" + Client.CLIENT_USER_AGENT + "' but value was '" + userAgent + "'"); Assert.assertTrue(userAgent.contains(Driver.DRIVER_CLIENT_NAME), "Expected to contain '" + Driver.DRIVER_CLIENT_NAME + "' but value was '" + userAgent + "'"); + return userAgent; + } + + @Test(groups = { "integration" }, dataProvider = "clientMetadataInUserAgentDP") + public void testClientMetadataInUserAgent(boolean setInUrl) throws Exception { + if (isCloud()) { + throw new SkipException("Cloud does not support flushing query_log"); + } + + final String metadata = "fv=flink/2.0.0,api=table"; + Properties info = new Properties(); + info.setProperty("user", "default"); + info.setProperty("password", ClickHouseServerForTest.getPassword()); + info.setProperty(ClientConfigProperties.DATABASE.getKey(), ClickHouseServerForTest.getDatabase()); + String url = getEndpointString(); + if (setInUrl) { + url += "?" + ClientConfigProperties.CLIENT_METADATA.getKey() + "=" + metadata; + } else { + info.setProperty(ClientConfigProperties.CLIENT_METADATA.getKey(), metadata); + } + + String expectedSuffix = "clickhouse-java-v2/" + Client.clientVersion + " (fv:flink/2.0.0; api:table; lv:java/" + + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + ")"; + try (Connection localConnection = new ConnectionImpl(url, info); + Statement stmt = localConnection.createStatement()) { + String userAgent = executeQueryAndVerifyUserAgent(stmt, null); + Assert.assertTrue(userAgent.endsWith(expectedSuffix), + "Expected to end with '" + expectedSuffix + "' but value was '" + userAgent + "'"); + + localConnection.setClientInfo(ClientInfoProperties.APPLICATION_NAME.getKey(), "my-app/1.0"); + userAgent = executeQueryAndVerifyUserAgent(stmt, "my-app/1.0"); + Assert.assertTrue(userAgent.endsWith(expectedSuffix), + "Expected to end with '" + expectedSuffix + "' but value was '" + userAgent + "'"); + } + } + + @DataProvider(name = "clientMetadataInUserAgentDP") + public static Object[][] clientMetadataInUserAgentDP() { + return new Object[][] {{false}, {true}}; } @Test(groups = { "integration" }) From aabc06e75efd8518ee1d818734a5e864325c571c Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:48:50 +0000 Subject: [PATCH 2/9] [client-v2] Move client identification into WorkloadIdentitySupport Address review feedback on #3213: - Add WorkloadIdentitySupport. It builds the User-Agent value (client name or custom User-Agent, then clickhouse-java-v2/ with the metadata, lv and os entries) and validates client metadata entries. HttpAPIClientHelper only reads a custom User-Agent header and sets the value that the class returns. - Replace Client.Builder#setClientMetadata(Map) with Client.Builder#setClientName(String, Map). setClientName(String) does not change the metadata. The client_metadata property is unchanged. - State in the docs and in the httpHeader Javadoc that applications set the client name, not the User-Agent header. Implements: https://github.com/ClickHouse/clickhouse-java/issues/3202 Co-Authored-By: Claude Opus 5.5 --- .../com/clickhouse/client/api/Client.java | 35 ++++--- .../client/api/ClientConfigProperties.java | 27 +----- .../client/api/internal/CommonSettings.java | 2 + .../api/internal/HttpAPIClientHelper.java | 29 +----- .../api/internal/WorkloadIdentitySupport.java | 95 +++++++++++++++++++ .../client/api/UserAgentUnitTest.java | 43 ++++++--- docs/clickhouse-docs/client.mdx | 12 +-- docs/clickhouse-docs/jdbc.mdx | 1 + docs/features.md | 2 +- docs/integration-client.md | 9 +- docs/integration-jdbc.md | 2 + history/latest/3202.md | 2 +- 12 files changed, 172 insertions(+), 87 deletions(-) create mode 100644 client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index d422983ca..68af64aa6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1004,6 +1004,8 @@ public Builder allowBinaryReaderToReuseBuffers(boolean reuse) { * Defines list of headers that should be sent with each request. The Client will use a header value * defined in {@code headers} instead of any other. * Operation settings may override these headers. + * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify + * the application, use {@link #setClientName(String)} instead of the {@code User-Agent} header. * * @see InsertSettings#httpHeaders(Map) * @see QuerySettings#httpHeaders(Map) @@ -1139,14 +1141,17 @@ public Builder useHTTPBasicAuth(boolean useBasicAuth) { } /** - * Sets additional information about calling application. This string will be passed to server as a client name. - * In case of HTTP protocol it will be passed as a {@code User-Agent} header. - * Warn: If custom value of User-Agent header is set it will override this value for HTTP transport - * Client name is used by server to identify client application when investigating {@code system.query_log}. In case of HTTP - * transport this value will be in the {@code system.query_log.http_user_agent} column. Currently only HTTP transport is used. - * Use {@link #setClientMetadata(Map)} to add key/value pairs to the comment group of the {@code User-Agent} header. + * Sets the client name, which identifies the application to the server. The client sends it in the + * {@code User-Agent} header, before its own part {@code clickhouse-java-v2/ (...)}. The server records + * the header in the {@code http_user_agent} column of {@code system.query_log}. + *

+ * To identify the application, set the client name. Do not set the {@code User-Agent} header with + * {@link #httpHeader(String, String)}: a custom {@code User-Agent} header replaces the client name. + *

+ * This method does not change the client metadata. To also add key/value pairs to the {@code User-Agent} + * header, use {@link #setClientName(String, Map)}. * - * @param clientName - client application display name. + * @param clientName - client application name, for example {@code my-app/1.0} * @return same instance of the builder */ public Builder setClientName(String clientName) { @@ -1155,20 +1160,26 @@ public Builder setClientName(String clientName) { } /** - * Sets key/value pairs that the client adds to the comment group of the {@code User-Agent} header, in the - * iteration order of {@code metadata}, before its own {@code lv} and {@code os} entries. For example, a client - * name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give + * Sets the client name and the client metadata, which identify the application to the server. See + * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the + * {@code User-Agent} header, in the iteration order of {@code metadata}, before its own {@code lv} and + * {@code os} entries. For example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} + * and {@code api=table} give * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; os:)}. + *

* Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, a value must not contain any of * {@code ;()}, and the keys {@code lv} and {@code os} are reserved; otherwise {@link #build()} throws - * {@link ClientMisconfigurationException}. This is a client-level setting. + * {@link ClientMisconfigurationException}. The metadata is a client-level setting: it is kept when the client + * name changes with {@link Client#updateClientName(String)}. * + * @param clientName - client application name, for example {@code my-app/1.0} * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata * @return same instance of the builder * @see ClientConfigProperties#CLIENT_METADATA */ - public Builder setClientMetadata(Map metadata) { + public Builder setClientName(String clientName, Map metadata) { + setClientName(clientName); if (metadata == null || metadata.isEmpty()) { this.configuration.remove(ClientConfigProperties.CLIENT_METADATA.getKey()); } else { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index 888892631..ae9bdb0fb 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -4,6 +4,7 @@ import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; import com.clickhouse.client.api.enums.SSLMode; import com.clickhouse.client.api.internal.CompressedBlockOutputStream; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; import org.slf4j.Logger; @@ -261,7 +262,7 @@ public Object parseValue(String value) { /** * Key/value pairs, written as {@code key1=value1,key2=value2}, that the client adds to the comment group of the * {@code User-Agent} header. A {@code ,} or {@code \} in a value is escaped with {@code \}. This is a client-level - * setting: it is not read from operation settings. See {@link Client.Builder#setClientMetadata(Map)} for the + * setting: it is not read from operation settings. See {@link Client.Builder#setClientName(String, Map)} for the * format and the rules for keys and values. */ CLIENT_METADATA("client_metadata", Map.class) { @@ -270,7 +271,7 @@ public Object parseValue(String value) { public Object parseValue(String value) { Map metadata = (Map) super.parseValue(value); if (metadata != null) { - metadata.forEach(ClientConfigProperties::checkClientMetadataEntry); + metadata.forEach(WorkloadIdentitySupport::checkClientMetadataEntry); } return metadata; } @@ -524,28 +525,6 @@ public static Map toKeyValuePairs(String str) { - private static void checkClientMetadataEntry(String key, String value) { - if (key.equalsIgnoreCase("lv") || key.equalsIgnoreCase("os")) { - throw new ClientMisconfigurationException("Invalid '" + CLIENT_METADATA.getKey() + "' entry '" + key - + "': the key is reserved by the client"); - } - if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) { - throw new ClientMisconfigurationException("Invalid '" + CLIENT_METADATA.getKey() + "' entry '" + key - + "=" + value + "': keys and values must be printable ASCII, a key must not contain spaces or" - + " any of ':;(),=', and a value must not contain any of ';()'"); - } - } - - private static boolean isClientMetadataText(String str, String forbiddenChars) { - for (int i = 0; i < str.length(); i++) { - char ch = str.charAt(i); - if (ch < 0x20 || ch > 0x7E || forbiddenChars.indexOf(ch) >= 0) { - return false; - } - } - return true; - } - /** * Inverse of {@link #toKeyValuePairs(String)}: escapes {@code \}, {@code ,} and {@code =} with {@code \} and * skips entries with a {@code null} key or value. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java index e84873fb5..b94422bb9 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java @@ -178,6 +178,8 @@ public String getDatabase() { /** * Defines list of headers that should be sent with current request. The Client will use a header value * defined in {@code headers} instead of any other. + * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify + * the application, use {@link Client.Builder#setClientName(String)} instead of the {@code User-Agent} header. * * @see Client.Builder#httpHeaders(Map) * @param key - header name. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index cc5dcc12b..22a0b7a18 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -1,7 +1,6 @@ package com.clickhouse.client.api.internal; import com.clickhouse.client.api.ClickHouseException; -import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientException; import com.clickhouse.client.api.ClientFaultCause; @@ -134,7 +133,7 @@ public class HttpAPIClientHelper { private final Set defaultRetryCauses; - private final String defaultUserAgent; + private final WorkloadIdentitySupport workloadIdentitySupport; private final Object metricsRegistry; ConnPoolControl poolControl; @@ -170,7 +169,7 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - this.defaultUserAgent = buildDefaultUserAgent(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); + this.workloadIdentitySupport = new WorkloadIdentitySupport(configuration); } /** @@ -1193,28 +1192,8 @@ private void correctUserAgentHeader(HttpRequest request, Map req Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); - String clientName = ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig); - String userAgentValue = defaultUserAgent; - if (userAgentHeader == null && clientName != null && !clientName.isEmpty()) { - userAgentValue = clientName + " " + defaultUserAgent; - } else if (userAgentHeader != null) { - userAgentValue = userAgentHeader.getValue() + " " + defaultUserAgent; - } - request.setHeader(HttpHeaders.USER_AGENT, userAgentValue); - } - - private static String buildDefaultUserAgent(Map clientMetadata) { - StringBuilder userAgent = new StringBuilder(); - userAgent.append(Client.CLIENT_USER_AGENT).append(Client.clientVersion).append(" ("); - if (clientMetadata != null) { - for (Map.Entry entry : clientMetadata.entrySet()) { - userAgent.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); - } - } - userAgent.append("lv:java/").append(System.getProperty("java.version")) - .append("; os:").append(System.getProperty("os.name")) - .append(')'); - return userAgent.toString(); + request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.getUserAgent( + userAgentHeader == null ? null : userAgentHeader.getValue(), requestConfig)); } public void close() { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java new file mode 100644 index 000000000..e104bbfbd --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -0,0 +1,95 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.ClientMisconfigurationException; + +import java.util.Map; + +/** + * Builds the identification of the client and of the application that uses it. The identification is the value of + * the {@code User-Agent} header, in the standard ClickHouse client format: + * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:)}. + *

+ * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), and the key/value pairs are + * the client metadata ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of + * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. + */ +public class WorkloadIdentitySupport { + + private static final String RESERVED_KEY_LANGUAGE_VERSION = "lv"; + + private static final String RESERVED_KEY_OS = "os"; + + private final String clientUserAgent; + + /** + * @param configuration - parsed client configuration + */ + public WorkloadIdentitySupport(Map configuration) { + this.clientUserAgent = buildClientUserAgent(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); + } + + /** + * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part. + * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and + * the client name otherwise. A blank value counts as not set, and a blank application part is left out. + * + * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null} + * @param requestConfig - settings of the request + * @return value of the {@code User-Agent} header + */ + public String getUserAgent(String customUserAgent, Map requestConfig) { + String application = isBlank(customUserAgent) + ? ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig) : customUserAgent; + return isBlank(application) ? clientUserAgent : application + " " + clientUserAgent; + } + + private static boolean isBlank(String str) { + return str == null || str.trim().isEmpty(); + } + + /** + * Checks that a client metadata entry does not break the format of the {@code User-Agent} header. + * + * @param key - trimmed, not empty key + * @param value - trimmed, not empty value + * @throws ClientMisconfigurationException if the key is reserved, or the key or the value contains a character + * that is not allowed + */ + public static void checkClientMetadataEntry(String key, String value) { + if (key.equalsIgnoreCase(RESERVED_KEY_LANGUAGE_VERSION) || key.equalsIgnoreCase(RESERVED_KEY_OS)) { + throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey() + + "' entry '" + key + "': the key is reserved by the client"); + } + if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) { + throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey() + + "' entry '" + key + "=" + value + "': keys and values must be printable ASCII, a key must not" + + " contain spaces or any of ':;(),=', and a value must not contain any of ';()'"); + } + } + + private static boolean isClientMetadataText(String str, String forbiddenChars) { + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if (ch < 0x20 || ch > 0x7E || forbiddenChars.indexOf(ch) >= 0) { + return false; + } + } + return true; + } + + private static String buildClientUserAgent(Map clientMetadata) { + StringBuilder userAgent = new StringBuilder(); + userAgent.append(Client.CLIENT_USER_AGENT).append(Client.clientVersion).append(" ("); + if (clientMetadata != null) { + for (Map.Entry entry : clientMetadata.entrySet()) { + userAgent.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); + } + } + userAgent.append(RESERVED_KEY_LANGUAGE_VERSION).append(":java/").append(System.getProperty("java.version")) + .append("; ").append(RESERVED_KEY_OS).append(':').append(System.getProperty("os.name")) + .append(')'); + return userAgent.toString(); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index 265382a70..0f8f98f9a 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -60,21 +60,37 @@ public static Object[][] userAgentData() { return new Object[][]{ {(Consumer) b -> { }, userAgent(null, null)}, - {(Consumer) b -> b.setClientName("Flink-ClickHouse-Sink/0.2.0") - .setClientMetadata(flinkMetadata), + {(Consumer) b -> b.setClientName("Flink-ClickHouse-Sink/0.2.0", flinkMetadata), userAgent("Flink-ClickHouse-Sink/0.2.0", "fv:flink/2.0.0; api:table")}, {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table, fv=flink/2.0.0"), userAgent(null, "api:table; fv:flink/2.0.0")}, - {(Consumer) b -> b.setClientMetadata(Collections.singletonMap("build", "a=b,c\\d")), - userAgent(null, "build:a=b,c\\d")}, - {(Consumer) b -> b.setClientMetadata(withEmptyValue), - userAgent(null, "fv:flink/2.0.0")}, - {(Consumer) b -> b.setClientMetadata(flinkMetadata).setClientMetadata(null), - userAgent(null, null)}, - {(Consumer) b -> b.setClientName("my-app/1.0") - .httpHeader(HttpHeaders.USER_AGENT, "custom-agent/2.0") - .setClientMetadata(Collections.singletonMap("api", "table")), + {(Consumer) b -> b.setClientName("my-app/1.0", + Collections.singletonMap("build", "a=b,c\\d")), + userAgent("my-app/1.0", "build:a=b,c\\d")}, + {(Consumer) b -> b.setClientName("my-app/1.0", withEmptyValue), + userAgent("my-app/1.0", "fv:flink/2.0.0")}, + {(Consumer) b -> b.setClientName("", flinkMetadata), + userAgent(null, "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName(" ", flinkMetadata), + userAgent(null, "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table") + .setClientName("my-app/1.0"), + userAgent("my-app/1.0", "api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table") + .setClientName("my-app/1.0", flinkMetadata), + userAgent("my-app/1.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0"), + userAgent("new-app/2.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0", null), + userAgent("new-app/2.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0", Collections.emptyMap()), + userAgent("new-app/2.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", Collections.singletonMap("api", "table")) + .httpHeader(HttpHeaders.USER_AGENT, "custom-agent/2.0"), userAgent("custom-agent/2.0", "api:table")}, }; } @@ -82,8 +98,7 @@ public static Object[][] userAgentData() { @Test public void testClientMetadataKeptWhenApplicationPartChanges() throws Exception { try (Client client = newClientBuilder() - .setClientName("my-app/1.0") - .setClientMetadata(Collections.singletonMap("api", "table")) + .setClientName("my-app/1.0", Collections.singletonMap("api", "table")) .build()) { client.updateClientName("new-app/2.0"); assertUserAgent(client, null, userAgent("new-app/2.0", "api:table")); @@ -98,7 +113,7 @@ public void testClientMetadataKeptWhenApplicationPartChanges() throws Exception @Test public void testInvalidClientMetadataRejectedOnBuild() { Client.Builder builder = newClientBuilder() - .setClientMetadata(Collections.singletonMap("api", "table; os:other")); + .setClientName("my-app/1.0", Collections.singletonMap("api", "table; os:other")); Assert.expectThrows(ClientMisconfigurationException.class, builder::build); } diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index ccf55862b..c5ba9390c 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -263,8 +263,8 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `setClientNetworkBufferSize(int size)` | `size` - size in bytes | Sets size of a buffer in application memory space that is used to copy data between socket and application. | `300000` | `client_network_buffer_size` | | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | -| `setClientName(String clientName)` | `clientName` - application name string | Sets additional information about calling application. Will be passed as `User-Agent` header. | - | `client_name` | -| `setClientMetadata(Map metadata)` | `metadata` - key/value pairs | Adds key/value pairs to the comment group of the `User-Agent` header, in the iteration order of the map, before the `lv` and `os` entries. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_metadata` | +| `setClientName(String clientName)` | `clientName` - application name string | Sets the client name, which identifies the application. Will be passed as the first part of the `User-Agent` header. Use it instead of setting the `User-Agent` header. Does not change the client metadata. | - | `client_name` | +| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, before the `lv` and `os` entries. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -279,7 +279,7 @@ There are two fields in a query log that identify application originated a reque `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:)`. -What is set by `setClientName` method comes first. Key/value pairs set by `setClientMetadata` are added to the comment group before `lv` and `os`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group before `lv` and `os`. For example: ```java showLineNumbers @@ -289,8 +289,7 @@ metadata.put("api", "table"); Client client = new Client.Builder() // ... - .setClientName("my-app-01/1.0") - .setClientMetadata(metadata) + .setClientName("my-app-01/1.0", metadata) .build(); ``` will result in the following `http_user_agent` value: @@ -298,7 +297,8 @@ will result in the following `http_user_agent` value: my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux) ``` -Application can set own http header `User-Agent` to identify itself. But part `clickhouse-java-v2/0.12.0 (...)` will be appended to the end of the header. +Use `setClientName` to identify the application. Do not set the `User-Agent` header for this: a custom `User-Agent` header replaces +the client name, and the client still appends its own part `clickhouse-java-v2/0.12.0 (...)` to the end of the header. ### Operation Identification {#operation-identification} diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 67e811005..638f1757b 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -153,6 +153,7 @@ To add key/value pairs to the comment group of the header, set the `client_metad `client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux)`. **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. +Use the client name to identify the application. Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. ### Operation Identification {#operation-identification} diff --git a/docs/features.md b/docs/features.md index 945cc194a..795557786 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise (omitted when both are empty). The header does not name the HTTP transport library. `Client.Builder.setClientMetadata(Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client#updateClientName`). +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise (omitted when both are empty). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/docs/integration-client.md b/docs/integration-client.md index be2a11758..3e14bd092 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -92,6 +92,8 @@ In production environments, a single ClickHouse cluster often handles diverse wo Use a structured format such as `/` or `:/` (for example, `order-service/1.2.0` or `etl-worker:cdc/2.0.1`). +Always identify the application with the client name. Do not set the `User-Agent` header (for example, with `httpHeader("User-Agent", ...)`): a custom `User-Agent` header replaces the client name in the header. + **1. Client Builder** (static setup at client creation): ```java @@ -125,7 +127,7 @@ The Java Client communicates over HTTP and passes the client name as the leading order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux) ``` -To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientMetadata`. The pairs are added in the iteration order of the map, before `lv` and `os`: +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, before `lv` and `os`: ```java Map metadata = new LinkedHashMap<>(); @@ -136,8 +138,7 @@ Client client = new Client.Builder() .addEndpoint("http://localhost:8123") .setUsername("default") .setPassword("secret") - .setClientName("Flink-ClickHouse-Sink/0.2.0") - .setClientMetadata(metadata) + .setClientName("Flink-ClickHouse-Sink/0.2.0", metadata) .build(); ``` @@ -145,7 +146,7 @@ Client client = new Client.Builder() Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux) ``` -The same pairs can be set with the `client_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. +The same pairs can be set with the `client_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting Java Client applications. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index f3d8087b0..ea671524e 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -160,6 +160,8 @@ order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os To add key/value pairs to the comment group of the header, set the `client_metadata` property (for example, `client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux)`). +Always identify the application with the client name (`client_name` or `ApplicationName`). Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. + > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting JDBC applications. diff --git a/history/latest/3202.md b/history/latest/3202.md index c642e5dd4..d05f78c9a 100644 --- a/history/latest/3202.md +++ b/history/latest/3202.md @@ -1,6 +1,6 @@ - **[client-v2, jdbc-v2]** Breaking change: the `User-Agent` header now uses the standard ClickHouse client format, ` clickhouse-java-v2/ (lv:java/; os:)`: `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing `Apache-HttpClient/` is removed. Added - `Client.Builder#setClientMetadata(Map)` (property `client_metadata`) to add key/value pairs to the comment group. + `Client.Builder#setClientName(String, Map)` (property `client_metadata`) to add key/value pairs to the comment group. Queries that parse `http_user_agent` in `system.query_log` must use the new format. (https://github.com/ClickHouse/clickhouse-java/issues/3202) From 3abea182277b43f195ec61d0cb5b91e6f7135225 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:59:48 +0000 Subject: [PATCH 3/9] [client-v2] Compose the User-Agent once and cache it - The mandatory part (clickhouse-java-v2/, lv, os) is a static constant, composed once. - The client metadata is validated when the client is built and composed once per client. - The complete value is cached and composed again only when the application part changes (client name or custom User-Agent). - The application part is trimmed; use ClientUtils#isBlank. Co-Authored-By: Claude Opus 5.5 --- .../api/internal/HttpAPIClientHelper.java | 1 - .../api/internal/WorkloadIdentitySupport.java | 86 +++++++++++++++---- .../client/api/UserAgentUnitTest.java | 2 + .../internal/WorkloadIdentitySupportTest.java | 46 ++++++++++ docs/features.md | 2 +- history/latest/3202.md | 1 + 6 files changed, 117 insertions(+), 21 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 22a0b7a18..ad5b2bd9b 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -1188,7 +1188,6 @@ public RuntimeException wrapException(String message, Exception cause, String qu } private void correctUserAgentHeader(HttpRequest request, Map requestConfig) { - //TODO: implement cache for user-agent Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index e104bbfbd..6d1cd7569 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -14,6 +14,11 @@ * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), and the key/value pairs are * the client metadata ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. + *

+ * The value is composed once and cached. The mandatory part is the same for every client and is composed when the + * class is initialized. The client metadata is validated when {@link Client.Builder#build()} creates the client and + * parses its configuration, and is composed once per client. The complete value is composed again only when the + * application part changes. */ public class WorkloadIdentitySupport { @@ -21,32 +26,63 @@ public class WorkloadIdentitySupport { private static final String RESERVED_KEY_OS = "os"; - private final String clientUserAgent; + /** Start of the client part, up to the comment group. */ + private static final String CLIENT_PART_START = Client.CLIENT_USER_AGENT + Client.clientVersion + " ("; + + /** Mandatory end of the comment group. */ + private static final String CLIENT_PART_END = RESERVED_KEY_LANGUAGE_VERSION + ":java/" + + System.getProperty("java.version") + "; " + RESERVED_KEY_OS + ":" + System.getProperty("os.name") + ")"; + + /** Client part when there is no client metadata. */ + private static final String CLIENT_PART_WITHOUT_METADATA = CLIENT_PART_START + CLIENT_PART_END; + + private final String clientPart; + + private volatile UserAgent lastUserAgent; /** + * Composes the client part, and the complete value for the client name of the configuration. + * * @param configuration - parsed client configuration */ public WorkloadIdentitySupport(Map configuration) { - this.clientUserAgent = buildClientUserAgent(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); + this.clientPart = composeClientPart(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); + String application = applicationPart(null, configuration); + this.lastUserAgent = new UserAgent(application, compose(application)); } /** * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part. * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and - * the client name otherwise. A blank value counts as not set, and a blank application part is left out. + * the client name otherwise. Leading and trailing whitespace is removed, a blank value counts as not set, and a blank + * application part is left out. The value is cached, so it is composed again only when the application part + * changes. * * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null} * @param requestConfig - settings of the request * @return value of the {@code User-Agent} header */ public String getUserAgent(String customUserAgent, Map requestConfig) { - String application = isBlank(customUserAgent) - ? ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig) : customUserAgent; - return isBlank(application) ? clientUserAgent : application + " " + clientUserAgent; + String application = applicationPart(customUserAgent, requestConfig); + UserAgent userAgent = lastUserAgent; + if (!application.equals(userAgent.application)) { + userAgent = new UserAgent(application, compose(application)); + lastUserAgent = userAgent; + } + return userAgent.value; + } + + /** + * Returns the trimmed application part, or an empty string if it is not set. + */ + private static String applicationPart(String customUserAgent, Map config) { + String application = ClientUtils.isBlank(customUserAgent) + ? ClientConfigProperties.CLIENT_NAME.getOrDefault(config) : customUserAgent; + return ClientUtils.isBlank(application) ? "" : application.trim(); } - private static boolean isBlank(String str) { - return str == null || str.trim().isEmpty(); + private String compose(String application) { + return application.isEmpty() ? clientPart : application + " " + clientPart; } /** @@ -79,17 +115,29 @@ private static boolean isClientMetadataText(String str, String forbiddenChars) { return true; } - private static String buildClientUserAgent(Map clientMetadata) { - StringBuilder userAgent = new StringBuilder(); - userAgent.append(Client.CLIENT_USER_AGENT).append(Client.clientVersion).append(" ("); - if (clientMetadata != null) { - for (Map.Entry entry : clientMetadata.entrySet()) { - userAgent.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); - } + private static String composeClientPart(Map clientMetadata) { + if (clientMetadata == null || clientMetadata.isEmpty()) { + return CLIENT_PART_WITHOUT_METADATA; + } + StringBuilder clientPart = new StringBuilder(CLIENT_PART_START); + for (Map.Entry entry : clientMetadata.entrySet()) { + clientPart.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); + } + return clientPart.append(CLIENT_PART_END).toString(); + } + + /** + * Complete value of the {@code User-Agent} header for one application part. + */ + private static final class UserAgent { + + private final String application; + + private final String value; + + private UserAgent(String application, String value) { + this.application = application; + this.value = value; } - userAgent.append(RESERVED_KEY_LANGUAGE_VERSION).append(":java/").append(System.getProperty("java.version")) - .append("; ").append(RESERVED_KEY_OS).append(':').append(System.getProperty("os.name")) - .append(')'); - return userAgent.toString(); } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index 0f8f98f9a..b8365a51d 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -92,6 +92,8 @@ public static Object[][] userAgentData() { {(Consumer) b -> b.setClientName("my-app/1.0", Collections.singletonMap("api", "table")) .httpHeader(HttpHeaders.USER_AGENT, "custom-agent/2.0"), userAgent("custom-agent/2.0", "api:table")}, + {(Consumer) b -> b.setClientName(" my-app/1.0 "), + userAgent("my-app/1.0", null)}, }; } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java new file mode 100644 index 000000000..1188d4156 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -0,0 +1,46 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class WorkloadIdentitySupportTest { + + @Test + public void testUserAgentComposedOncePerApplicationPart() { + Map rawConfig = new HashMap<>(); + rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); + rawConfig.put(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table"); + Map config = ClientConfigProperties.parseConfigMap(rawConfig); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + + String userAgent = identity.getUserAgent(null, config); + Assert.assertEquals(userAgent, userAgent("my-app/1.0 ", "api:table; ")); + Assert.assertSame(identity.getUserAgent(null, new HashMap<>(config)), userAgent); + Assert.assertSame(identity.getUserAgent(" ", config), userAgent); + + String customUserAgent = identity.getUserAgent(" custom-agent/2.0 ", config); + Assert.assertEquals(customUserAgent, userAgent("custom-agent/2.0 ", "api:table; ")); + Assert.assertSame(identity.getUserAgent("custom-agent/2.0", config), customUserAgent); + + Assert.assertEquals(identity.getUserAgent(null, config), userAgent); + } + + @Test + public void testMandatoryPartSharedByClients() { + Map config = Collections.emptyMap(); + String userAgent = new WorkloadIdentitySupport(config).getUserAgent(null, config); + Assert.assertEquals(userAgent, userAgent("", "")); + Assert.assertSame(new WorkloadIdentitySupport(config).getUserAgent(null, config), userAgent); + } + + private static String userAgent(String application, String metadata) { + return application + "clickhouse-java-v2/" + Client.clientVersion + " (" + metadata + + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + ")"; + } +} diff --git a/docs/features.md b/docs/features.md index 795557786..201a4dfa2 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise (omitted when both are empty). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/history/latest/3202.md b/history/latest/3202.md index d05f78c9a..1a33661e5 100644 --- a/history/latest/3202.md +++ b/history/latest/3202.md @@ -2,5 +2,6 @@ ` clickhouse-java-v2/ (lv:java/; os:)`: `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing `Apache-HttpClient/` is removed. Added `Client.Builder#setClientName(String, Map)` (property `client_metadata`) to add key/value pairs to the comment group. + Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. Queries that parse `http_user_agent` in `system.query_log` must use the new format. (https://github.com/ClickHouse/clickhouse-java/issues/3202) From d7be3dadb0ccae68252fc8e0bc610dacfc5f4d3c Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:23:37 +0000 Subject: [PATCH 4/9] [client-v2] Hold the cached User-Agent in an AtomicReference SonarCloud rule java:S3077 flags the volatile reference to the cached User-Agent. Use an AtomicReference, as CredentialsManager does. The behavior does not change. Co-Authored-By: Claude Opus 5.5 --- .../client/api/internal/WorkloadIdentitySupport.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index 6d1cd7569..b0cdf800b 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -5,6 +5,7 @@ import com.clickhouse.client.api.ClientMisconfigurationException; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; /** * Builds the identification of the client and of the application that uses it. The identification is the value of @@ -38,7 +39,7 @@ public class WorkloadIdentitySupport { private final String clientPart; - private volatile UserAgent lastUserAgent; + private final AtomicReference lastUserAgent = new AtomicReference<>(); /** * Composes the client part, and the complete value for the client name of the configuration. @@ -48,7 +49,7 @@ public class WorkloadIdentitySupport { public WorkloadIdentitySupport(Map configuration) { this.clientPart = composeClientPart(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); String application = applicationPart(null, configuration); - this.lastUserAgent = new UserAgent(application, compose(application)); + this.lastUserAgent.set(new UserAgent(application, compose(application))); } /** @@ -64,10 +65,10 @@ public WorkloadIdentitySupport(Map configuration) { */ public String getUserAgent(String customUserAgent, Map requestConfig) { String application = applicationPart(customUserAgent, requestConfig); - UserAgent userAgent = lastUserAgent; + UserAgent userAgent = lastUserAgent.get(); if (!application.equals(userAgent.application)) { userAgent = new UserAgent(application, compose(application)); - lastUserAgent = userAgent; + lastUserAgent.set(userAgent); } return userAgent.value; } From 8fa4ebdfb84e8a6e71892307000435022ee532ea Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:28:39 +0000 Subject: [PATCH 5/9] [client-v2] Compose the User-Agent client part in one method Review follow-up for #3213: - Remove the static constants for parts of the User-Agent (start, end, value without metadata). composeClientPart now composes the complete client part with one code path, once per client, when the client is created. This replaces the per-JVM constants from 3abea1822. - Read client_metadata only if it is set (applyIfSet), with no default. - Replace 0x20 / 0x7E with isPrintableAscii (' ' to '~'), and explain the forbidden characters of a metadata entry. - Tests: replace testMandatoryPartSharedByClients (it pinned the shared static part) with a test for no / empty metadata that also catches state shared between instances; add rows for the ends of the printable ASCII range (0x1F, '~', 0x7F). Implements: https://github.com/ClickHouse/clickhouse-java/issues/3202 Co-Authored-By: Claude Opus 5.5 --- .../api/internal/WorkloadIdentitySupport.java | 49 ++++++++++--------- .../api/ClientConfigPropertiesTest.java | 3 +- .../internal/WorkloadIdentitySupportTest.java | 18 +++++-- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index b0cdf800b..0e2971011 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -5,6 +5,7 @@ import com.clickhouse.client.api.ClientMisconfigurationException; import java.util.Map; +import java.util.StringJoiner; import java.util.concurrent.atomic.AtomicReference; /** @@ -16,9 +17,8 @@ * the client metadata ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. *

- * The value is composed once and cached. The mandatory part is the same for every client and is composed when the - * class is initialized. The client metadata is validated when {@link Client.Builder#build()} creates the client and - * parses its configuration, and is composed once per client. The complete value is composed again only when the + * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. The client part is + * composed once, when the client is created. The complete value is cached, and is composed again only when the * application part changes. */ public class WorkloadIdentitySupport { @@ -27,16 +27,6 @@ public class WorkloadIdentitySupport { private static final String RESERVED_KEY_OS = "os"; - /** Start of the client part, up to the comment group. */ - private static final String CLIENT_PART_START = Client.CLIENT_USER_AGENT + Client.clientVersion + " ("; - - /** Mandatory end of the comment group. */ - private static final String CLIENT_PART_END = RESERVED_KEY_LANGUAGE_VERSION + ":java/" - + System.getProperty("java.version") + "; " + RESERVED_KEY_OS + ":" + System.getProperty("os.name") + ")"; - - /** Client part when there is no client metadata. */ - private static final String CLIENT_PART_WITHOUT_METADATA = CLIENT_PART_START + CLIENT_PART_END; - private final String clientPart; private final AtomicReference lastUserAgent = new AtomicReference<>(); @@ -47,7 +37,7 @@ public class WorkloadIdentitySupport { * @param configuration - parsed client configuration */ public WorkloadIdentitySupport(Map configuration) { - this.clientPart = composeClientPart(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration)); + this.clientPart = composeClientPart(configuration); String application = applicationPart(null, configuration); this.lastUserAgent.set(new UserAgent(application, compose(application))); } @@ -99,6 +89,8 @@ public static void checkClientMetadataEntry(String key, String value) { throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey() + "' entry '" + key + "': the key is reserved by the client"); } + // ':' separates a key from its value, ';' separates entries, '(' and ')' delimit the comment group, + // and ',' and '=' are separators of the client_metadata property value if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) { throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey() + "' entry '" + key + "=" + value + "': keys and values must be printable ASCII, a key must not" @@ -109,22 +101,31 @@ public static void checkClientMetadataEntry(String key, String value) { private static boolean isClientMetadataText(String str, String forbiddenChars) { for (int i = 0; i < str.length(); i++) { char ch = str.charAt(i); - if (ch < 0x20 || ch > 0x7E || forbiddenChars.indexOf(ch) >= 0) { + if (!isPrintableAscii(ch) || forbiddenChars.indexOf(ch) >= 0) { return false; } } return true; } - private static String composeClientPart(Map clientMetadata) { - if (clientMetadata == null || clientMetadata.isEmpty()) { - return CLIENT_PART_WITHOUT_METADATA; - } - StringBuilder clientPart = new StringBuilder(CLIENT_PART_START); - for (Map.Entry entry : clientMetadata.entrySet()) { - clientPart.append(entry.getKey()).append(':').append(entry.getValue()).append("; "); - } - return clientPart.append(CLIENT_PART_END).toString(); + /** + * Returns {@code true} if the character is printable ASCII: from the space character to {@code '~'}. + */ + private static boolean isPrintableAscii(char ch) { + return ch >= ' ' && ch <= '~'; + } + + /** + * Composes the client part: {@code clickhouse-java-v2/ (:; ...; lv:java/; + * os:)}. The key/value pairs are the client metadata, if it is set. + */ + private static String composeClientPart(Map configuration) { + StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); + ClientConfigProperties.CLIENT_METADATA.>applyIfSet(configuration, + metadata -> metadata.forEach((key, value) -> comment.add(key + ":" + value))); + comment.add(RESERVED_KEY_LANGUAGE_VERSION + ":java/" + System.getProperty("java.version")); + comment.add(RESERVED_KEY_OS + ":" + System.getProperty("os.name")); + return comment.toString(); } /** diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index d044ff731..4f9625ce2 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -85,6 +85,7 @@ public static Object[][] validClientMetadata() { {"fv=flink/2.0.0,api=table", new String[]{"fv", "flink/2.0.0", "api", "table"}}, {" api = table , os_version = Mac OS X ", new String[]{"api", "table", "os_version", "Mac OS X"}}, {"url=http://host:8123/path?x=1", new String[]{"url", "http://host:8123/path?x=1"}}, + {"ver~=~1.0", new String[]{"ver~", "~1.0"}}, {"api=,fv=flink/2.0.0,=x", new String[]{"fv", "flink/2.0.0"}}, {"", new String[0]}, }; @@ -105,7 +106,7 @@ public static Object[][] invalidClientMetadata() { return new Object[][]{ {"a:b=c"}, {"a;b=c"}, {"a(b=c"}, {"a)b=c"}, {"a\\,b=c"}, {"a\\=b=c"}, {"my key=c"}, {"k=v;x"}, {"k=v(x"}, {"k=v)x"}, - {"k=a\nb"}, {"k=a\tb"}, {"k=caf\u00e9"}, {"k\u00e9=v"}, + {"k=a\nb"}, {"k=a\tb"}, {"k=a\u001fb"}, {"k=a\u007fb"}, {"k\u007f=v"}, {"k=caf\u00e9"}, {"k\u00e9=v"}, {"lv=java/8"}, {"os=Linux"}, {"OS=Linux"}, {"fv=flink/2.0.0,api=table; lv:java/8"}, {",api=table"}, {"fv=flink/2.0.0,,api=table"}, {"fv=flink/2.0.0,api,ver=1"}, diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index 1188d4156..a177a7923 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -32,11 +32,21 @@ public void testUserAgentComposedOncePerApplicationPart() { } @Test - public void testMandatoryPartSharedByClients() { + public void testUserAgentWithoutClientMetadata() { + Map withMetadata = ClientConfigProperties.parseConfigMap( + Collections.singletonMap(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table")); + WorkloadIdentitySupport identityWithMetadata = new WorkloadIdentitySupport(withMetadata); + Map config = Collections.emptyMap(); - String userAgent = new WorkloadIdentitySupport(config).getUserAgent(null, config); - Assert.assertEquals(userAgent, userAgent("", "")); - Assert.assertSame(new WorkloadIdentitySupport(config).getUserAgent(null, config), userAgent); + Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), userAgent("", "")); + + Map emptyMetadata = ClientConfigProperties.parseConfigMap( + Collections.singletonMap(ClientConfigProperties.CLIENT_METADATA.getKey(), "")); + Assert.assertEquals(new WorkloadIdentitySupport(emptyMetadata).getUserAgent(null, emptyMetadata), + userAgent("", "")); + + Assert.assertEquals(identityWithMetadata.getUserAgent("my-app/1.0", withMetadata), + userAgent("my-app/1.0 ", "api:table; ")); } private static String userAgent(String application, String metadata) { From 0fbad2e370d04dbc074547fa11f2a642b8ddb3db Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:31:53 +0000 Subject: [PATCH 6/9] [client-v2] Keep one User-Agent value per client, compose it again on updateClientName WorkloadIdentitySupport composes the User-Agent value for the client configuration once, in the constructor, and keeps it in a final field. The application part comes from the client-level User-Agent header if the configuration has one, and from the client name otherwise. A request with another application part (a User-Agent header or client name set for one operation) gets a new value and does not change the kept value. The cache of the latest value and its holder class are removed. Client#updateClientName now composes the value again and keeps it: HttpAPIClientHelper#updateWorkloadIdentity replaces the immutable WorkloadIdentitySupport instance. The client logs a warning when the kept User-Agent value is longer than 200 characters. A value composed for one request is not logged. Co-Authored-By: Claude Opus 5.5 --- .../com/clickhouse/client/api/Client.java | 7 ++ .../api/internal/HttpAPIClientHelper.java | 17 ++++- .../api/internal/WorkloadIdentitySupport.java | 70 +++++++++---------- .../api/internal/HttpAPIClientHelperTest.java | 65 +++++++++++++++++ .../internal/WorkloadIdentitySupportTest.java | 27 +++++-- docs/features.md | 2 +- history/latest/3202.md | 3 +- 7 files changed, 146 insertions(+), 45 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 68af64aa6..9dad6550a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -2599,8 +2599,15 @@ public void setDBRoles(Collection dbRoles) { this.unmodifiableDbRolesView = ImmutableList.copyOf(tmp); } + /** + * Updates the client name for all subsequent requests created by this client. The client composes the value of + * the {@code User-Agent} header again, and keeps it. + * + * @param name - client application name, for example {@code my-app/1.0} + */ public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); + this.httpClientHelper.updateWorkloadIdentity(this.configuration); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index ad5b2bd9b..1826bfd93 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -109,6 +109,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.regex.Pattern; @@ -133,7 +134,7 @@ public class HttpAPIClientHelper { private final Set defaultRetryCauses; - private final WorkloadIdentitySupport workloadIdentitySupport; + private final AtomicReference workloadIdentitySupport; private final Object metricsRegistry; ConnPoolControl poolControl; @@ -169,7 +170,17 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - this.workloadIdentitySupport = new WorkloadIdentitySupport(configuration); + this.workloadIdentitySupport = new AtomicReference<>(new WorkloadIdentitySupport(configuration)); + } + + /** + * Composes the value of the {@code User-Agent} header again for the client configuration, and keeps it. Call it + * after the client name changes. + * + * @param configuration - client configuration + */ + public void updateWorkloadIdentity(Map configuration) { + workloadIdentitySupport.set(new WorkloadIdentitySupport(configuration)); } /** @@ -1191,7 +1202,7 @@ private void correctUserAgentHeader(HttpRequest request, Map req Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); - request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.getUserAgent( + request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.get().getUserAgent( userAgentHeader == null ? null : userAgentHeader.getValue(), requestConfig)); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index 0e2971011..1f867c7a3 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -3,51 +3,71 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientMisconfigurationException; +import org.apache.hc.core5.http.HttpHeaders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Map; import java.util.StringJoiner; -import java.util.concurrent.atomic.AtomicReference; /** * Builds the identification of the client and of the application that uses it. The identification is the value of * the {@code User-Agent} header, in the standard ClickHouse client format: * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:)}. *

- * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), and the key/value pairs are - * the client metadata ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of + * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), or the custom + * {@code User-Agent} header if the application set one. The key/value pairs are the client metadata + * ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. *

- * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. The client part is - * composed once, when the client is created. The complete value is cached, and is composed again only when the - * application part changes. + * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. An instance composes + * the value for the client configuration once and keeps it. When the client name changes, the client creates a new + * instance. */ public class WorkloadIdentitySupport { + private static final Logger LOG = LoggerFactory.getLogger(WorkloadIdentitySupport.class); + private static final String RESERVED_KEY_LANGUAGE_VERSION = "lv"; private static final String RESERVED_KEY_OS = "os"; + /** + * Length of the {@code User-Agent} value above which the client logs a warning. Common browsers send about 120 + * characters. + */ + private static final int USER_AGENT_WARNING_LENGTH = 200; + private final String clientPart; - private final AtomicReference lastUserAgent = new AtomicReference<>(); + private final String clientApplication; + + private final String clientUserAgent; /** - * Composes the client part, and the complete value for the client name of the configuration. + * Composes the value of the {@code User-Agent} header for the client configuration, and logs a warning if the + * value is longer than 200 characters. * * @param configuration - parsed client configuration */ public WorkloadIdentitySupport(Map configuration) { this.clientPart = composeClientPart(configuration); - String application = applicationPart(null, configuration); - this.lastUserAgent.set(new UserAgent(application, compose(application))); + this.clientApplication = applicationPart( + (String) configuration.get(ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT)), configuration); + this.clientUserAgent = compose(clientApplication); + if (clientUserAgent.length() > USER_AGENT_WARNING_LENGTH) { + LOG.warn("The User-Agent value is {} characters long, more than {}: '{}'. Use a shorter client name or" + + " less client metadata.", clientUserAgent.length(), USER_AGENT_WARNING_LENGTH, clientUserAgent); + } } /** * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part. * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and * the client name otherwise. Leading and trailing whitespace is removed, a blank value counts as not set, and a blank - * application part is left out. The value is cached, so it is composed again only when the application part - * changes. + * application part is left out. If the application part is the one of the client configuration, the value that + * this instance keeps is returned. Otherwise, for example for a {@code User-Agent} header of one operation, a new + * value is composed and not kept. * * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null} * @param requestConfig - settings of the request @@ -55,21 +75,16 @@ public WorkloadIdentitySupport(Map configuration) { */ public String getUserAgent(String customUserAgent, Map requestConfig) { String application = applicationPart(customUserAgent, requestConfig); - UserAgent userAgent = lastUserAgent.get(); - if (!application.equals(userAgent.application)) { - userAgent = new UserAgent(application, compose(application)); - lastUserAgent.set(userAgent); - } - return userAgent.value; + return application.equals(clientApplication) ? clientUserAgent : compose(application); } /** * Returns the trimmed application part, or an empty string if it is not set. */ private static String applicationPart(String customUserAgent, Map config) { - String application = ClientUtils.isBlank(customUserAgent) + String value = ClientUtils.isBlank(customUserAgent) ? ClientConfigProperties.CLIENT_NAME.getOrDefault(config) : customUserAgent; - return ClientUtils.isBlank(application) ? "" : application.trim(); + return ClientUtils.isBlank(value) ? "" : value.trim(); } private String compose(String application) { @@ -127,19 +142,4 @@ private static String composeClientPart(Map configuration) { comment.add(RESERVED_KEY_OS + ":" + System.getProperty("os.name")); return comment.toString(); } - - /** - * Complete value of the {@code User-Agent} header for one application part. - */ - private static final class UserAgent { - - private final String application; - - private final String value; - - private UserAgent(String application, String value) { - this.application = application; - this.value = value; - } - } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index ddc3aa8fc..7037d3708 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -1,5 +1,6 @@ package com.clickhouse.client.api.internal; +import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientFaultCause; import com.clickhouse.client.api.ServerException; @@ -54,6 +55,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; public class HttpAPIClientHelperTest { @@ -512,6 +514,69 @@ private static Map compressionConfig(boolean clientCompression, return reqConfig; } + /** + * {@link Client#updateClientName(String)} composes the {@code User-Agent} value again and keeps it, so every + * following request with that client name gets the same value. + */ + @Test + public void testUserAgentKeptAfterClientNameUpdate() throws Exception { + try (Client client = new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("") + .setClientName("my-app/1.0") + .build()) { + client.updateClientName("new-app/2.0"); + Field helperField = Client.class.getDeclaredField("httpClientHelper"); + helperField.setAccessible(true); + HttpAPIClientHelper helper = (HttpAPIClientHelper) helperField.get(client); + + String userAgent = userAgentHeader(helper, "new-app/2.0"); + assertEquals(userAgent, "new-app/2.0 " + clientPart()); + assertSame(userAgentHeader(helper, "new-app/2.0"), userAgent); + } + } + + /** + * A {@code User-Agent} value of the client configuration that is longer than 200 characters is logged as a + * warning, when the client is created and when the client name changes. A value composed for one request is not + * logged, so that the log does not repeat for every request. + */ + @Test + public void testLongUserAgentLoggedAsWarning() { + String atLimit = String.join("", Collections.nCopies(200 - 1 - clientPart().length(), "x")); + String overLimit = atLimit + "x"; + String warning = "The User-Agent value is "; + + assertFalse(captureStdErr(() -> newHelper(atLimit)).contains(warning)); + assertTrue(captureStdErr(() -> newHelper(overLimit)).contains(warning + "201 characters long, more than 200")); + + HttpAPIClientHelper helper = newHelper("my-app/1.0"); + assertFalse(captureStdErr(() -> userAgentHeader(helper, overLimit)).contains(warning)); + assertTrue(captureStdErr(() -> helper.updateWorkloadIdentity(clientNameConfig(overLimit))).contains(warning)); + } + + private static HttpAPIClientHelper newHelper(String clientName) { + return HttpAPIClientHelperFactory.newHelper(clientNameConfig(clientName), LZ4Factory.fastestInstance()); + } + + private static Map clientNameConfig(String clientName) { + Map config = new HashMap<>(); + config.put(ClientConfigProperties.CLIENT_NAME.getKey(), clientName); + return config; + } + + private static String userAgentHeader(HttpAPIClientHelper helper, String clientName) { + HttpPost req = helper.createRequest(new HttpEndpoint("localhost", 8123, false, "/"), + clientNameConfig(clientName), "SELECT 1").getDelegate(); + return headerValue(req, HttpHeaders.USER_AGENT); + } + + private static String clientPart() { + return "clickhouse-java-v2/" + Client.clientVersion + " (lv:java/" + System.getProperty("java.version") + + "; os:" + System.getProperty("os.name") + ")"; + } + private static String headerValue(HttpPost req, String name) { Header header = req.getFirstHeader(name); return header == null ? null : header.getValue(); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index a177a7923..ceece8e3b 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -2,6 +2,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; +import org.apache.hc.core5.http.HttpHeaders; import org.testng.Assert; import org.testng.annotations.Test; @@ -12,7 +13,7 @@ public class WorkloadIdentitySupportTest { @Test - public void testUserAgentComposedOncePerApplicationPart() { + public void testUserAgentComposedOncePerClient() { Map rawConfig = new HashMap<>(); rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); rawConfig.put(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table"); @@ -24,11 +25,27 @@ public void testUserAgentComposedOncePerApplicationPart() { Assert.assertSame(identity.getUserAgent(null, new HashMap<>(config)), userAgent); Assert.assertSame(identity.getUserAgent(" ", config), userAgent); - String customUserAgent = identity.getUserAgent(" custom-agent/2.0 ", config); - Assert.assertEquals(customUserAgent, userAgent("custom-agent/2.0 ", "api:table; ")); - Assert.assertSame(identity.getUserAgent("custom-agent/2.0", config), customUserAgent); + Map requestConfig = new HashMap<>(config); + requestConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "op-app/3.0"); + Assert.assertEquals(identity.getUserAgent(null, requestConfig), userAgent("op-app/3.0 ", "api:table; ")); + Assert.assertEquals(identity.getUserAgent(" custom-agent/2.0 ", config), + userAgent("custom-agent/2.0 ", "api:table; ")); - Assert.assertEquals(identity.getUserAgent(null, config), userAgent); + Assert.assertSame(identity.getUserAgent(null, config), userAgent); + } + + @Test + public void testUserAgentComposedOnceForConfiguredHeader() { + Map rawConfig = new HashMap<>(); + rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); + rawConfig.put(ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT), "custom-agent/2.0"); + Map config = ClientConfigProperties.parseConfigMap(rawConfig); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + + String userAgent = identity.getUserAgent("custom-agent/2.0", config); + Assert.assertEquals(userAgent, userAgent("custom-agent/2.0 ", "")); + Assert.assertSame(identity.getUserAgent("custom-agent/2.0", config), userAgent); + Assert.assertEquals(identity.getUserAgent(null, config), userAgent("my-app/1.0 ", "")); } @Test diff --git a/docs/features.md b/docs/features.md index 201a4dfa2..f037343c3 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/history/latest/3202.md b/history/latest/3202.md index 1a33661e5..a02faa4f6 100644 --- a/history/latest/3202.md +++ b/history/latest/3202.md @@ -2,6 +2,7 @@ ` clickhouse-java-v2/ (lv:java/; os:)`: `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing `Apache-HttpClient/` is removed. Added `Client.Builder#setClientName(String, Map)` (property `client_metadata`) to add key/value pairs to the comment group. - Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. + Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. The client + logs a warning if the `User-Agent` value of the client configuration is longer than 200 characters. Queries that parse `http_user_agent` in `system.query_log` must use the new format. (https://github.com/ClickHouse/clickhouse-java/issues/3202) From 014290c10cb7f454a626616a101c48265905d5bb Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:28:25 +0000 Subject: [PATCH 7/9] [client-v2] Create the workload identity once per client, add ahc, rename client_id_metadata - HttpAPIClientHelper keeps one final WorkloadIdentitySupport (no AtomicReference). Client#updateClientName calls WorkloadIdentitySupport#updateClientName, which composes the value again and keeps it (synchronized; volatile strings). - A blank application part for one operation gets the kept value. - Client metadata can set lv, os and ahc (keys compared ignoring case); the client adds its own entry only for a key that is not set. - New ahc entry: the Apache HttpClient version, from the generated class DependencyVersions (templating-maven-plugin), so the version is correct also in the shaded jar. - Property client_metadata renamed to client_id_metadata. - compose -> composeUserAgent; START_OF_ASCII_RANGE / END_OF_ASCII_RANGE constants. Co-Authored-By: Claude Opus 5.5 --- client-v2/pom.xml | 13 ++ .../api/internal/DependencyVersions.java | 16 +++ .../com/clickhouse/client/api/Client.java | 24 ++-- .../client/api/ClientConfigProperties.java | 2 +- .../api/internal/HttpAPIClientHelper.java | 11 +- .../api/internal/WorkloadIdentitySupport.java | 116 +++++++++++------- .../clickhouse/client/HttpTransportTests.java | 6 +- .../api/ClientConfigPropertiesTest.java | 8 +- .../client/api/UserAgentUnitTest.java | 29 ++++- .../api/internal/HttpAPIClientHelperTest.java | 4 +- .../internal/WorkloadIdentitySupportTest.java | 82 ++++++++++++- docs/clickhouse-docs/client.mdx | 8 +- docs/clickhouse-docs/jdbc.mdx | 6 +- docs/features.md | 4 +- docs/integration-client.md | 10 +- docs/integration-jdbc.md | 6 +- history/latest/3202.md | 13 +- .../com/clickhouse/jdbc/ConnectionTest.java | 8 +- pom.xml | 6 + 19 files changed, 267 insertions(+), 105 deletions(-) create mode 100644 client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java diff --git a/client-v2/pom.xml b/client-v2/pom.xml index ad1f767cf..fb1b0ab90 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -231,6 +231,19 @@ 8 + + + org.codehaus.mojo + templating-maven-plugin + + + filter-sources + + filter-sources + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java new file mode 100644 index 000000000..fc07d746b --- /dev/null +++ b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java @@ -0,0 +1,16 @@ +package com.clickhouse.client.api.internal; + +/** + * Versions of the libraries that the client is built with. The build generates this class from a template, so the + * versions are correct also when the libraries are shaded into the client and cannot report their own version. + */ +public final class DependencyVersions { + + /** + * Version of the Apache HttpClient library. + */ + public static final String APACHE_HTTP_CLIENT = "${apache.httpclient.version}"; + + private DependencyVersions() { + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 9dad6550a..7cd92b185 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1162,28 +1162,28 @@ public Builder setClientName(String clientName) { /** * Sets the client name and the client metadata, which identify the application to the server. See * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the - * {@code User-Agent} header, in the iteration order of {@code metadata}, before its own {@code lv} and - * {@code os} entries. For example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} - * and {@code api=table} give - * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; os:)}. + * {@code User-Agent} header, in the iteration order of {@code metadata}, and then its own {@code lv}, + * {@code os} and {@code ahc} (Apache HttpClient version) entries if the metadata does not set these keys. For + * example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give + * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; + * os:; ahc:)}. *

* Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be - * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, a value must not contain any of - * {@code ;()}, and the keys {@code lv} and {@code os} are reserved; otherwise {@link #build()} throws - * {@link ClientMisconfigurationException}. The metadata is a client-level setting: it is kept when the client - * name changes with {@link Client#updateClientName(String)}. + * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not contain any of + * {@code ;()}; otherwise {@link #build()} throws {@link ClientMisconfigurationException}. The metadata is a + * client-level setting: it is kept when the client name changes with {@link Client#updateClientName(String)}. * * @param clientName - client application name, for example {@code my-app/1.0} * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata * @return same instance of the builder - * @see ClientConfigProperties#CLIENT_METADATA + * @see ClientConfigProperties#CLIENT_ID_METADATA */ public Builder setClientName(String clientName, Map metadata) { setClientName(clientName); if (metadata == null || metadata.isEmpty()) { - this.configuration.remove(ClientConfigProperties.CLIENT_METADATA.getKey()); + this.configuration.remove(ClientConfigProperties.CLIENT_ID_METADATA.getKey()); } else { - this.configuration.put(ClientConfigProperties.CLIENT_METADATA.getKey(), + this.configuration.put(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), ClientConfigProperties.fromKeyValuePairs(metadata)); } return this; @@ -2607,7 +2607,7 @@ public void setDBRoles(Collection dbRoles) { */ public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); - this.httpClientHelper.updateWorkloadIdentity(this.configuration); + this.httpClientHelper.updateClientName(this.configuration); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index ae9bdb0fb..17d4f701c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -265,7 +265,7 @@ public Object parseValue(String value) { * setting: it is not read from operation settings. See {@link Client.Builder#setClientName(String, Map)} for the * format and the rules for keys and values. */ - CLIENT_METADATA("client_metadata", Map.class) { + CLIENT_ID_METADATA("client_id_metadata", Map.class) { @Override @SuppressWarnings("unchecked") public Object parseValue(String value) { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 1826bfd93..3a3b57c9f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -109,7 +109,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.regex.Pattern; @@ -134,7 +133,7 @@ public class HttpAPIClientHelper { private final Set defaultRetryCauses; - private final AtomicReference workloadIdentitySupport; + private final WorkloadIdentitySupport workloadIdentitySupport; private final Object metricsRegistry; ConnPoolControl poolControl; @@ -170,7 +169,7 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - this.workloadIdentitySupport = new AtomicReference<>(new WorkloadIdentitySupport(configuration)); + this.workloadIdentitySupport = new WorkloadIdentitySupport(configuration); } /** @@ -179,8 +178,8 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi * * @param configuration - client configuration */ - public void updateWorkloadIdentity(Map configuration) { - workloadIdentitySupport.set(new WorkloadIdentitySupport(configuration)); + public void updateClientName(Map configuration) { + workloadIdentitySupport.updateClientName(configuration); } /** @@ -1202,7 +1201,7 @@ private void correctUserAgentHeader(HttpRequest request, Map req Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); - request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.get().getUserAgent( + request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.getUserAgent( userAgentHeader == null ? null : userAgentHeader.getValue(), requestConfig)); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index 1f867c7a3..4cc640651 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -7,30 +7,41 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.LinkedHashMap; import java.util.Map; import java.util.StringJoiner; /** * Builds the identification of the client and of the application that uses it. The identification is the value of * the {@code User-Agent} header, in the standard ClickHouse client format: - * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:)}. + * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:; + * ahc:)}. *

* The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), or the custom * {@code User-Agent} header if the application set one. The key/value pairs are the client metadata - * ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of + * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). A transport sends the value of * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. *

- * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. An instance composes - * the value for the client configuration once and keeps it. When the client name changes, the client creates a new - * instance. + * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. The client creates + * one instance. The instance composes the value for the client configuration when it is created and when the client + * name changes, and keeps it. */ -public class WorkloadIdentitySupport { +public final class WorkloadIdentitySupport { private static final Logger LOG = LoggerFactory.getLogger(WorkloadIdentitySupport.class); - private static final String RESERVED_KEY_LANGUAGE_VERSION = "lv"; + private static final String METADATA_KEY_LANGUAGE_VERSION = "lv"; - private static final String RESERVED_KEY_OS = "os"; + private static final String METADATA_KEY_OS = "os"; + + private static final String METADATA_KEY_HTTP_CLIENT = "ahc"; + + /** + * First and last characters of the printable ASCII range, which client metadata keys and values must be in. + */ + private static final char START_OF_ASCII_RANGE = ' '; + + private static final char END_OF_ASCII_RANGE = '~'; /** * Length of the {@code User-Agent} value above which the client logs a warning. Common browsers send about 120 @@ -40,34 +51,48 @@ public class WorkloadIdentitySupport { private final String clientPart; - private final String clientApplication; + private volatile String clientApplication; - private final String clientUserAgent; + private volatile String clientUserAgent; /** - * Composes the value of the {@code User-Agent} header for the client configuration, and logs a warning if the - * value is longer than 200 characters. + * Composes the client part and the value of the {@code User-Agent} header for the client configuration. * * @param configuration - parsed client configuration */ public WorkloadIdentitySupport(Map configuration) { this.clientPart = composeClientPart(configuration); - this.clientApplication = applicationPart( + updateClientName(configuration); + } + + /** + * Composes the value of the {@code User-Agent} header for the client configuration again, and keeps it. Call it + * after the client name changes. Logs a warning if the value is longer than 200 characters. + *

+ * Updates are synchronized, so the kept application part and value always belong together. A request that runs + * at the same time as an update can get the value of the new client name. + * + * @param configuration - client configuration + */ + public synchronized void updateClientName(Map configuration) { + String application = applicationPart( (String) configuration.get(ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT)), configuration); - this.clientUserAgent = compose(clientApplication); - if (clientUserAgent.length() > USER_AGENT_WARNING_LENGTH) { + String userAgent = composeUserAgent(application); + if (userAgent.length() > USER_AGENT_WARNING_LENGTH) { LOG.warn("The User-Agent value is {} characters long, more than {}: '{}'. Use a shorter client name or" - + " less client metadata.", clientUserAgent.length(), USER_AGENT_WARNING_LENGTH, clientUserAgent); + + " less client metadata.", userAgent.length(), USER_AGENT_WARNING_LENGTH, userAgent); } + // the value is written first, so a request that reads the new application part also reads the new value + this.clientUserAgent = userAgent; + this.clientApplication = application; } /** * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part. * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and - * the client name otherwise. Leading and trailing whitespace is removed, a blank value counts as not set, and a blank - * application part is left out. If the application part is the one of the client configuration, the value that - * this instance keeps is returned. Otherwise, for example for a {@code User-Agent} header of one operation, a new - * value is composed and not kept. + * the client name otherwise. Leading and trailing whitespace is removed. If the application part is blank or is + * the one of the client configuration, the value that this instance keeps is returned. Otherwise, for example for + * a {@code User-Agent} header of one operation, a new value is composed and not kept. * * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null} * @param requestConfig - settings of the request @@ -75,7 +100,8 @@ public WorkloadIdentitySupport(Map configuration) { */ public String getUserAgent(String customUserAgent, Map requestConfig) { String application = applicationPart(customUserAgent, requestConfig); - return application.equals(clientApplication) ? clientUserAgent : compose(application); + return application.isEmpty() || application.equals(clientApplication) + ? clientUserAgent : composeUserAgent(application); } /** @@ -87,7 +113,7 @@ private static String applicationPart(String customUserAgent, Map= 0) { + if (ch < START_OF_ASCII_RANGE || ch > END_OF_ASCII_RANGE || forbiddenChars.indexOf(ch) >= 0) { return false; } } @@ -124,22 +145,31 @@ private static boolean isClientMetadataText(String str, String forbiddenChars) { } /** - * Returns {@code true} if the character is printable ASCII: from the space character to {@code '~'}. + * Composes the client part: {@code clickhouse-java-v2/ (:; ...)}. The key/value pairs are the + * client metadata, if it is set, then {@code lv:java/}, {@code os:} and + * {@code ahc:} if the client metadata does not set these keys (ignoring case). */ - private static boolean isPrintableAscii(char ch) { - return ch >= ' ' && ch <= '~'; + private static String composeClientPart(Map configuration) { + Map metadata = new LinkedHashMap<>(); + ClientConfigProperties.CLIENT_ID_METADATA.>applyIfSet(configuration, metadata::putAll); + addIfNotSet(metadata, METADATA_KEY_LANGUAGE_VERSION, "java/" + System.getProperty("java.version")); + addIfNotSet(metadata, METADATA_KEY_OS, System.getProperty("os.name")); + addIfNotSet(metadata, METADATA_KEY_HTTP_CLIENT, DependencyVersions.APACHE_HTTP_CLIENT); + + StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); + metadata.forEach((key, value) -> comment.add(key + ":" + value)); + return comment.toString(); } /** - * Composes the client part: {@code clickhouse-java-v2/ (:; ...; lv:java/; - * os:)}. The key/value pairs are the client metadata, if it is set. + * Adds the entry if the client metadata does not have the key. Keys are compared ignoring case. */ - private static String composeClientPart(Map configuration) { - StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); - ClientConfigProperties.CLIENT_METADATA.>applyIfSet(configuration, - metadata -> metadata.forEach((key, value) -> comment.add(key + ":" + value))); - comment.add(RESERVED_KEY_LANGUAGE_VERSION + ":java/" + System.getProperty("java.version")); - comment.add(RESERVED_KEY_OS + ":" + System.getProperty("os.name")); - return comment.toString(); + private static void addIfNotSet(Map metadata, String key, String value) { + for (String metadataKey : metadata.keySet()) { + if (metadataKey.equalsIgnoreCase(key)) { + return; + } + } + metadata.put(key, value); } } diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index a19eb8ba2..8b3fa31a4 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -1201,9 +1201,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider") public static Object[][] testUserAgentHasCompleteProductName_dataProvider() { return new Object[][] { - { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)") }, - { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")}, - { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")}}; + { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)") }, + { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)")}, + { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)")}}; } @Test(dataProvider = "testClientNameDataProvider") diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index 4f9625ce2..a721ad248 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -86,6 +86,7 @@ public static Object[][] validClientMetadata() { {" api = table , os_version = Mac OS X ", new String[]{"api", "table", "os_version", "Mac OS X"}}, {"url=http://host:8123/path?x=1", new String[]{"url", "http://host:8123/path?x=1"}}, {"ver~=~1.0", new String[]{"ver~", "~1.0"}}, + {"os=Android 14,lv=java/8", new String[]{"os", "Android 14", "lv", "java/8"}}, {"api=,fv=flink/2.0.0,=x", new String[]{"fv", "flink/2.0.0"}}, {"", new String[0]}, }; @@ -97,7 +98,7 @@ public void testClientMetadataParsed(String raw, String[] expectedEntries) { for (int i = 0; i < expectedEntries.length; i += 2) { expected.put(expectedEntries[i], expectedEntries[i + 1]); } - Map parsed = (Map) ClientConfigProperties.CLIENT_METADATA.parseValue(raw); + Map parsed = (Map) ClientConfigProperties.CLIENT_ID_METADATA.parseValue(raw); Assert.assertEquals(new ArrayList<>(parsed.entrySet()), new ArrayList<>(expected.entrySet())); } @@ -107,7 +108,6 @@ public static Object[][] invalidClientMetadata() { {"a:b=c"}, {"a;b=c"}, {"a(b=c"}, {"a)b=c"}, {"a\\,b=c"}, {"a\\=b=c"}, {"my key=c"}, {"k=v;x"}, {"k=v(x"}, {"k=v)x"}, {"k=a\nb"}, {"k=a\tb"}, {"k=a\u001fb"}, {"k=a\u007fb"}, {"k\u007f=v"}, {"k=caf\u00e9"}, {"k\u00e9=v"}, - {"lv=java/8"}, {"os=Linux"}, {"OS=Linux"}, {"fv=flink/2.0.0,api=table; lv:java/8"}, {",api=table"}, {"fv=flink/2.0.0,,api=table"}, {"fv=flink/2.0.0,api,ver=1"}, }; @@ -116,12 +116,12 @@ public static Object[][] invalidClientMetadata() { @Test(groups = {"unit"}, dataProvider = "invalidClientMetadata") public void testInvalidClientMetadataRejected(String raw) { Assert.expectThrows(ClientMisconfigurationException.class, - () -> ClientConfigProperties.CLIENT_METADATA.parseValue(raw)); + () -> ClientConfigProperties.CLIENT_ID_METADATA.parseValue(raw)); } @Test(groups = {"unit"}) public void testClientMetadataUnsetParsesToNull() { - Assert.assertNull(ClientConfigProperties.CLIENT_METADATA.parseValue(null)); + Assert.assertNull(ClientConfigProperties.CLIENT_ID_METADATA.parseValue(null)); } @Test(groups = {"unit"}) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index b8365a51d..bd0fb9e28 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -1,5 +1,6 @@ package com.clickhouse.client.api; +import com.clickhouse.client.api.internal.DependencyVersions; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.client.api.query.QuerySettings; import com.github.tomakehurst.wiremock.WireMockServer; @@ -54,6 +55,10 @@ public static Object[][] userAgentData() { flinkMetadata.put("fv", "flink/2.0.0"); flinkMetadata.put("api", "table"); + Map osMetadata = new LinkedHashMap<>(); + osMetadata.put("os", "Android 14"); + osMetadata.put("api", "table"); + Map withEmptyValue = new LinkedHashMap<>(); withEmptyValue.put("api", ""); withEmptyValue.put("fv", "flink/2.0.0"); @@ -62,7 +67,7 @@ public static Object[][] userAgentData() { {(Consumer) b -> { }, userAgent(null, null)}, {(Consumer) b -> b.setClientName("Flink-ClickHouse-Sink/0.2.0", flinkMetadata), userAgent("Flink-ClickHouse-Sink/0.2.0", "fv:flink/2.0.0; api:table")}, - {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table, fv=flink/2.0.0"), userAgent(null, "api:table; fv:flink/2.0.0")}, {(Consumer) b -> b.setClientName("my-app/1.0", @@ -74,10 +79,10 @@ public static Object[][] userAgentData() { userAgent(null, "fv:flink/2.0.0; api:table")}, {(Consumer) b -> b.setClientName(" ", flinkMetadata), userAgent(null, "fv:flink/2.0.0; api:table")}, - {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table") + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table") .setClientName("my-app/1.0"), userAgent("my-app/1.0", "api:table")}, - {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table") + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table") .setClientName("my-app/1.0", flinkMetadata), userAgent("my-app/1.0", "fv:flink/2.0.0; api:table")}, {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) @@ -94,6 +99,10 @@ public static Object[][] userAgentData() { userAgent("custom-agent/2.0", "api:table")}, {(Consumer) b -> b.setClientName(" my-app/1.0 "), userAgent("my-app/1.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", osMetadata), + "my-app/1.0 clickhouse-java-v2/" + Client.clientVersion + " (os:Android 14; api:table; lv:java/" + + System.getProperty("java.version") + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + + ")"}, }; } @@ -107,11 +116,20 @@ public void testClientMetadataKeptWhenApplicationPartChanges() throws Exception assertUserAgent(client, new QuerySettings().httpHeader(HttpHeaders.USER_AGENT, "op-agent/3.0"), userAgent("op-agent/3.0", "api:table")); assertUserAgent(client, (QuerySettings) new QuerySettings() - .setOption(ClientConfigProperties.CLIENT_METADATA.getKey(), "fv=flink/2.0.0"), + .setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "fv=flink/2.0.0"), userAgent("new-app/2.0", "api:table")); } } + @Test + public void testBlankOperationClientNameGetsClientValue() throws Exception { + try (Client client = newClientBuilder().setClientName("my-app/1.0").build()) { + assertUserAgent(client, (QuerySettings) new QuerySettings() + .setOption(ClientConfigProperties.CLIENT_NAME.getKey(), " "), + userAgent("my-app/1.0", null)); + } + } + @Test public void testInvalidClientMetadataRejectedOnBuild() { Client.Builder builder = newClientBuilder() @@ -143,6 +161,7 @@ private static String userAgent(String application, String metadata) { + "clickhouse-java-v2/" + Client.clientVersion + " (" + (metadata == null ? "" : metadata + "; ") + "lv:java/" + System.getProperty("java.version") - + "; os:" + System.getProperty("os.name") + ")"; + + "; os:" + System.getProperty("os.name") + + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index 7037d3708..c74b6a2c8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -553,7 +553,7 @@ public void testLongUserAgentLoggedAsWarning() { HttpAPIClientHelper helper = newHelper("my-app/1.0"); assertFalse(captureStdErr(() -> userAgentHeader(helper, overLimit)).contains(warning)); - assertTrue(captureStdErr(() -> helper.updateWorkloadIdentity(clientNameConfig(overLimit))).contains(warning)); + assertTrue(captureStdErr(() -> helper.updateClientName(clientNameConfig(overLimit))).contains(warning)); } private static HttpAPIClientHelper newHelper(String clientName) { @@ -574,7 +574,7 @@ private static String userAgentHeader(HttpAPIClientHelper helper, String clientN private static String clientPart() { return "clickhouse-java-v2/" + Client.clientVersion + " (lv:java/" + System.getProperty("java.version") - + "; os:" + System.getProperty("os.name") + ")"; + + "; os:" + System.getProperty("os.name") + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; } private static String headerValue(HttpPost req, String name) { diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index ceece8e3b..9cf03c134 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -2,8 +2,11 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; +import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.util.VersionInfo; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Collections; @@ -16,7 +19,7 @@ public class WorkloadIdentitySupportTest { public void testUserAgentComposedOncePerClient() { Map rawConfig = new HashMap<>(); rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); - rawConfig.put(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table"); + rawConfig.put(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table"); Map config = ClientConfigProperties.parseConfigMap(rawConfig); WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); @@ -46,19 +49,89 @@ public void testUserAgentComposedOnceForConfiguredHeader() { Assert.assertEquals(userAgent, userAgent("custom-agent/2.0 ", "")); Assert.assertSame(identity.getUserAgent("custom-agent/2.0", config), userAgent); Assert.assertEquals(identity.getUserAgent(null, config), userAgent("my-app/1.0 ", "")); + + Map updatedConfig = new HashMap<>(config); + updatedConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "new-app/2.0"); + identity.updateClientName(updatedConfig); + Assert.assertEquals(identity.getUserAgent("custom-agent/2.0", updatedConfig), userAgent); + } + + @Test + public void testUserAgentComposedOnceOnClientNameUpdate() { + Map config = ClientConfigProperties.parseConfigMap( + Collections.singletonMap(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0")); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + + Map updatedConfig = new HashMap<>(config); + updatedConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "new-app/2.0"); + identity.updateClientName(updatedConfig); + + String userAgent = identity.getUserAgent(null, updatedConfig); + Assert.assertEquals(userAgent, userAgent("new-app/2.0 ", "")); + Assert.assertSame(identity.getUserAgent(null, updatedConfig), userAgent); + Assert.assertEquals(identity.getUserAgent(null, config), userAgent("my-app/1.0 ", "")); + } + + @Test(dataProvider = "blankValues") + public void testBlankApplicationPartGetsKeptValue(String blank) { + Map config = ClientConfigProperties.parseConfigMap( + Collections.singletonMap(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0")); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + String userAgent = identity.getUserAgent(null, config); + + Map requestConfig = new HashMap<>(config); + requestConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), blank); + Assert.assertSame(identity.getUserAgent(null, requestConfig), userAgent); + Assert.assertSame(identity.getUserAgent(blank, requestConfig), userAgent); + } + + @DataProvider(name = "blankValues") + public static Object[][] blankValues() { + return new Object[][]{{""}, {" "}}; + } + + @Test(dataProvider = "clientEntriesMetadata") + public void testClientMetadataReplacesClientEntries(String metadata, String expectedComment) { + Map config = ClientConfigProperties.parseConfigMap( + Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), metadata)); + Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), + "clickhouse-java-v2/" + Client.clientVersion + " (" + expectedComment + ")"); + } + + @DataProvider(name = "clientEntriesMetadata") + public static Object[][] clientEntriesMetadata() { + String languageVersion = "lv:java/" + System.getProperty("java.version"); + String os = "os:" + System.getProperty("os.name"); + String httpClient = "ahc:" + DependencyVersions.APACHE_HTTP_CLIENT; + return new Object[][]{ + {"os=Android 14", "os:Android 14; " + languageVersion + "; " + httpClient}, + {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + os + "; " + httpClient}, + {"ahc=okhttp/4.12", "ahc:okhttp/4.12; " + languageVersion + "; " + os}, + {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + httpClient}, + {"os=Android 14,ahc=okhttp/4.12,api=table,lv=kotlin/2.0", + "os:Android 14; ahc:okhttp/4.12; api:table; lv:kotlin/2.0"}, + }; + } + + @Test + public void testApacheHttpClientVersionGenerated() { + VersionInfo versionInfo = VersionInfo.loadVersionInfo("org.apache.hc.client5", + HttpClients.class.getClassLoader()); + Assert.assertNotNull(versionInfo); + Assert.assertEquals(DependencyVersions.APACHE_HTTP_CLIENT, versionInfo.getRelease()); } @Test public void testUserAgentWithoutClientMetadata() { Map withMetadata = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_METADATA.getKey(), "api=table")); + Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table")); WorkloadIdentitySupport identityWithMetadata = new WorkloadIdentitySupport(withMetadata); Map config = Collections.emptyMap(); Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), userAgent("", "")); Map emptyMetadata = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_METADATA.getKey(), "")); + Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "")); Assert.assertEquals(new WorkloadIdentitySupport(emptyMetadata).getUserAgent(null, emptyMetadata), userAgent("", "")); @@ -68,6 +141,7 @@ public void testUserAgentWithoutClientMetadata() { private static String userAgent(String application, String metadata) { return application + "clickhouse-java-v2/" + Client.clientVersion + " (" + metadata - + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + ")"; + + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; } } diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index c5ba9390c..b9fb0a81f 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -264,7 +264,7 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | | `setClientName(String clientName)` | `clientName` - application name string | Sets the client name, which identifies the application. Will be passed as the first part of the `User-Agent` header. Use it instead of setting the `User-Agent` header. Does not change the client metadata. | - | `client_name` | -| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, before the `lv` and `os` entries. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_metadata` | +| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os` and `ahc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -278,8 +278,8 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. There are two fields in a query log that identify application originated a request: `client_name` and `http_user_agent`. Native TCP protocol uses `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. -The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:)`. -What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group before `lv` and `os`. +The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; ahc:)`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os` and `ahc` if the pairs do not set these keys. For example: ```java showLineNumbers @@ -294,7 +294,7 @@ Client client = new Client.Builder() ``` will result in the following `http_user_agent` value: ``` -my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux) +my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; ahc:5.6.4) ``` Use `setClientName` to identify the application. Do not set the `User-Agent` header for this: a custom `User-Agent` header replaces diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 638f1757b..612d192e7 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -146,11 +146,11 @@ conn.setClientInfo(com.clickhouse.jdbc.ClientInfoProperties.APPLICATION_NAME.get Both ways will result in the following `http_user_agent` value in the query log: ``` -my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux) +my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; ahc:5.6.4) ``` -To add key/value pairs to the comment group of the header, set the `client_metadata` property. For example, -`client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux)`. +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property. For example, +`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; ahc:5.6.4)`. **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. Use the client name to identify the application. Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/docs/features.md b/docs/features.md index f037343c3..d35acbcbf 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The header does not name the HTTP transport library. `Client.Builder.setClientName(String, Map)` (or the `client_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, before `lv` and `os`; keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; ahc:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `ahc` entry is the version of the Apache HttpClient library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `ahc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: @@ -98,7 +98,7 @@ Compatibility-sensitive traits: - Nested columns: Un-flattened `Nested(f1 T1, ..., fN TN)` columns (tables created with `flatten_nested = 0`) are exposed as JDBC `ARRAY` whose element type is `Tuple(f1 T1, ..., fN TN)`. They can be inserted through `Connection#createArrayOf`/`setArray` or `setObject` (a Java array of tuples) and read back through `getArray`/`getObject`; `java.sql.Array#getResultSet()` iterates the nested rows as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple. An `Array(Nested(...))` column is read as a JDBC `ARRAY` whose elements are the nested values, each an array of tuples (`Object[]` of `Object[]`); its `getResultSet()` returns each element as a `java.sql.Array` equal to a top-level `Nested(...)` value. - Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape. For ClickHouse `26.8+` the variant also carries `MultiPoint`. - MultiPoint type mapping: For ClickHouse `26.8+`, JDBC exposes `MultiPoint` as `ARRAY` with type name `MultiPoint`, returns `double[][]` from `getObject()` and a `java.sql.Array` from `getArray()`, and accepts a nested `Array` (`createArrayOf("Array(Point)", ...)`) for prepared-statement inserts, the same as `Ring` and `LineString`. -- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. The `client_metadata` connection property is forwarded to the client and adds key/value pairs to the `User-Agent` header (see the `client-v2` `User-Agent` entry). +- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. The `client_id_metadata` connection property is forwarded to the client and adds key/value pairs to the `User-Agent` header (see the `client-v2` `User-Agent` entry). - Wrapper support: Implements standard JDBC `Wrapper` and `unwrap` behavior on major JDBC objects. - Packaging and runtime compatibility: Ships as a JDBC 4.2 driver, depends on `client-v2`, and includes native-image metadata for GraalVM users. diff --git a/docs/integration-client.md b/docs/integration-client.md index 3e14bd092..c70fcf8ea 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -121,13 +121,13 @@ public void updateClientWorkload(Client client, String workloadOrTenant) { #### How it is observed on the server (`User-Agent` header) -The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`) and operating system (`os`), in the standard ClickHouse client format: +The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), and the Apache HttpClient version (`ahc`), in the standard ClickHouse client format: ```text -order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux) +order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; ahc:5.6.4) ``` -To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, before `lv` and `os`: +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `ahc` if the map does not set these keys (set them to replace the values of the client): ```java Map metadata = new LinkedHashMap<>(); @@ -143,10 +143,10 @@ Client client = new Client.Builder() ``` ```text -Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux) +Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; ahc:5.6.4) ``` -The same pairs can be set with the `client_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, a value must not contain any of `;()`, and the keys `lv` and `os` are reserved. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. +The same pairs can be set with the `client_id_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting Java Client applications. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index ea671524e..b612087a0 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -152,13 +152,13 @@ public void executeWorkloadTask(Connection conn, String taskName) throws SQLExce #### How it is observed on the server (`User-Agent` header) -The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`) and operating system (`os`): +The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`), the operating system (`os`), and the Apache HttpClient version (`ahc`): ```text -order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux) +order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; ahc:5.6.4) ``` -To add key/value pairs to the comment group of the header, set the `client_metadata` property (for example, `client_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux)`). +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; ahc:5.6.4)`). Always identify the application with the client name (`client_name` or `ApplicationName`). Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/history/latest/3202.md b/history/latest/3202.md index a02faa4f6..41492d33e 100644 --- a/history/latest/3202.md +++ b/history/latest/3202.md @@ -1,8 +1,11 @@ - **[client-v2, jdbc-v2]** Breaking change: the `User-Agent` header now uses the standard ClickHouse client format, - ` clickhouse-java-v2/ (lv:java/; os:)`: `jvm:` is now - `lv:java/`, the OS is the `os:` entry, and the trailing `Apache-HttpClient/` is removed. Added - `Client.Builder#setClientName(String, Map)` (property `client_metadata`) to add key/value pairs to the comment group. - Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. The client - logs a warning if the `User-Agent` value of the client configuration is longer than 200 characters. + ` clickhouse-java-v2/ (lv:java/; os:; ahc:)`: + `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing + `Apache-HttpClient/` is replaced by the `ahc:` entry (the Apache HttpClient version that the + client is built with). Added `Client.Builder#setClientName(String, Map)` (property `client_id_metadata`) to add + key/value pairs to the comment group. The pairs can set `lv`, `os` and `ahc` to replace the values of the client. + Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. A blank value + set for one operation counts as not set. The client logs a warning if the `User-Agent` value of the client + configuration is longer than 200 characters. Queries that parse `http_user_agent` in `system.query_log` must use the new format. (https://github.com/ClickHouse/clickhouse-java/issues/3202) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 7c035796a..2457d8548 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -7,6 +7,7 @@ import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.ServerException; +import com.clickhouse.client.api.internal.DependencyVersions; import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.data.ClickHouseFormat; @@ -391,13 +392,14 @@ public void testClientMetadataInUserAgent(boolean setInUrl) throws Exception { info.setProperty(ClientConfigProperties.DATABASE.getKey(), ClickHouseServerForTest.getDatabase()); String url = getEndpointString(); if (setInUrl) { - url += "?" + ClientConfigProperties.CLIENT_METADATA.getKey() + "=" + metadata; + url += "?" + ClientConfigProperties.CLIENT_ID_METADATA.getKey() + "=" + metadata; } else { - info.setProperty(ClientConfigProperties.CLIENT_METADATA.getKey(), metadata); + info.setProperty(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), metadata); } String expectedSuffix = "clickhouse-java-v2/" + Client.clientVersion + " (fv:flink/2.0.0; api:table; lv:java/" - + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + ")"; + + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; ahc:" + + DependencyVersions.APACHE_HTTP_CLIENT + ")"; try (Connection localConnection = new ConnectionImpl(url, info); Statement stmt = localConnection.createStatement()) { String userAgent = executeQueryAndVerifyUserAgent(stmt, null); diff --git a/pom.xml b/pom.xml index b70722404..521a55d18 100644 --- a/pom.xml +++ b/pom.xml @@ -139,6 +139,7 @@ 3.3.1 1.6.13 3.5.3 + 3.0.0 3.2.0 2.18.0 3.3.1 @@ -473,6 +474,11 @@ build-helper-maven-plugin ${helper-plugin.version} + + org.codehaus.mojo + templating-maven-plugin + ${templating-plugin.version} + org.codehaus.mojo flatten-maven-plugin From a545bcfd95af492835c598b824a801cc18c06df8 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:31:02 +0000 Subject: [PATCH 8/9] [client-v2] Keep the workload identity in Client, use hc:ahc_, add Versions - Client creates the WorkloadIdentitySupport, keeps it in a final field and updates it in updateClientName. HttpAPIClientHelper gets it in its constructor and only sets the User-Agent header from it; HttpAPIClientHelper#updateClientName is removed. - The HTTP client entry is now hc:ahc_. - The generated class DependencyVersions is now Versions. It also has the client version (CLIENT), so client-v2-version.properties is not read any more and is removed, with the resource filtering block. Client.clientVersion comes from Versions.CLIENT and is not a compile-time constant. - addIfNotSet moved to ClientUtils#putIfAbsentIgnoreCase (null key is rejected). Co-Authored-By: Claude Opus 5.5 --- client-v2/pom.xml | 8 +--- .../api/internal/DependencyVersions.java | 16 ------- .../client/api/internal/Versions.java | 22 ++++++++++ .../com/clickhouse/client/api/Client.java | 20 +++++---- .../client/api/internal/ClientUtils.java | 22 ++++++++++ .../api/internal/HttpAPIClientHelper.java | 21 ++++----- .../api/internal/WorkloadIdentitySupport.java | 37 ++++++---------- .../resources/client-v2-version.properties | 2 - .../clickhouse/client/HttpTransportTests.java | 6 +-- .../InsertExceptionClassificationTest.java | 4 +- .../client/api/UserAgentUnitTest.java | 6 +-- .../client/api/internal/ClientUtilsTest.java | 35 +++++++++++++++ .../internal/HttpAPIClientHelperFactory.java | 6 ++- .../api/internal/HttpAPIClientHelperTest.java | 43 +++++++++++-------- .../client/api/internal/VersionsTest.java | 19 ++++++++ .../internal/WorkloadIdentitySupportTest.java | 20 +++------ docs/clickhouse-docs/client.mdx | 8 ++-- docs/clickhouse-docs/jdbc.mdx | 4 +- docs/features.md | 2 +- docs/integration-client.md | 8 ++-- docs/integration-jdbc.md | 6 +-- history/latest/3202.md | 6 +-- .../com/clickhouse/jdbc/ConnectionTest.java | 6 +-- 23 files changed, 196 insertions(+), 131 deletions(-) delete mode 100644 client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java create mode 100644 client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java delete mode 100644 client-v2/src/main/resources/client-v2-version.properties create mode 100644 client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java diff --git a/client-v2/pom.xml b/client-v2/pom.xml index fb1b0ab90..7b1db4ab3 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -210,12 +210,6 @@ - - - src/main/resources - true - - org.apache.maven.plugins @@ -232,7 +226,7 @@ - + org.codehaus.mojo templating-maven-plugin diff --git a/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java deleted file mode 100644 index fc07d746b..000000000 --- a/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/DependencyVersions.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.clickhouse.client.api.internal; - -/** - * Versions of the libraries that the client is built with. The build generates this class from a template, so the - * versions are correct also when the libraries are shaded into the client and cannot report their own version. - */ -public final class DependencyVersions { - - /** - * Version of the Apache HttpClient library. - */ - public static final String APACHE_HTTP_CLIENT = "${apache.httpclient.version}"; - - private DependencyVersions() { - } -} diff --git a/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java new file mode 100644 index 000000000..7d2afcbf7 --- /dev/null +++ b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java @@ -0,0 +1,22 @@ +package com.clickhouse.client.api.internal; + +/** + * Versions of the client and of the libraries that the client is built with. The build generates this class from a + * template, so the versions are correct also when the libraries are shaded into the client and cannot report their + * own version. + */ +public final class Versions { + + /** + * Version of the client. + */ + public static final String CLIENT = "${revision}"; + + /** + * Version of the Apache HttpClient library. + */ + public static final String APACHE_HTTP_CLIENT = "${apache.httpclient.version}"; + + private Versions() { + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 7cd92b185..acadea0ff 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -25,6 +25,8 @@ import com.clickhouse.client.api.internal.MapUtils; import com.clickhouse.client.api.internal.TableSchemaParser; import com.clickhouse.client.api.internal.ValidationUtils; +import com.clickhouse.client.api.internal.Versions; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.client.api.metadata.ColumnToMethodMatchingStrategy; import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy; import com.clickhouse.client.api.metadata.TableSchema; @@ -49,7 +51,6 @@ import com.clickhouse.client.api.transport.HttpEndpoint; import com.clickhouse.client.api.transport.internal.TransportRequest; import com.clickhouse.client.api.transport.internal.TransportResponse; -import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; @@ -129,6 +130,8 @@ public class Client implements AutoCloseable { private HttpAPIClientHelper httpClientHelper = null; + private final WorkloadIdentitySupport workloadIdentitySupport; + private final List endpoints; private final Map configuration; private final Session session; @@ -235,8 +238,9 @@ private Client(Collection endpoints, Map configuration, this.lz4Factory = LZ4Factory.fastestJavaInstance(); } + this.workloadIdentitySupport = new WorkloadIdentitySupport(this.configuration); this.httpClientHelper = new HttpAPIClientHelper(this.configuration, metricsRegistry, initSslContext, lz4Factory, - this.spanRecorder); + this.spanRecorder, this.workloadIdentitySupport); this.serverVersion = configuration.getOrDefault(ClientConfigProperties.SERVER_VERSION.getKey(), "unknown"); this.dbUser = configuration.getOrDefault(ClientConfigProperties.USER.getKey(), ClientConfigProperties.USER.getDefObjVal()); this.typeHintMapping = (Map>) this.configuration.get(ClientConfigProperties.TYPE_HINT_MAPPING.getKey()); @@ -1163,10 +1167,10 @@ public Builder setClientName(String clientName) { * Sets the client name and the client metadata, which identify the application to the server. See * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the * {@code User-Agent} header, in the iteration order of {@code metadata}, and then its own {@code lv}, - * {@code os} and {@code ahc} (Apache HttpClient version) entries if the metadata does not set these keys. For - * example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give + * {@code os} and {@code hc} (HTTP client) entries if the metadata does not set these keys. For example, the + * client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; - * os:; ahc:)}. + * os:; hc:ahc_)}. *

* Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not contain any of @@ -2607,7 +2611,7 @@ public void setDBRoles(Collection dbRoles) { */ public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); - this.httpClientHelper.updateClientName(this.configuration); + this.workloadIdentitySupport.updateClientName(this.configuration); } /** @@ -2618,8 +2622,8 @@ public void updateSessionId(String sessionId) { this.session.updateSessionId(sessionId); } - public static final String clientVersion = - ClickHouseClientOption.readVersionFromResource("client-v2-version.properties"); + // not a compile-time constant, so that code compiled against another version reads the version of this client + public static final String clientVersion = String.valueOf(Versions.CLIENT); public static final String CLIENT_USER_AGENT = "clickhouse-java-v2/"; private Collection unmodifiableDbRolesView = Collections.emptyList(); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java index 9635fc4d7..9814c27b1 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java @@ -3,6 +3,8 @@ import org.slf4j.Logger; import java.io.Closeable; +import java.util.Map; +import java.util.Objects; /** * Class containing utility methods used across the client. @@ -28,4 +30,24 @@ public static void quietClose(Closeable closeable, Logger log) { } } } + + /** + * Puts the entry into the map if the map does not have the key. Keys are compared ignoring case, so the map is not + * changed if it has the key in a different case. A key that the map has is not replaced, also if its value is + * {@code null}. + * + * @param map - map to update + * @param key - key of the entry, not {@code null} + * @param value - value of the entry + * @param - type of the values + */ + public static void putIfAbsentIgnoreCase(Map map, String key, V value) { + Objects.requireNonNull(key, "key"); + for (String mapKey : map.keySet()) { + if (key.equalsIgnoreCase(mapKey)) { + return; + } + } + map.put(key, value); + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 3a3b57c9f..d2c188c14 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -133,6 +133,10 @@ public class HttpAPIClientHelper { private final Set defaultRetryCauses; + /** + * Identification of the client, which gives the value of the {@code User-Agent} header. {@code Client} creates + * and updates it. Never {@code null}. + */ private final WorkloadIdentitySupport workloadIdentitySupport; private final Object metricsRegistry; @@ -150,9 +154,12 @@ public class HttpAPIClientHelper { private final SpanRecorder spanRecorder; public HttpAPIClientHelper(Map configuration, Object metricsRegistry, boolean initSslContext, - LZ4Factory lz4Factory, SpanRecorder spanRecorder) { + LZ4Factory lz4Factory, SpanRecorder spanRecorder, + WorkloadIdentitySupport workloadIdentitySupport) { this.spanRecorder = Objects.requireNonNull(spanRecorder, "spanRecorder is required; use DefaultSpanRecorder.NOOP to record nothing"); + this.workloadIdentitySupport = Objects.requireNonNull(workloadIdentitySupport, + "workloadIdentitySupport is required"); this.metricsRegistry = metricsRegistry; this.httpClient = createHttpClient(initSslContext, configuration); this.lz4Factory = lz4Factory; @@ -168,18 +175,6 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi if (defaultRetryCauses.contains(ClientFaultCause.None)) { defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - - this.workloadIdentitySupport = new WorkloadIdentitySupport(configuration); - } - - /** - * Composes the value of the {@code User-Agent} header again for the client configuration, and keeps it. Call it - * after the client name changes. - * - * @param configuration - client configuration - */ - public void updateClientName(Map configuration) { - workloadIdentitySupport.updateClientName(configuration); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index 4cc640651..29106c527 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -15,16 +15,17 @@ * Builds the identification of the client and of the application that uses it. The identification is the value of * the {@code User-Agent} header, in the standard ClickHouse client format: * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:; - * ahc:)}. + * hc:ahc_)}. *

* The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), or the custom * {@code User-Agent} header if the application set one. The key/value pairs are the client metadata - * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). A transport sends the value of - * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport. + * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). The {@code hc} entry is the HTTP client that sends the requests: + * {@code ahc_} (Apache HttpClient) and its version. *

- * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. The client creates - * one instance. The instance composes the value for the client configuration when it is created and when the client - * name changes, and keeps it. + * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. {@link Client} + * creates one instance and updates it when the client name changes. The instance composes the value for the client + * configuration at these times, and keeps it. A transport only sends the value of {@link #getUserAgent(String, Map)} + * with every request. */ public final class WorkloadIdentitySupport { @@ -34,7 +35,7 @@ public final class WorkloadIdentitySupport { private static final String METADATA_KEY_OS = "os"; - private static final String METADATA_KEY_HTTP_CLIENT = "ahc"; + private static final String METADATA_KEY_HTTP_CLIENT = "hc"; /** * First and last characters of the printable ASCII range, which client metadata keys and values must be in. @@ -147,29 +148,19 @@ private static boolean isClientMetadataText(String str, String forbiddenChars) { /** * Composes the client part: {@code clickhouse-java-v2/ (:; ...)}. The key/value pairs are the * client metadata, if it is set, then {@code lv:java/}, {@code os:} and - * {@code ahc:} if the client metadata does not set these keys (ignoring case). + * {@code hc:ahc_} if the client metadata does not set these keys (ignoring case). The + * client sends requests over HTTP with Apache HttpClient ({@code ahc}). */ private static String composeClientPart(Map configuration) { Map metadata = new LinkedHashMap<>(); ClientConfigProperties.CLIENT_ID_METADATA.>applyIfSet(configuration, metadata::putAll); - addIfNotSet(metadata, METADATA_KEY_LANGUAGE_VERSION, "java/" + System.getProperty("java.version")); - addIfNotSet(metadata, METADATA_KEY_OS, System.getProperty("os.name")); - addIfNotSet(metadata, METADATA_KEY_HTTP_CLIENT, DependencyVersions.APACHE_HTTP_CLIENT); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_LANGUAGE_VERSION, + "java/" + System.getProperty("java.version")); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_OS, System.getProperty("os.name")); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_HTTP_CLIENT, "ahc_" + Versions.APACHE_HTTP_CLIENT); StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); metadata.forEach((key, value) -> comment.add(key + ":" + value)); return comment.toString(); } - - /** - * Adds the entry if the client metadata does not have the key. Keys are compared ignoring case. - */ - private static void addIfNotSet(Map metadata, String key, String value) { - for (String metadataKey : metadata.keySet()) { - if (metadataKey.equalsIgnoreCase(key)) { - return; - } - } - metadata.put(key, value); - } } diff --git a/client-v2/src/main/resources/client-v2-version.properties b/client-v2/src/main/resources/client-v2-version.properties deleted file mode 100644 index a54c8f97d..000000000 --- a/client-v2/src/main/resources/client-v2-version.properties +++ /dev/null @@ -1,2 +0,0 @@ -version=${revision} -build.date=${build_timestamp} \ No newline at end of file diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index 8b3fa31a4..608a9e5a9 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -1201,9 +1201,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider") public static Object[][] testUserAgentHasCompleteProductName_dataProvider() { return new Object[][] { - { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)") }, - { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)")}, - { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; ahc:[\\d.]+\\)")}}; + { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)") }, + { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}, + { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}}; } @Test(dataProvider = "testClientNameDataProvider") diff --git a/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java b/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java index f9fa0a383..091cff3e1 100644 --- a/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.DataTransferException; import com.clickhouse.client.api.internal.HttpAPIClientHelper; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.observability.DefaultSpanRecorder; import com.clickhouse.client.api.serde.DataSerializationException; @@ -101,7 +102,8 @@ private static final class CallbackHttpClientHelper extends HttpAPIClientHelper private final OutputStream outputStream; private IOCallback writeCallback; private CallbackHttpClientHelper(OutputStream outputStream) { - super(Collections.emptyMap(), null, false, LZ4Factory.fastestJavaInstance(), DefaultSpanRecorder.NOOP); + super(Collections.emptyMap(), null, false, LZ4Factory.fastestJavaInstance(), DefaultSpanRecorder.NOOP, + new WorkloadIdentitySupport(Collections.emptyMap())); this.outputStream = outputStream; } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index bd0fb9e28..c1ff1110d 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -1,6 +1,6 @@ package com.clickhouse.client.api; -import com.clickhouse.client.api.internal.DependencyVersions; +import com.clickhouse.client.api.internal.Versions; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.client.api.query.QuerySettings; import com.github.tomakehurst.wiremock.WireMockServer; @@ -101,7 +101,7 @@ public static Object[][] userAgentData() { userAgent("my-app/1.0", null)}, {(Consumer) b -> b.setClientName("my-app/1.0", osMetadata), "my-app/1.0 clickhouse-java-v2/" + Client.clientVersion + " (os:Android 14; api:table; lv:java/" - + System.getProperty("java.version") + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + + System.getProperty("java.version") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"}, }; } @@ -162,6 +162,6 @@ private static String userAgent(String application, String metadata) { + " (" + (metadata == null ? "" : metadata + "; ") + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") - + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; + + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java index 88c5d8526..7be73b265 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java @@ -3,10 +3,13 @@ import org.mockito.Mockito; import org.slf4j.Logger; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.Closeable; import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; public class ClientUtilsTest { @@ -45,4 +48,36 @@ public void testQuietCloseWithNull() { Mockito.verifyNoInteractions(log); Assert.assertTrue(true); } + + @Test(groups = {"unit"}, dataProvider = "putIfAbsentIgnoreCase") + public void testPutIfAbsentIgnoreCase(String[] existingKeys, String key, String expectedMap) { + Map map = new LinkedHashMap<>(); + for (String existingKey : existingKeys) { + map.put(existingKey, "set"); + } + + ClientUtils.putIfAbsentIgnoreCase(map, key, "added"); + + Assert.assertEquals(map.toString(), expectedMap); + } + + @DataProvider(name = "putIfAbsentIgnoreCase") + public static Object[][] putIfAbsentIgnoreCaseData() { + return new Object[][]{ + {new String[]{}, "os", "{os=added}"}, + {new String[]{"os"}, "lv", "{os=set, lv=added}"}, + {new String[]{"os"}, "os", "{os=set}"}, + {new String[]{"os"}, "OS", "{os=set}"}, + {new String[]{"OS"}, "os", "{OS=set}"}, + {new String[]{"api", "OS"}, "os", "{api=set, OS=set}"}, + }; + } + + @Test(groups = {"unit"}) + public void testPutIfAbsentIgnoreCaseNullKey() { + Map map = new LinkedHashMap<>(); + + Assert.expectThrows(NullPointerException.class, () -> ClientUtils.putIfAbsentIgnoreCase(map, null, "added")); + Assert.assertTrue(map.isEmpty()); + } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java index d24fdd08c..1352379f8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java @@ -17,13 +17,15 @@ private HttpAPIClientHelperFactory() { /** * Creates a helper with no metrics registry, no SSL context and observability disabled - the - * configuration every test that does not exercise those features needs. + * configuration every test that does not exercise those features needs. The helper gets a new + * {@link WorkloadIdentitySupport} for {@code configuration}, as {@code Client} gives it one. * * @param configuration - client configuration * @param lz4Factory - LZ4 factory the helper should use * @return a new helper instance */ public static HttpAPIClientHelper newHelper(Map configuration, LZ4Factory lz4Factory) { - return new HttpAPIClientHelper(configuration, null, false, lz4Factory, DefaultSpanRecorder.NOOP); + return new HttpAPIClientHelper(configuration, null, false, lz4Factory, DefaultSpanRecorder.NOOP, + new WorkloadIdentitySupport(configuration)); } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index c74b6a2c8..33ad619fa 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -520,16 +520,9 @@ private static Map compressionConfig(boolean clientCompression, */ @Test public void testUserAgentKeptAfterClientNameUpdate() throws Exception { - try (Client client = new Client.Builder() - .addEndpoint("http://localhost:8123") - .setUsername("default") - .setPassword("") - .setClientName("my-app/1.0") - .build()) { + try (Client client = newClient("my-app/1.0")) { client.updateClientName("new-app/2.0"); - Field helperField = Client.class.getDeclaredField("httpClientHelper"); - helperField.setAccessible(true); - HttpAPIClientHelper helper = (HttpAPIClientHelper) helperField.get(client); + HttpAPIClientHelper helper = helperOf(client); String userAgent = userAgentHeader(helper, "new-app/2.0"); assertEquals(userAgent, "new-app/2.0 " + clientPart()); @@ -543,21 +536,35 @@ public void testUserAgentKeptAfterClientNameUpdate() throws Exception { * logged, so that the log does not repeat for every request. */ @Test - public void testLongUserAgentLoggedAsWarning() { + public void testLongUserAgentLoggedAsWarning() throws Exception { String atLimit = String.join("", Collections.nCopies(200 - 1 - clientPart().length(), "x")); String overLimit = atLimit + "x"; String warning = "The User-Agent value is "; - assertFalse(captureStdErr(() -> newHelper(atLimit)).contains(warning)); - assertTrue(captureStdErr(() -> newHelper(overLimit)).contains(warning + "201 characters long, more than 200")); + assertFalse(captureStdErr(() -> newClient(atLimit).close()).contains(warning)); + assertTrue(captureStdErr(() -> newClient(overLimit).close()) + .contains(warning + "201 characters long, more than 200")); - HttpAPIClientHelper helper = newHelper("my-app/1.0"); - assertFalse(captureStdErr(() -> userAgentHeader(helper, overLimit)).contains(warning)); - assertTrue(captureStdErr(() -> helper.updateClientName(clientNameConfig(overLimit))).contains(warning)); + try (Client client = newClient("my-app/1.0")) { + HttpAPIClientHelper helper = helperOf(client); + assertFalse(captureStdErr(() -> userAgentHeader(helper, overLimit)).contains(warning)); + assertTrue(captureStdErr(() -> client.updateClientName(overLimit)).contains(warning)); + } + } + + private static Client newClient(String clientName) { + return new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("") + .setClientName(clientName) + .build(); } - private static HttpAPIClientHelper newHelper(String clientName) { - return HttpAPIClientHelperFactory.newHelper(clientNameConfig(clientName), LZ4Factory.fastestInstance()); + private static HttpAPIClientHelper helperOf(Client client) throws Exception { + Field helperField = Client.class.getDeclaredField("httpClientHelper"); + helperField.setAccessible(true); + return (HttpAPIClientHelper) helperField.get(client); } private static Map clientNameConfig(String clientName) { @@ -574,7 +581,7 @@ private static String userAgentHeader(HttpAPIClientHelper helper, String clientN private static String clientPart() { return "clickhouse-java-v2/" + Client.clientVersion + " (lv:java/" + System.getProperty("java.version") - + "; os:" + System.getProperty("os.name") + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; + + "; os:" + System.getProperty("os.name") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; } private static String headerValue(HttpPost req, String name) { diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java new file mode 100644 index 000000000..0a005061f --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java @@ -0,0 +1,19 @@ +package com.clickhouse.client.api.internal; + +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.util.VersionInfo; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class VersionsTest { + + @Test + public void testVersionsGenerated() { + Assert.assertTrue(Versions.CLIENT.matches("\\d+\\.\\d+\\.\\d+.*"), Versions.CLIENT); + + VersionInfo versionInfo = VersionInfo.loadVersionInfo("org.apache.hc.client5", + HttpClients.class.getClassLoader()); + Assert.assertNotNull(versionInfo); + Assert.assertEquals(Versions.APACHE_HTTP_CLIENT, versionInfo.getRelease()); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index 9cf03c134..3b7e7bee6 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -2,9 +2,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; -import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.util.VersionInfo; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -102,25 +100,17 @@ public void testClientMetadataReplacesClientEntries(String metadata, String expe public static Object[][] clientEntriesMetadata() { String languageVersion = "lv:java/" + System.getProperty("java.version"); String os = "os:" + System.getProperty("os.name"); - String httpClient = "ahc:" + DependencyVersions.APACHE_HTTP_CLIENT; + String httpClient = "hc:ahc_" + Versions.APACHE_HTTP_CLIENT; return new Object[][]{ {"os=Android 14", "os:Android 14; " + languageVersion + "; " + httpClient}, {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + os + "; " + httpClient}, - {"ahc=okhttp/4.12", "ahc:okhttp/4.12; " + languageVersion + "; " + os}, + {"hc=okhttp_4.12", "hc:okhttp_4.12; " + languageVersion + "; " + os}, {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + httpClient}, - {"os=Android 14,ahc=okhttp/4.12,api=table,lv=kotlin/2.0", - "os:Android 14; ahc:okhttp/4.12; api:table; lv:kotlin/2.0"}, + {"os=Android 14,hc=okhttp_4.12,api=table,lv=kotlin/2.0", + "os:Android 14; hc:okhttp_4.12; api:table; lv:kotlin/2.0"}, }; } - @Test - public void testApacheHttpClientVersionGenerated() { - VersionInfo versionInfo = VersionInfo.loadVersionInfo("org.apache.hc.client5", - HttpClients.class.getClassLoader()); - Assert.assertNotNull(versionInfo); - Assert.assertEquals(DependencyVersions.APACHE_HTTP_CLIENT, versionInfo.getRelease()); - } - @Test public void testUserAgentWithoutClientMetadata() { Map withMetadata = ClientConfigProperties.parseConfigMap( @@ -142,6 +132,6 @@ public void testUserAgentWithoutClientMetadata() { private static String userAgent(String application, String metadata) { return application + "clickhouse-java-v2/" + Client.clientVersion + " (" + metadata + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") - + "; ahc:" + DependencyVersions.APACHE_HTTP_CLIENT + ")"; + + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; } } diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index b9fb0a81f..35c7dfaad 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -264,7 +264,7 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | | `setClientName(String clientName)` | `clientName` - application name string | Sets the client name, which identifies the application. Will be passed as the first part of the `User-Agent` header. Use it instead of setting the `User-Agent` header. Does not change the client metadata. | - | `client_name` | -| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os` and `ahc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | +| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os` and `hc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -278,8 +278,8 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. There are two fields in a query log that identify application originated a request: `client_name` and `http_user_agent`. Native TCP protocol uses `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. -The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; ahc:)`. -What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os` and `ahc` if the pairs do not set these keys. +The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os` and `hc` if the pairs do not set these keys. For example: ```java showLineNumbers @@ -294,7 +294,7 @@ Client client = new Client.Builder() ``` will result in the following `http_user_agent` value: ``` -my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; ahc:5.6.4) +my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) ``` Use `setClientName` to identify the application. Do not set the `User-Agent` header for this: a custom `User-Agent` header replaces diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 612d192e7..d24a75e97 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -146,11 +146,11 @@ conn.setClientInfo(com.clickhouse.jdbc.ClientInfoProperties.APPLICATION_NAME.get Both ways will result in the following `http_user_agent` value in the query log: ``` -my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; ahc:5.6.4) +my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) ``` To add key/value pairs to the comment group of the header, set the `client_id_metadata` property. For example, -`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; ahc:5.6.4)`. +`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4)`. **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. Use the client name to identify the application. Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/docs/features.md b/docs/features.md index d35acbcbf..8832ce820 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; ahc:)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `ahc` entry is the version of the Apache HttpClient library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `ahc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/docs/integration-client.md b/docs/integration-client.md index c70fcf8ea..5a6cf51ad 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -121,13 +121,13 @@ public void updateClientWorkload(Client client, String workloadOrTenant) { #### How it is observed on the server (`User-Agent` header) -The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), and the Apache HttpClient version (`ahc`), in the standard ClickHouse client format: +The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version), in the standard ClickHouse client format: ```text -order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; ahc:5.6.4) +order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) ``` -To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `ahc` if the map does not set these keys (set them to replace the values of the client): +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `hc` if the map does not set these keys (set them to replace the values of the client): ```java Map metadata = new LinkedHashMap<>(); @@ -143,7 +143,7 @@ Client client = new Client.Builder() ``` ```text -Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; ahc:5.6.4) +Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) ``` The same pairs can be set with the `client_id_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index b612087a0..8188f8883 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -152,13 +152,13 @@ public void executeWorkloadTask(Connection conn, String taskName) throws SQLExce #### How it is observed on the server (`User-Agent` header) -The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`), the operating system (`os`), and the Apache HttpClient version (`ahc`): +The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version): ```text -order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; ahc:5.6.4) +order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) ``` -To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; ahc:5.6.4)`). +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4)`). Always identify the application with the client name (`client_name` or `ApplicationName`). Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/history/latest/3202.md b/history/latest/3202.md index 41492d33e..a2e6b65ff 100644 --- a/history/latest/3202.md +++ b/history/latest/3202.md @@ -1,9 +1,9 @@ - **[client-v2, jdbc-v2]** Breaking change: the `User-Agent` header now uses the standard ClickHouse client format, - ` clickhouse-java-v2/ (lv:java/; os:; ahc:)`: + ` clickhouse-java-v2/ (lv:java/; os:; hc:ahc_)`: `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing - `Apache-HttpClient/` is replaced by the `ahc:` entry (the Apache HttpClient version that the + `Apache-HttpClient/` is replaced by the `hc:ahc_` entry (the Apache HttpClient version that the client is built with). Added `Client.Builder#setClientName(String, Map)` (property `client_id_metadata`) to add - key/value pairs to the comment group. The pairs can set `lv`, `os` and `ahc` to replace the values of the client. + key/value pairs to the comment group. The pairs can set `lv`, `os` and `hc` to replace the values of the client. Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. A blank value set for one operation counts as not set. The client logs a warning if the `User-Agent` value of the client configuration is longer than 200 characters. diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 2457d8548..93930459c 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -7,8 +7,8 @@ import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.ServerException; -import com.clickhouse.client.api.internal.DependencyVersions; import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.internal.Versions; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.data.ClickHouseFormat; import com.fasterxml.jackson.databind.JsonNode; @@ -398,8 +398,8 @@ public void testClientMetadataInUserAgent(boolean setInUrl) throws Exception { } String expectedSuffix = "clickhouse-java-v2/" + Client.clientVersion + " (fv:flink/2.0.0; api:table; lv:java/" - + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; ahc:" - + DependencyVersions.APACHE_HTTP_CLIENT + ")"; + + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; hc:ahc_" + + Versions.APACHE_HTTP_CLIENT + ")"; try (Connection localConnection = new ConnectionImpl(url, info); Statement stmt = localConnection.createStatement()) { String userAgent = executeQueryAndVerifyUserAgent(stmt, null); From 3d8771aa017edc5eb0b1596a558a44ec6ea34251 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:04:54 +0000 Subject: [PATCH 9/9] [client-v2] Use data providers in WorkloadIdentitySupportTest, wrap doc examples as methods - WorkloadIdentitySupportTest: one data-provider test for the value of a request and one for client name updates, plus a test that two instances do not share the client part. - UserAgentUnitTest: entries with a null key or a null value in the setClientName(String, Map) metadata are ignored. - Client.Builder#setClientName(String, Map) Javadoc and features.md: a null key or value is ignored. - docs: wrap the client metadata examples as methods. Co-Authored-By: Claude Opus 5.5 --- .../com/clickhouse/client/api/Client.java | 9 +- .../client/api/UserAgentUnitTest.java | 11 +- .../internal/WorkloadIdentitySupportTest.java | 178 ++++++++++-------- docs/clickhouse-docs/client.mdx | 21 ++- docs/features.md | 2 +- docs/integration-client.md | 25 ++- 6 files changed, 139 insertions(+), 107 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index acadea0ff..4ecde77c6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1172,10 +1172,11 @@ public Builder setClientName(String clientName) { * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; * os:; hc:ahc_)}. *

- * Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be - * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not contain any of - * {@code ;()}; otherwise {@link #build()} throws {@link ClientMisconfigurationException}. The metadata is a - * client-level setting: it is kept when the client name changes with {@link Client#updateClientName(String)}. + * Keys and values are trimmed, and entries with a {@code null} or empty key or value are ignored. Keys and + * values must be printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not + * contain any of {@code ;()}; otherwise {@link #build()} throws {@link ClientMisconfigurationException}. The + * metadata is a client-level setting: it is kept when the client name changes with + * {@link Client#updateClientName(String)}. * * @param clientName - client application name, for example {@code my-app/1.0} * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index c1ff1110d..723d825ae 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -59,9 +59,12 @@ public static Object[][] userAgentData() { osMetadata.put("os", "Android 14"); osMetadata.put("api", "table"); - Map withEmptyValue = new LinkedHashMap<>(); - withEmptyValue.put("api", ""); - withEmptyValue.put("fv", "flink/2.0.0"); + Map withIgnoredEntries = new LinkedHashMap<>(); + withIgnoredEntries.put("api", ""); + withIgnoredEntries.put(" ", "x"); + withIgnoredEntries.put("os", null); + withIgnoredEntries.put(null, "x"); + withIgnoredEntries.put("fv", "flink/2.0.0"); return new Object[][]{ {(Consumer) b -> { }, userAgent(null, null)}, @@ -73,7 +76,7 @@ public static Object[][] userAgentData() { {(Consumer) b -> b.setClientName("my-app/1.0", Collections.singletonMap("build", "a=b,c\\d")), userAgent("my-app/1.0", "build:a=b,c\\d")}, - {(Consumer) b -> b.setClientName("my-app/1.0", withEmptyValue), + {(Consumer) b -> b.setClientName("my-app/1.0", withIgnoredEntries), userAgent("my-app/1.0", "fv:flink/2.0.0")}, {(Consumer) b -> b.setClientName("", flinkMetadata), userAgent(null, "fv:flink/2.0.0; api:table")}, diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index 3b7e7bee6..b3861e960 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -7,131 +7,145 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.util.Collections; import java.util.HashMap; import java.util.Map; public class WorkloadIdentitySupportTest { - @Test - public void testUserAgentComposedOncePerClient() { - Map rawConfig = new HashMap<>(); - rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); - rawConfig.put(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table"); - Map config = ClientConfigProperties.parseConfigMap(rawConfig); - WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + private static final String NAME = ClientConfigProperties.CLIENT_NAME.getKey(); - String userAgent = identity.getUserAgent(null, config); - Assert.assertEquals(userAgent, userAgent("my-app/1.0 ", "api:table; ")); - Assert.assertSame(identity.getUserAgent(null, new HashMap<>(config)), userAgent); - Assert.assertSame(identity.getUserAgent(" ", config), userAgent); + private static final String METADATA = ClientConfigProperties.CLIENT_ID_METADATA.getKey(); - Map requestConfig = new HashMap<>(config); - requestConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "op-app/3.0"); - Assert.assertEquals(identity.getUserAgent(null, requestConfig), userAgent("op-app/3.0 ", "api:table; ")); - Assert.assertEquals(identity.getUserAgent(" custom-agent/2.0 ", config), - userAgent("custom-agent/2.0 ", "api:table; ")); + private static final String USER_AGENT = ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT); - Assert.assertSame(identity.getUserAgent(null, config), userAgent); - } + private static final String LANGUAGE_VERSION = "lv:java/" + System.getProperty("java.version"); - @Test - public void testUserAgentComposedOnceForConfiguredHeader() { - Map rawConfig = new HashMap<>(); - rawConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0"); - rawConfig.put(ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT), "custom-agent/2.0"); - Map config = ClientConfigProperties.parseConfigMap(rawConfig); - WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + private static final String OS = "os:" + System.getProperty("os.name"); - String userAgent = identity.getUserAgent("custom-agent/2.0", config); - Assert.assertEquals(userAgent, userAgent("custom-agent/2.0 ", "")); - Assert.assertSame(identity.getUserAgent("custom-agent/2.0", config), userAgent); - Assert.assertEquals(identity.getUserAgent(null, config), userAgent("my-app/1.0 ", "")); + private static final String HTTP_CLIENT = "hc:ahc_" + Versions.APACHE_HTTP_CLIENT; - Map updatedConfig = new HashMap<>(config); - updatedConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "new-app/2.0"); - identity.updateClientName(updatedConfig); - Assert.assertEquals(identity.getUserAgent("custom-agent/2.0", updatedConfig), userAgent); - } - - @Test - public void testUserAgentComposedOnceOnClientNameUpdate() { - Map config = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0")); + @Test(dataProvider = "requestUserAgents") + public void testGetUserAgent(Map clientSettings, String customUserAgent, String requestClientName, + String expected) { + Map config = ClientConfigProperties.parseConfigMap(clientSettings); WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + String clientUserAgent = clientUserAgent(identity, config); + Map requestConfig = new HashMap<>(config); + if (requestClientName != null) { + requestConfig.put(NAME, requestClientName); + } - Map updatedConfig = new HashMap<>(config); - updatedConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), "new-app/2.0"); - identity.updateClientName(updatedConfig); + String userAgent = identity.getUserAgent(customUserAgent, requestConfig); + + Assert.assertEquals(userAgent, expected); + if (expected.equals(clientUserAgent)) { + Assert.assertSame(userAgent, clientUserAgent); + } + Assert.assertSame(clientUserAgent(identity, config), clientUserAgent); + } - String userAgent = identity.getUserAgent(null, updatedConfig); - Assert.assertEquals(userAgent, userAgent("new-app/2.0 ", "")); - Assert.assertSame(identity.getUserAgent(null, updatedConfig), userAgent); - Assert.assertEquals(identity.getUserAgent(null, config), userAgent("my-app/1.0 ", "")); + @DataProvider(name = "requestUserAgents") + public static Object[][] requestUserAgents() { + return new Object[][]{ + {settings(), null, null, userAgent(null)}, + {settings(METADATA, ""), null, null, userAgent(null)}, + {settings(NAME, "my-app/1.0"), null, null, userAgent("my-app/1.0")}, + {settings(NAME, " my-app/1.0 "), null, null, userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), null, null, + userAgent("my-app/1.0", "api:table")}, + {settings(NAME, "my-app/1.0"), " ", null, userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0"), null, "", userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0"), "", " ", userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), null, "op-app/3.0", + userAgent("op-app/3.0", "api:table")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), " custom-agent/2.0 ", null, + userAgent("custom-agent/2.0", "api:table")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), "custom-agent/2.0", null, + userAgent("custom-agent/2.0")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), null, null, + userAgent("my-app/1.0")}, + }; } - @Test(dataProvider = "blankValues") - public void testBlankApplicationPartGetsKeptValue(String blank) { - Map config = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_NAME.getKey(), "my-app/1.0")); + @Test(dataProvider = "clientNameUpdates") + public void testUpdateClientName(Map clientSettings, String clientName, String expected) { + Map config = ClientConfigProperties.parseConfigMap(clientSettings); WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); - String userAgent = identity.getUserAgent(null, config); + Map updatedConfig = new HashMap<>(config); + updatedConfig.put(NAME, clientName); - Map requestConfig = new HashMap<>(config); - requestConfig.put(ClientConfigProperties.CLIENT_NAME.getKey(), blank); - Assert.assertSame(identity.getUserAgent(null, requestConfig), userAgent); - Assert.assertSame(identity.getUserAgent(blank, requestConfig), userAgent); + identity.updateClientName(updatedConfig); + String userAgent = clientUserAgent(identity, updatedConfig); + + Assert.assertEquals(userAgent, expected); + Assert.assertSame(clientUserAgent(identity, updatedConfig), userAgent); } - @DataProvider(name = "blankValues") - public static Object[][] blankValues() { - return new Object[][]{{""}, {" "}}; + @DataProvider(name = "clientNameUpdates") + public static Object[][] clientNameUpdates() { + return new Object[][]{ + {settings(NAME, "my-app/1.0"), "new-app/2.0", userAgent("new-app/2.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), "new-app/2.0", + userAgent("new-app/2.0", "api:table")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), "new-app/2.0", + userAgent("custom-agent/2.0")}, + }; } @Test(dataProvider = "clientEntriesMetadata") public void testClientMetadataReplacesClientEntries(String metadata, String expectedComment) { - Map config = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), metadata)); + Map config = ClientConfigProperties.parseConfigMap(settings(METADATA, metadata)); Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), "clickhouse-java-v2/" + Client.clientVersion + " (" + expectedComment + ")"); } @DataProvider(name = "clientEntriesMetadata") public static Object[][] clientEntriesMetadata() { - String languageVersion = "lv:java/" + System.getProperty("java.version"); - String os = "os:" + System.getProperty("os.name"); - String httpClient = "hc:ahc_" + Versions.APACHE_HTTP_CLIENT; return new Object[][]{ - {"os=Android 14", "os:Android 14; " + languageVersion + "; " + httpClient}, - {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + os + "; " + httpClient}, - {"hc=okhttp_4.12", "hc:okhttp_4.12; " + languageVersion + "; " + os}, - {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + httpClient}, + {"os=Android 14", "os:Android 14; " + LANGUAGE_VERSION + "; " + HTTP_CLIENT}, + {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + OS + "; " + HTTP_CLIENT}, + {"hc=okhttp_4.12", "hc:okhttp_4.12; " + LANGUAGE_VERSION + "; " + OS}, + {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + HTTP_CLIENT}, {"os=Android 14,hc=okhttp_4.12,api=table,lv=kotlin/2.0", "os:Android 14; hc:okhttp_4.12; api:table; lv:kotlin/2.0"}, }; } @Test - public void testUserAgentWithoutClientMetadata() { - Map withMetadata = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table")); + public void testClientPartComposedPerInstance() { + Map withMetadata = ClientConfigProperties.parseConfigMap(settings(METADATA, "api=table")); WorkloadIdentitySupport identityWithMetadata = new WorkloadIdentitySupport(withMetadata); + Map withoutMetadata = ClientConfigProperties.parseConfigMap(settings()); + WorkloadIdentitySupport identityWithoutMetadata = new WorkloadIdentitySupport(withoutMetadata); - Map config = Collections.emptyMap(); - Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), userAgent("", "")); + Assert.assertEquals(identityWithMetadata.getUserAgent("my-app/1.0", withMetadata), + userAgent("my-app/1.0", "api:table")); + Assert.assertEquals(identityWithoutMetadata.getUserAgent("my-app/1.0", withoutMetadata), + userAgent("my-app/1.0")); + } - Map emptyMetadata = ClientConfigProperties.parseConfigMap( - Collections.singletonMap(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "")); - Assert.assertEquals(new WorkloadIdentitySupport(emptyMetadata).getUserAgent(null, emptyMetadata), - userAgent("", "")); + /** + * Returns the value for a request without operation settings. The transport passes the configured + * {@code User-Agent} header as the custom value. + */ + private static String clientUserAgent(WorkloadIdentitySupport identity, Map config) { + return identity.getUserAgent((String) config.get(USER_AGENT), config); + } - Assert.assertEquals(identityWithMetadata.getUserAgent("my-app/1.0", withMetadata), - userAgent("my-app/1.0 ", "api:table; ")); + private static Map settings(String... keysAndValues) { + Map settings = new HashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + settings.put(keysAndValues[i], keysAndValues[i + 1]); + } + return settings; + } + + private static String userAgent(String application) { + return userAgent(application, null); } private static String userAgent(String application, String metadata) { - return application + "clickhouse-java-v2/" + Client.clientVersion + " (" + metadata - + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") - + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; + return (application == null ? "" : application + " ") + "clickhouse-java-v2/" + Client.clientVersion + " (" + + (metadata == null ? "" : metadata + "; ") + LANGUAGE_VERSION + "; " + OS + "; " + HTTP_CLIENT + ")"; } } diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index 35c7dfaad..91c32c137 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -283,14 +283,21 @@ What is set by `setClientName` method comes first. Key/value pairs set by `setCl For example: ```java showLineNumbers -Map metadata = new LinkedHashMap<>(); -metadata.put("fv", "flink/2.0.0"); -metadata.put("api", "table"); +import com.clickhouse.client.api.Client; -Client client = new Client.Builder() - // ... - .setClientName("my-app-01/1.0", metadata) - .build(); +import java.util.LinkedHashMap; +import java.util.Map; + +public Client createClientWithMetadata() { + Map metadata = new LinkedHashMap<>(); + metadata.put("fv", "flink/2.0.0"); + metadata.put("api", "table"); + + return new Client.Builder() + // ... + .setClientName("my-app-01/1.0", metadata) + .build(); +} ``` will result in the following `http_user_agent` value: ``` diff --git a/docs/features.md b/docs/features.md index 8832ce820..ca416e24e 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,7 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .

` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with an empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with a `null` or empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/docs/integration-client.md b/docs/integration-client.md index 5a6cf51ad..89219b385 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -130,16 +130,23 @@ order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_ To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `hc` if the map does not set these keys (set them to replace the values of the client): ```java -Map metadata = new LinkedHashMap<>(); -metadata.put("fv", "flink/2.0.0"); -metadata.put("api", "table"); +import com.clickhouse.client.api.Client; -Client client = new Client.Builder() - .addEndpoint("http://localhost:8123") - .setUsername("default") - .setPassword("secret") - .setClientName("Flink-ClickHouse-Sink/0.2.0", metadata) - .build(); +import java.util.LinkedHashMap; +import java.util.Map; + +public Client createClientWithMetadata() { + Map metadata = new LinkedHashMap<>(); + metadata.put("fv", "flink/2.0.0"); + metadata.put("api", "table"); + + return new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("secret") + .setClientName("Flink-ClickHouse-Sink/0.2.0", metadata) + .build(); +} ``` ```text