diff --git a/client-v2/pom.xml b/client-v2/pom.xml index ad1f767cf..7b1db4ab3 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -210,12 +210,6 @@ - - - src/main/resources - true - - org.apache.maven.plugins @@ -231,6 +225,19 @@ 8 + + + org.codehaus.mojo + templating-maven-plugin + + + filter-sources + + filter-sources + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java new file mode 100644 index 000000000..7d2afcbf7 --- /dev/null +++ b/client-v2/src/main/java-templates/com/clickhouse/client/api/internal/Versions.java @@ -0,0 +1,22 @@ +package com.clickhouse.client.api.internal; + +/** + * Versions of the client and of the libraries that the client is built with. The build generates this class from a + * template, so the versions are correct also when the libraries are shaded into the client and cannot report their + * own version. + */ +public final class Versions { + + /** + * Version of the client. + */ + public static final String CLIENT = "${revision}"; + + /** + * Version of the Apache HttpClient library. + */ + public static final String APACHE_HTTP_CLIENT = "${apache.httpclient.version}"; + + private Versions() { + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 621f88efb..4ecde77c6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -25,6 +25,8 @@ import com.clickhouse.client.api.internal.MapUtils; import com.clickhouse.client.api.internal.TableSchemaParser; import com.clickhouse.client.api.internal.ValidationUtils; +import com.clickhouse.client.api.internal.Versions; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.client.api.metadata.ColumnToMethodMatchingStrategy; import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy; import com.clickhouse.client.api.metadata.TableSchema; @@ -49,7 +51,6 @@ import com.clickhouse.client.api.transport.HttpEndpoint; import com.clickhouse.client.api.transport.internal.TransportRequest; import com.clickhouse.client.api.transport.internal.TransportResponse; -import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; @@ -129,6 +130,8 @@ public class Client implements AutoCloseable { private HttpAPIClientHelper httpClientHelper = null; + private final WorkloadIdentitySupport workloadIdentitySupport; + private final List endpoints; private final Map configuration; private final Session session; @@ -235,8 +238,9 @@ private Client(Collection endpoints, Map configuration, this.lz4Factory = LZ4Factory.fastestJavaInstance(); } + this.workloadIdentitySupport = new WorkloadIdentitySupport(this.configuration); this.httpClientHelper = new HttpAPIClientHelper(this.configuration, metricsRegistry, initSslContext, lz4Factory, - this.spanRecorder); + this.spanRecorder, this.workloadIdentitySupport); this.serverVersion = configuration.getOrDefault(ClientConfigProperties.SERVER_VERSION.getKey(), "unknown"); this.dbUser = configuration.getOrDefault(ClientConfigProperties.USER.getKey(), ClientConfigProperties.USER.getDefObjVal()); this.typeHintMapping = (Map>) this.configuration.get(ClientConfigProperties.TYPE_HINT_MAPPING.getKey()); @@ -1004,6 +1008,8 @@ public Builder allowBinaryReaderToReuseBuffers(boolean reuse) { * Defines list of headers that should be sent with each request. The Client will use a header value * defined in {@code headers} instead of any other. * Operation settings may override these headers. + * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify + * the application, use {@link #setClientName(String)} instead of the {@code User-Agent} header. * * @see InsertSettings#httpHeaders(Map) * @see QuerySettings#httpHeaders(Map) @@ -1139,13 +1145,17 @@ public Builder useHTTPBasicAuth(boolean useBasicAuth) { } /** - * Sets additional information about calling application. This string will be passed to server as a client name. - * In case of HTTP protocol it will be passed as a {@code User-Agent} header. - * Warn: If custom value of User-Agent header is set it will override this value for HTTP transport - * Client name is used by server to identify client application when investigating {@code system.query_log}. In case of HTTP - * transport this value will be in the {@code system.query_log.http_user_agent} column. Currently only HTTP transport is used. + * Sets the client name, which identifies the application to the server. The client sends it in the + * {@code User-Agent} header, before its own part {@code clickhouse-java-v2/ (...)}. The server records + * the header in the {@code http_user_agent} column of {@code system.query_log}. + *

+ * To identify the application, set the client name. Do not set the {@code User-Agent} header with + * {@link #httpHeader(String, String)}: a custom {@code User-Agent} header replaces the client name. + *

+ * This method does not change the client metadata. To also add key/value pairs to the {@code User-Agent} + * header, use {@link #setClientName(String, Map)}. * - * @param clientName - client application display name. + * @param clientName - client application name, for example {@code my-app/1.0} * @return same instance of the builder */ public Builder setClientName(String clientName) { @@ -1153,6 +1163,37 @@ public Builder setClientName(String clientName) { return this; } + /** + * Sets the client name and the client metadata, which identify the application to the server. See + * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the + * {@code User-Agent} header, in the iteration order of {@code metadata}, and then its own {@code lv}, + * {@code os} and {@code hc} (HTTP client) entries if the metadata does not set these keys. For example, the + * client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give + * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; + * os:; hc:ahc_)}. + *

+ * Keys and values are trimmed, and entries with a {@code null} or empty key or value are ignored. Keys and + * values must be printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not + * contain any of {@code ;()}; otherwise {@link #build()} throws {@link ClientMisconfigurationException}. The + * metadata is a client-level setting: it is kept when the client name changes with + * {@link Client#updateClientName(String)}. + * + * @param clientName - client application name, for example {@code my-app/1.0} + * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata + * @return same instance of the builder + * @see ClientConfigProperties#CLIENT_ID_METADATA + */ + public Builder setClientName(String clientName, Map metadata) { + setClientName(clientName); + if (metadata == null || metadata.isEmpty()) { + this.configuration.remove(ClientConfigProperties.CLIENT_ID_METADATA.getKey()); + } else { + this.configuration.put(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), + ClientConfigProperties.fromKeyValuePairs(metadata)); + } + return this; + } + /** * Sets client options from provided map. Values are copied as is * @param options - map of client options @@ -2563,8 +2604,15 @@ public void setDBRoles(Collection dbRoles) { this.unmodifiableDbRolesView = ImmutableList.copyOf(tmp); } + /** + * Updates the client name for all subsequent requests created by this client. The client composes the value of + * the {@code User-Agent} header again, and keeps it. + * + * @param name - client application name, for example {@code my-app/1.0} + */ public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); + this.workloadIdentitySupport.updateClientName(this.configuration); } /** @@ -2575,8 +2623,8 @@ public void updateSessionId(String sessionId) { this.session.updateSessionId(sessionId); } - public static final String clientVersion = - ClickHouseClientOption.readVersionFromResource("client-v2-version.properties"); + // not a compile-time constant, so that code compiled against another version reads the version of this client + public static final String clientVersion = String.valueOf(Versions.CLIENT); public static final String CLIENT_USER_AGENT = "clickhouse-java-v2/"; private Collection unmodifiableDbRolesView = Collections.emptyList(); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index 63173c5a2..17d4f701c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -4,6 +4,7 @@ import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; import com.clickhouse.client.api.enums.SSLMode; import com.clickhouse.client.api.internal.CompressedBlockOutputStream; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; import org.slf4j.Logger; @@ -257,6 +258,24 @@ public Object parseValue(String value) { .collect(Collectors.toList()); } }, + + /** + * Key/value pairs, written as {@code key1=value1,key2=value2}, that the client adds to the comment group of the + * {@code User-Agent} header. A {@code ,} or {@code \} in a value is escaped with {@code \}. This is a client-level + * setting: it is not read from operation settings. See {@link Client.Builder#setClientName(String, Map)} for the + * format and the rules for keys and values. + */ + CLIENT_ID_METADATA("client_id_metadata", Map.class) { + @Override + @SuppressWarnings("unchecked") + public Object parseValue(String value) { + Map metadata = (Map) super.parseValue(value); + if (metadata != null) { + metadata.forEach(WorkloadIdentitySupport::checkClientMetadataEntry); + } + return metadata; + } + }, ; private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); @@ -506,6 +525,36 @@ public static Map toKeyValuePairs(String str) { + /** + * Inverse of {@link #toKeyValuePairs(String)}: escapes {@code \}, {@code ,} and {@code =} with {@code \} and + * skips entries with a {@code null} key or value. + */ + static String fromKeyValuePairs(Map map) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + continue; + } + if (sb.length() > 0) { + sb.append(','); + } + appendEscaped(sb, entry.getKey()).append('='); + appendEscaped(sb, entry.getValue()); + } + return sb.toString(); + } + + private static StringBuilder appendEscaped(StringBuilder sb, String str) { + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if (ch == '\\' || ch == ',' || ch == '=') { + sb.append('\\'); + } + sb.append(ch); + } + return sb; + } + public static String mapToString(Map map, Function valueConverter) { StringBuilder sb = new StringBuilder(); for (Map.Entry entry : map.entrySet()) { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java index 9635fc4d7..9814c27b1 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClientUtils.java @@ -3,6 +3,8 @@ import org.slf4j.Logger; import java.io.Closeable; +import java.util.Map; +import java.util.Objects; /** * Class containing utility methods used across the client. @@ -28,4 +30,24 @@ public static void quietClose(Closeable closeable, Logger log) { } } } + + /** + * Puts the entry into the map if the map does not have the key. Keys are compared ignoring case, so the map is not + * changed if it has the key in a different case. A key that the map has is not replaced, also if its value is + * {@code null}. + * + * @param map - map to update + * @param key - key of the entry, not {@code null} + * @param value - value of the entry + * @param - type of the values + */ + public static void putIfAbsentIgnoreCase(Map map, String key, V value) { + Objects.requireNonNull(key, "key"); + for (String mapKey : map.keySet()) { + if (key.equalsIgnoreCase(mapKey)) { + return; + } + } + map.put(key, value); + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java index e84873fb5..b94422bb9 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java @@ -178,6 +178,8 @@ public String getDatabase() { /** * Defines list of headers that should be sent with current request. The Client will use a header value * defined in {@code headers} instead of any other. + * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify + * the application, use {@link Client.Builder#setClientName(String)} instead of the {@code User-Agent} header. * * @see Client.Builder#httpHeaders(Map) * @param key - header name. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 70db8d438..d2c188c14 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -1,7 +1,6 @@ package com.clickhouse.client.api.internal; import com.clickhouse.client.api.ClickHouseException; -import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientException; import com.clickhouse.client.api.ClientFaultCause; @@ -105,7 +104,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; @@ -135,7 +133,11 @@ public class HttpAPIClientHelper { private final Set defaultRetryCauses; - private final String defaultUserAgent; + /** + * Identification of the client, which gives the value of the {@code User-Agent} header. {@code Client} creates + * and updates it. Never {@code null}. + */ + private final WorkloadIdentitySupport workloadIdentitySupport; private final Object metricsRegistry; ConnPoolControl poolControl; @@ -152,9 +154,12 @@ public class HttpAPIClientHelper { private final SpanRecorder spanRecorder; public HttpAPIClientHelper(Map configuration, Object metricsRegistry, boolean initSslContext, - LZ4Factory lz4Factory, SpanRecorder spanRecorder) { + LZ4Factory lz4Factory, SpanRecorder spanRecorder, + WorkloadIdentitySupport workloadIdentitySupport) { this.spanRecorder = Objects.requireNonNull(spanRecorder, "spanRecorder is required; use DefaultSpanRecorder.NOOP to record nothing"); + this.workloadIdentitySupport = Objects.requireNonNull(workloadIdentitySupport, + "workloadIdentitySupport is required"); this.metricsRegistry = metricsRegistry; this.httpClient = createHttpClient(initSslContext, configuration); this.lz4Factory = lz4Factory; @@ -170,8 +175,6 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi if (defaultRetryCauses.contains(ClientFaultCause.None)) { defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } - - this.defaultUserAgent = buildDefaultUserAgent(); } /** @@ -1190,62 +1193,11 @@ public RuntimeException wrapException(String message, Exception cause, String qu } private void correctUserAgentHeader(HttpRequest request, Map requestConfig) { - //TODO: implement cache for user-agent Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); - String clientName = ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig); - String userAgentValue = defaultUserAgent; - if (userAgentHeader == null && clientName != null && !clientName.isEmpty()) { - userAgentValue = clientName + " " + defaultUserAgent; - } else if (userAgentHeader != null) { - userAgentValue = userAgentHeader.getValue() + " " + defaultUserAgent; - } - request.setHeader(HttpHeaders.USER_AGENT, userAgentValue); - } - - private String buildDefaultUserAgent() { - StringBuilder userAgent = new StringBuilder(); - userAgent.append(Client.CLIENT_USER_AGENT); - - String clientVersion = Client.clientVersion; - - userAgent.append(clientVersion); - - userAgent.append(" ("); - userAgent.append(System.getProperty("os.name")); - userAgent.append("; "); - userAgent.append("jvm:").append(System.getProperty("java.version")); - userAgent.append("; "); - - userAgent.setLength(userAgent.length() - 2); - userAgent.append(')'); - - try { - String httpClientVersion = this.httpClient.getClass().getPackage().getImplementationVersion(); - if (Objects.equals(this.httpClient.getClass().getPackage().getImplementationTitle(), this.getClass().getPackage().getImplementationTitle())) { - // shaded jar - all packages have same implementation title - httpClientVersion = "unknown"; - try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("client-v2-version.properties")) { - Properties p = new Properties(); - p.load(in); - - String tmp = p.getProperty("apache.http.client.version"); - if (tmp != null && !tmp.isEmpty() && !tmp.equals("${apache.httpclient.version}")) { - httpClientVersion = tmp; - } - } catch (Exception e) { - LOG.debug("Failed to read HTTP client version from client-v2-version.properties", e); - } - } - userAgent.append(" ") - .append("Apache-HttpClient") - .append('/') - .append(httpClientVersion); - } catch (Exception e) { - LOG.warn("failed to construct http client version string", e); - } - return userAgent.toString(); + request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.getUserAgent( + userAgentHeader == null ? null : userAgentHeader.getValue(), requestConfig)); } public void close() { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java new file mode 100644 index 000000000..29106c527 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -0,0 +1,166 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.ClientMisconfigurationException; +import org.apache.hc.core5.http.HttpHeaders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.StringJoiner; + +/** + * Builds the identification of the client and of the application that uses it. The identification is the value of + * the {@code User-Agent} header, in the standard ClickHouse client format: + * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:; + * hc:ahc_)}. + *

+ * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), or the custom + * {@code User-Agent} header if the application set one. The key/value pairs are the client metadata + * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). The {@code hc} entry is the HTTP client that sends the requests: + * {@code ahc_} (Apache HttpClient) and its version. + *

+ * The client metadata is validated when {@link Client.Builder#build()} parses the configuration. {@link Client} + * creates one instance and updates it when the client name changes. The instance composes the value for the client + * configuration at these times, and keeps it. A transport only sends the value of {@link #getUserAgent(String, Map)} + * with every request. + */ +public final class WorkloadIdentitySupport { + + private static final Logger LOG = LoggerFactory.getLogger(WorkloadIdentitySupport.class); + + private static final String METADATA_KEY_LANGUAGE_VERSION = "lv"; + + private static final String METADATA_KEY_OS = "os"; + + private static final String METADATA_KEY_HTTP_CLIENT = "hc"; + + /** + * First and last characters of the printable ASCII range, which client metadata keys and values must be in. + */ + private static final char START_OF_ASCII_RANGE = ' '; + + private static final char END_OF_ASCII_RANGE = '~'; + + /** + * Length of the {@code User-Agent} value above which the client logs a warning. Common browsers send about 120 + * characters. + */ + private static final int USER_AGENT_WARNING_LENGTH = 200; + + private final String clientPart; + + private volatile String clientApplication; + + private volatile String clientUserAgent; + + /** + * Composes the client part and the value of the {@code User-Agent} header for the client configuration. + * + * @param configuration - parsed client configuration + */ + public WorkloadIdentitySupport(Map configuration) { + this.clientPart = composeClientPart(configuration); + updateClientName(configuration); + } + + /** + * Composes the value of the {@code User-Agent} header for the client configuration again, and keeps it. Call it + * after the client name changes. Logs a warning if the value is longer than 200 characters. + *

+ * Updates are synchronized, so the kept application part and value always belong together. A request that runs + * at the same time as an update can get the value of the new client name. + * + * @param configuration - client configuration + */ + public synchronized void updateClientName(Map configuration) { + String application = applicationPart( + (String) configuration.get(ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT)), configuration); + String userAgent = composeUserAgent(application); + if (userAgent.length() > USER_AGENT_WARNING_LENGTH) { + LOG.warn("The User-Agent value is {} characters long, more than {}: '{}'. Use a shorter client name or" + + " less client metadata.", userAgent.length(), USER_AGENT_WARNING_LENGTH, userAgent); + } + // the value is written first, so a request that reads the new application part also reads the new value + this.clientUserAgent = userAgent; + this.clientApplication = application; + } + + /** + * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part. + * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and + * the client name otherwise. Leading and trailing whitespace is removed. If the application part is blank or is + * the one of the client configuration, the value that this instance keeps is returned. Otherwise, for example for + * a {@code User-Agent} header of one operation, a new value is composed and not kept. + * + * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null} + * @param requestConfig - settings of the request + * @return value of the {@code User-Agent} header + */ + public String getUserAgent(String customUserAgent, Map requestConfig) { + String application = applicationPart(customUserAgent, requestConfig); + return application.isEmpty() || application.equals(clientApplication) + ? clientUserAgent : composeUserAgent(application); + } + + /** + * Returns the trimmed application part, or an empty string if it is not set. + */ + private static String applicationPart(String customUserAgent, Map config) { + String value = ClientUtils.isBlank(customUserAgent) + ? ClientConfigProperties.CLIENT_NAME.getOrDefault(config) : customUserAgent; + return ClientUtils.isBlank(value) ? "" : value.trim(); + } + + private String composeUserAgent(String application) { + return application.isEmpty() ? clientPart : application + " " + clientPart; + } + + /** + * Checks that a client metadata entry does not break the format of the {@code User-Agent} header. + * + * @param key - trimmed, not empty key + * @param value - trimmed, not empty value + * @throws ClientMisconfigurationException if the key or the value contains a character that is not allowed + */ + public static void checkClientMetadataEntry(String key, String value) { + // ':' separates a key from its value, ';' separates entries, '(' and ')' delimit the comment group, + // and ',' and '=' are separators of the client_id_metadata property value + if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) { + throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_ID_METADATA.getKey() + + "' entry '" + key + "=" + value + "': keys and values must be printable ASCII, a key must not" + + " contain spaces or any of ':;(),=', and a value must not contain any of ';()'"); + } + } + + private static boolean isClientMetadataText(String str, String forbiddenChars) { + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if (ch < START_OF_ASCII_RANGE || ch > END_OF_ASCII_RANGE || forbiddenChars.indexOf(ch) >= 0) { + return false; + } + } + return true; + } + + /** + * Composes the client part: {@code clickhouse-java-v2/ (:; ...)}. The key/value pairs are the + * client metadata, if it is set, then {@code lv:java/}, {@code os:} and + * {@code hc:ahc_} if the client metadata does not set these keys (ignoring case). The + * client sends requests over HTTP with Apache HttpClient ({@code ahc}). + */ + private static String composeClientPart(Map configuration) { + Map metadata = new LinkedHashMap<>(); + ClientConfigProperties.CLIENT_ID_METADATA.>applyIfSet(configuration, metadata::putAll); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_LANGUAGE_VERSION, + "java/" + System.getProperty("java.version")); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_OS, System.getProperty("os.name")); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_HTTP_CLIENT, "ahc_" + Versions.APACHE_HTTP_CLIENT); + + StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); + metadata.forEach((key, value) -> comment.add(key + ":" + value)); + return comment.toString(); + } +} diff --git a/client-v2/src/main/resources/client-v2-version.properties b/client-v2/src/main/resources/client-v2-version.properties deleted file mode 100644 index a617d20fb..000000000 --- a/client-v2/src/main/resources/client-v2-version.properties +++ /dev/null @@ -1,3 +0,0 @@ -version=${revision} -build.date=${build_timestamp} -apache.http.client.version=${apache.httpclient.version} \ No newline at end of file diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index b8b412742..608a9e5a9 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -1201,9 +1201,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider") public static Object[][] testUserAgentHasCompleteProductName_dataProvider() { return new Object[][] { - { "", Pattern.compile("clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$") }, - { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")}, - { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")}}; + { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)") }, + { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}, + { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}}; } @Test(dataProvider = "testClientNameDataProvider") diff --git a/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java b/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java index f9fa0a383..091cff3e1 100644 --- a/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/InsertExceptionClassificationTest.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.DataTransferException; import com.clickhouse.client.api.internal.HttpAPIClientHelper; +import com.clickhouse.client.api.internal.WorkloadIdentitySupport; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.observability.DefaultSpanRecorder; import com.clickhouse.client.api.serde.DataSerializationException; @@ -101,7 +102,8 @@ private static final class CallbackHttpClientHelper extends HttpAPIClientHelper private final OutputStream outputStream; private IOCallback writeCallback; private CallbackHttpClientHelper(OutputStream outputStream) { - super(Collections.emptyMap(), null, false, LZ4Factory.fastestJavaInstance(), DefaultSpanRecorder.NOOP); + super(Collections.emptyMap(), null, false, LZ4Factory.fastestJavaInstance(), DefaultSpanRecorder.NOOP, + new WorkloadIdentitySupport(Collections.emptyMap())); this.outputStream = outputStream; } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index 9c23ca522..a721ad248 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -5,9 +5,11 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -76,4 +78,58 @@ public void testParseConfigMapSanitizesSslCipherSuites() { Assert.assertEquals(parsed.get(ClientConfigProperties.SSL_CIPHER_SUITES.getKey()), Arrays.asList("TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256")); } + + @DataProvider(name = "validClientMetadata") + public static Object[][] validClientMetadata() { + return new Object[][]{ + {"fv=flink/2.0.0,api=table", new String[]{"fv", "flink/2.0.0", "api", "table"}}, + {" api = table , os_version = Mac OS X ", new String[]{"api", "table", "os_version", "Mac OS X"}}, + {"url=http://host:8123/path?x=1", new String[]{"url", "http://host:8123/path?x=1"}}, + {"ver~=~1.0", new String[]{"ver~", "~1.0"}}, + {"os=Android 14,lv=java/8", new String[]{"os", "Android 14", "lv", "java/8"}}, + {"api=,fv=flink/2.0.0,=x", new String[]{"fv", "flink/2.0.0"}}, + {"", new String[0]}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "validClientMetadata") + public void testClientMetadataParsed(String raw, String[] expectedEntries) { + Map expected = new LinkedHashMap<>(); + for (int i = 0; i < expectedEntries.length; i += 2) { + expected.put(expectedEntries[i], expectedEntries[i + 1]); + } + Map parsed = (Map) ClientConfigProperties.CLIENT_ID_METADATA.parseValue(raw); + Assert.assertEquals(new ArrayList<>(parsed.entrySet()), new ArrayList<>(expected.entrySet())); + } + + @DataProvider(name = "invalidClientMetadata") + public static Object[][] invalidClientMetadata() { + return new Object[][]{ + {"a:b=c"}, {"a;b=c"}, {"a(b=c"}, {"a)b=c"}, {"a\\,b=c"}, {"a\\=b=c"}, {"my key=c"}, + {"k=v;x"}, {"k=v(x"}, {"k=v)x"}, + {"k=a\nb"}, {"k=a\tb"}, {"k=a\u001fb"}, {"k=a\u007fb"}, {"k\u007f=v"}, {"k=caf\u00e9"}, {"k\u00e9=v"}, + {"fv=flink/2.0.0,api=table; lv:java/8"}, + {",api=table"}, {"fv=flink/2.0.0,,api=table"}, {"fv=flink/2.0.0,api,ver=1"}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "invalidClientMetadata") + public void testInvalidClientMetadataRejected(String raw) { + Assert.expectThrows(ClientMisconfigurationException.class, + () -> ClientConfigProperties.CLIENT_ID_METADATA.parseValue(raw)); + } + + @Test(groups = {"unit"}) + public void testClientMetadataUnsetParsesToNull() { + Assert.assertNull(ClientConfigProperties.CLIENT_ID_METADATA.parseValue(null)); + } + + @Test(groups = {"unit"}) + public void testFromKeyValuePairsRoundTrip() { + Map pairs = new LinkedHashMap<>(); + pairs.put("a=b", "c,d\\e=f"); + pairs.put("x,y", "z"); + Map parsed = ClientConfigProperties.toKeyValuePairs(ClientConfigProperties.fromKeyValuePairs(pairs)); + Assert.assertEquals(new ArrayList<>(parsed.entrySet()), new ArrayList<>(pairs.entrySet())); + } } \ No newline at end of file diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java new file mode 100644 index 000000000..723d825ae --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -0,0 +1,170 @@ +package com.clickhouse.client.api; + +import com.clickhouse.client.api.internal.Versions; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import org.apache.hc.core5.http.HttpHeaders; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +public class UserAgentUnitTest { + + private WireMockServer mockServer; + + @BeforeMethod + public void setUp() { + mockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + mockServer.start(); + mockServer.stubFor(WireMock.post(WireMock.anyUrl()) + .willReturn(WireMock.aResponse().withStatus(200))); + } + + @AfterMethod + public void tearDown() { + if (mockServer != null) { + mockServer.stop(); + } + } + + @Test(dataProvider = "userAgentData") + public void testUserAgentHeader(Consumer configure, String expectedUserAgent) throws Exception { + Client.Builder builder = newClientBuilder(); + configure.accept(builder); + try (Client client = builder.build()) { + assertUserAgent(client, null, expectedUserAgent); + } + } + + @DataProvider(name = "userAgentData") + public static Object[][] userAgentData() { + Map flinkMetadata = new LinkedHashMap<>(); + flinkMetadata.put("fv", "flink/2.0.0"); + flinkMetadata.put("api", "table"); + + Map osMetadata = new LinkedHashMap<>(); + osMetadata.put("os", "Android 14"); + osMetadata.put("api", "table"); + + Map withIgnoredEntries = new LinkedHashMap<>(); + withIgnoredEntries.put("api", ""); + withIgnoredEntries.put(" ", "x"); + withIgnoredEntries.put("os", null); + withIgnoredEntries.put(null, "x"); + withIgnoredEntries.put("fv", "flink/2.0.0"); + + return new Object[][]{ + {(Consumer) b -> { }, userAgent(null, null)}, + {(Consumer) b -> b.setClientName("Flink-ClickHouse-Sink/0.2.0", flinkMetadata), + userAgent("Flink-ClickHouse-Sink/0.2.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), + "api=table, fv=flink/2.0.0"), + userAgent(null, "api:table; fv:flink/2.0.0")}, + {(Consumer) b -> b.setClientName("my-app/1.0", + Collections.singletonMap("build", "a=b,c\\d")), + userAgent("my-app/1.0", "build:a=b,c\\d")}, + {(Consumer) b -> b.setClientName("my-app/1.0", withIgnoredEntries), + userAgent("my-app/1.0", "fv:flink/2.0.0")}, + {(Consumer) b -> b.setClientName("", flinkMetadata), + userAgent(null, "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName(" ", flinkMetadata), + userAgent(null, "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table") + .setClientName("my-app/1.0"), + userAgent("my-app/1.0", "api:table")}, + {(Consumer) b -> b.setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "api=table") + .setClientName("my-app/1.0", flinkMetadata), + userAgent("my-app/1.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0"), + userAgent("new-app/2.0", "fv:flink/2.0.0; api:table")}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0", null), + userAgent("new-app/2.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", flinkMetadata) + .setClientName("new-app/2.0", Collections.emptyMap()), + userAgent("new-app/2.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", Collections.singletonMap("api", "table")) + .httpHeader(HttpHeaders.USER_AGENT, "custom-agent/2.0"), + userAgent("custom-agent/2.0", "api:table")}, + {(Consumer) b -> b.setClientName(" my-app/1.0 "), + userAgent("my-app/1.0", null)}, + {(Consumer) b -> b.setClientName("my-app/1.0", osMetadata), + "my-app/1.0 clickhouse-java-v2/" + Client.clientVersion + " (os:Android 14; api:table; lv:java/" + + System.getProperty("java.version") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + + ")"}, + }; + } + + @Test + public void testClientMetadataKeptWhenApplicationPartChanges() throws Exception { + try (Client client = newClientBuilder() + .setClientName("my-app/1.0", Collections.singletonMap("api", "table")) + .build()) { + client.updateClientName("new-app/2.0"); + assertUserAgent(client, null, userAgent("new-app/2.0", "api:table")); + assertUserAgent(client, new QuerySettings().httpHeader(HttpHeaders.USER_AGENT, "op-agent/3.0"), + userAgent("op-agent/3.0", "api:table")); + assertUserAgent(client, (QuerySettings) new QuerySettings() + .setOption(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), "fv=flink/2.0.0"), + userAgent("new-app/2.0", "api:table")); + } + } + + @Test + public void testBlankOperationClientNameGetsClientValue() throws Exception { + try (Client client = newClientBuilder().setClientName("my-app/1.0").build()) { + assertUserAgent(client, (QuerySettings) new QuerySettings() + .setOption(ClientConfigProperties.CLIENT_NAME.getKey(), " "), + userAgent("my-app/1.0", null)); + } + } + + @Test + public void testInvalidClientMetadataRejectedOnBuild() { + Client.Builder builder = newClientBuilder() + .setClientName("my-app/1.0", Collections.singletonMap("api", "table; os:other")); + Assert.expectThrows(ClientMisconfigurationException.class, builder::build); + } + + private Client.Builder newClientBuilder() { + return new Client.Builder() + .addEndpoint("http://localhost:" + mockServer.port()) + .setUsername("default") + .setPassword(""); + } + + private void assertUserAgent(Client client, QuerySettings settings, String expectedUserAgent) throws Exception { + mockServer.resetRequests(); + try (QueryResponse response = client.query("SELECT 1", settings).get(10, TimeUnit.SECONDS)) { + Assert.assertNotNull(response); + } + List requests = mockServer.findAll(WireMock.postRequestedFor(WireMock.anyUrl())); + Assert.assertFalse(requests.isEmpty(), "no request was sent"); + for (LoggedRequest request : requests) { + Assert.assertEquals(request.getHeader(HttpHeaders.USER_AGENT), expectedUserAgent); + } + } + + private static String userAgent(String application, String metadata) { + return (application == null ? "" : application + " ") + + "clickhouse-java-v2/" + Client.clientVersion + + " (" + (metadata == null ? "" : metadata + "; ") + + "lv:java/" + System.getProperty("java.version") + + "; os:" + System.getProperty("os.name") + + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java index 88c5d8526..7be73b265 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClientUtilsTest.java @@ -3,10 +3,13 @@ import org.mockito.Mockito; import org.slf4j.Logger; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.Closeable; import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; public class ClientUtilsTest { @@ -45,4 +48,36 @@ public void testQuietCloseWithNull() { Mockito.verifyNoInteractions(log); Assert.assertTrue(true); } + + @Test(groups = {"unit"}, dataProvider = "putIfAbsentIgnoreCase") + public void testPutIfAbsentIgnoreCase(String[] existingKeys, String key, String expectedMap) { + Map map = new LinkedHashMap<>(); + for (String existingKey : existingKeys) { + map.put(existingKey, "set"); + } + + ClientUtils.putIfAbsentIgnoreCase(map, key, "added"); + + Assert.assertEquals(map.toString(), expectedMap); + } + + @DataProvider(name = "putIfAbsentIgnoreCase") + public static Object[][] putIfAbsentIgnoreCaseData() { + return new Object[][]{ + {new String[]{}, "os", "{os=added}"}, + {new String[]{"os"}, "lv", "{os=set, lv=added}"}, + {new String[]{"os"}, "os", "{os=set}"}, + {new String[]{"os"}, "OS", "{os=set}"}, + {new String[]{"OS"}, "os", "{OS=set}"}, + {new String[]{"api", "OS"}, "os", "{api=set, OS=set}"}, + }; + } + + @Test(groups = {"unit"}) + public void testPutIfAbsentIgnoreCaseNullKey() { + Map map = new LinkedHashMap<>(); + + Assert.expectThrows(NullPointerException.class, () -> ClientUtils.putIfAbsentIgnoreCase(map, null, "added")); + Assert.assertTrue(map.isEmpty()); + } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java index d24fdd08c..1352379f8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperFactory.java @@ -17,13 +17,15 @@ private HttpAPIClientHelperFactory() { /** * Creates a helper with no metrics registry, no SSL context and observability disabled - the - * configuration every test that does not exercise those features needs. + * configuration every test that does not exercise those features needs. The helper gets a new + * {@link WorkloadIdentitySupport} for {@code configuration}, as {@code Client} gives it one. * * @param configuration - client configuration * @param lz4Factory - LZ4 factory the helper should use * @return a new helper instance */ public static HttpAPIClientHelper newHelper(Map configuration, LZ4Factory lz4Factory) { - return new HttpAPIClientHelper(configuration, null, false, lz4Factory, DefaultSpanRecorder.NOOP); + return new HttpAPIClientHelper(configuration, null, false, lz4Factory, DefaultSpanRecorder.NOOP, + new WorkloadIdentitySupport(configuration)); } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index ddc3aa8fc..33ad619fa 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -1,5 +1,6 @@ package com.clickhouse.client.api.internal; +import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientFaultCause; import com.clickhouse.client.api.ServerException; @@ -54,6 +55,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; public class HttpAPIClientHelperTest { @@ -512,6 +514,76 @@ private static Map compressionConfig(boolean clientCompression, return reqConfig; } + /** + * {@link Client#updateClientName(String)} composes the {@code User-Agent} value again and keeps it, so every + * following request with that client name gets the same value. + */ + @Test + public void testUserAgentKeptAfterClientNameUpdate() throws Exception { + try (Client client = newClient("my-app/1.0")) { + client.updateClientName("new-app/2.0"); + HttpAPIClientHelper helper = helperOf(client); + + String userAgent = userAgentHeader(helper, "new-app/2.0"); + assertEquals(userAgent, "new-app/2.0 " + clientPart()); + assertSame(userAgentHeader(helper, "new-app/2.0"), userAgent); + } + } + + /** + * A {@code User-Agent} value of the client configuration that is longer than 200 characters is logged as a + * warning, when the client is created and when the client name changes. A value composed for one request is not + * logged, so that the log does not repeat for every request. + */ + @Test + public void testLongUserAgentLoggedAsWarning() throws Exception { + String atLimit = String.join("", Collections.nCopies(200 - 1 - clientPart().length(), "x")); + String overLimit = atLimit + "x"; + String warning = "The User-Agent value is "; + + assertFalse(captureStdErr(() -> newClient(atLimit).close()).contains(warning)); + assertTrue(captureStdErr(() -> newClient(overLimit).close()) + .contains(warning + "201 characters long, more than 200")); + + try (Client client = newClient("my-app/1.0")) { + HttpAPIClientHelper helper = helperOf(client); + assertFalse(captureStdErr(() -> userAgentHeader(helper, overLimit)).contains(warning)); + assertTrue(captureStdErr(() -> client.updateClientName(overLimit)).contains(warning)); + } + } + + private static Client newClient(String clientName) { + return new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("") + .setClientName(clientName) + .build(); + } + + private static HttpAPIClientHelper helperOf(Client client) throws Exception { + Field helperField = Client.class.getDeclaredField("httpClientHelper"); + helperField.setAccessible(true); + return (HttpAPIClientHelper) helperField.get(client); + } + + private static Map clientNameConfig(String clientName) { + Map config = new HashMap<>(); + config.put(ClientConfigProperties.CLIENT_NAME.getKey(), clientName); + return config; + } + + private static String userAgentHeader(HttpAPIClientHelper helper, String clientName) { + HttpPost req = helper.createRequest(new HttpEndpoint("localhost", 8123, false, "/"), + clientNameConfig(clientName), "SELECT 1").getDelegate(); + return headerValue(req, HttpHeaders.USER_AGENT); + } + + private static String clientPart() { + return "clickhouse-java-v2/" + Client.clientVersion + " (lv:java/" + System.getProperty("java.version") + + "; os:" + System.getProperty("os.name") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; + } + private static String headerValue(HttpPost req, String name) { Header header = req.getFirstHeader(name); return header == null ? null : header.getValue(); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java new file mode 100644 index 000000000..0a005061f --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/VersionsTest.java @@ -0,0 +1,19 @@ +package com.clickhouse.client.api.internal; + +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.util.VersionInfo; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class VersionsTest { + + @Test + public void testVersionsGenerated() { + Assert.assertTrue(Versions.CLIENT.matches("\\d+\\.\\d+\\.\\d+.*"), Versions.CLIENT); + + VersionInfo versionInfo = VersionInfo.loadVersionInfo("org.apache.hc.client5", + HttpClients.class.getClassLoader()); + Assert.assertNotNull(versionInfo); + Assert.assertEquals(Versions.APACHE_HTTP_CLIENT, versionInfo.getRelease()); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java new file mode 100644 index 000000000..b3861e960 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -0,0 +1,151 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import org.apache.hc.core5.http.HttpHeaders; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +public class WorkloadIdentitySupportTest { + + private static final String NAME = ClientConfigProperties.CLIENT_NAME.getKey(); + + private static final String METADATA = ClientConfigProperties.CLIENT_ID_METADATA.getKey(); + + private static final String USER_AGENT = ClientConfigProperties.httpHeader(HttpHeaders.USER_AGENT); + + private static final String LANGUAGE_VERSION = "lv:java/" + System.getProperty("java.version"); + + private static final String OS = "os:" + System.getProperty("os.name"); + + private static final String HTTP_CLIENT = "hc:ahc_" + Versions.APACHE_HTTP_CLIENT; + + @Test(dataProvider = "requestUserAgents") + public void testGetUserAgent(Map clientSettings, String customUserAgent, String requestClientName, + String expected) { + Map config = ClientConfigProperties.parseConfigMap(clientSettings); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + String clientUserAgent = clientUserAgent(identity, config); + Map requestConfig = new HashMap<>(config); + if (requestClientName != null) { + requestConfig.put(NAME, requestClientName); + } + + String userAgent = identity.getUserAgent(customUserAgent, requestConfig); + + Assert.assertEquals(userAgent, expected); + if (expected.equals(clientUserAgent)) { + Assert.assertSame(userAgent, clientUserAgent); + } + Assert.assertSame(clientUserAgent(identity, config), clientUserAgent); + } + + @DataProvider(name = "requestUserAgents") + public static Object[][] requestUserAgents() { + return new Object[][]{ + {settings(), null, null, userAgent(null)}, + {settings(METADATA, ""), null, null, userAgent(null)}, + {settings(NAME, "my-app/1.0"), null, null, userAgent("my-app/1.0")}, + {settings(NAME, " my-app/1.0 "), null, null, userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), null, null, + userAgent("my-app/1.0", "api:table")}, + {settings(NAME, "my-app/1.0"), " ", null, userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0"), null, "", userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0"), "", " ", userAgent("my-app/1.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), null, "op-app/3.0", + userAgent("op-app/3.0", "api:table")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), " custom-agent/2.0 ", null, + userAgent("custom-agent/2.0", "api:table")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), "custom-agent/2.0", null, + userAgent("custom-agent/2.0")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), null, null, + userAgent("my-app/1.0")}, + }; + } + + @Test(dataProvider = "clientNameUpdates") + public void testUpdateClientName(Map clientSettings, String clientName, String expected) { + Map config = ClientConfigProperties.parseConfigMap(clientSettings); + WorkloadIdentitySupport identity = new WorkloadIdentitySupport(config); + Map updatedConfig = new HashMap<>(config); + updatedConfig.put(NAME, clientName); + + identity.updateClientName(updatedConfig); + String userAgent = clientUserAgent(identity, updatedConfig); + + Assert.assertEquals(userAgent, expected); + Assert.assertSame(clientUserAgent(identity, updatedConfig), userAgent); + } + + @DataProvider(name = "clientNameUpdates") + public static Object[][] clientNameUpdates() { + return new Object[][]{ + {settings(NAME, "my-app/1.0"), "new-app/2.0", userAgent("new-app/2.0")}, + {settings(NAME, "my-app/1.0", METADATA, "api=table"), "new-app/2.0", + userAgent("new-app/2.0", "api:table")}, + {settings(NAME, "my-app/1.0", USER_AGENT, "custom-agent/2.0"), "new-app/2.0", + userAgent("custom-agent/2.0")}, + }; + } + + @Test(dataProvider = "clientEntriesMetadata") + public void testClientMetadataReplacesClientEntries(String metadata, String expectedComment) { + Map config = ClientConfigProperties.parseConfigMap(settings(METADATA, metadata)); + Assert.assertEquals(new WorkloadIdentitySupport(config).getUserAgent(null, config), + "clickhouse-java-v2/" + Client.clientVersion + " (" + expectedComment + ")"); + } + + @DataProvider(name = "clientEntriesMetadata") + public static Object[][] clientEntriesMetadata() { + return new Object[][]{ + {"os=Android 14", "os:Android 14; " + LANGUAGE_VERSION + "; " + HTTP_CLIENT}, + {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + OS + "; " + HTTP_CLIENT}, + {"hc=okhttp_4.12", "hc:okhttp_4.12; " + LANGUAGE_VERSION + "; " + OS}, + {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + HTTP_CLIENT}, + {"os=Android 14,hc=okhttp_4.12,api=table,lv=kotlin/2.0", + "os:Android 14; hc:okhttp_4.12; api:table; lv:kotlin/2.0"}, + }; + } + + @Test + public void testClientPartComposedPerInstance() { + Map withMetadata = ClientConfigProperties.parseConfigMap(settings(METADATA, "api=table")); + WorkloadIdentitySupport identityWithMetadata = new WorkloadIdentitySupport(withMetadata); + Map withoutMetadata = ClientConfigProperties.parseConfigMap(settings()); + WorkloadIdentitySupport identityWithoutMetadata = new WorkloadIdentitySupport(withoutMetadata); + + Assert.assertEquals(identityWithMetadata.getUserAgent("my-app/1.0", withMetadata), + userAgent("my-app/1.0", "api:table")); + Assert.assertEquals(identityWithoutMetadata.getUserAgent("my-app/1.0", withoutMetadata), + userAgent("my-app/1.0")); + } + + /** + * Returns the value for a request without operation settings. The transport passes the configured + * {@code User-Agent} header as the custom value. + */ + private static String clientUserAgent(WorkloadIdentitySupport identity, Map config) { + return identity.getUserAgent((String) config.get(USER_AGENT), config); + } + + private static Map settings(String... keysAndValues) { + Map settings = new HashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + settings.put(keysAndValues[i], keysAndValues[i + 1]); + } + return settings; + } + + private static String userAgent(String application) { + return userAgent(application, null); + } + + private static String userAgent(String application, String metadata) { + return (application == null ? "" : application + " ") + "clickhouse-java-v2/" + Client.clientVersion + " (" + + (metadata == null ? "" : metadata + "; ") + LANGUAGE_VERSION + "; " + OS + "; " + HTTP_CLIENT + ")"; + } +} diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index b408ed64a..91c32c137 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -263,7 +263,8 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `setClientNetworkBufferSize(int size)` | `size` - size in bytes | Sets size of a buffer in application memory space that is used to copy data between socket and application. | `300000` | `client_network_buffer_size` | | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | -| `setClientName(String clientName)` | `clientName` - application name string | Sets additional information about calling application. Will be passed as `User-Agent` header. | - | `client_name` | +| `setClientName(String clientName)` | `clientName` - application name string | Sets the client name, which identifies the application. Will be passed as the first part of the `User-Agent` header. Use it instead of setting the `User-Agent` header. Does not change the client metadata. | - | `client_name` | +| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os` and `hc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -277,19 +278,34 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. There are two fields in a query log that identify application originated a request: `client_name` and `http_user_agent`. Native TCP protocol uses `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. -The field `http_user_agent` is set according to `User-Agent` header common format: `application-name[/version] [(operating-system; architecture; ...)]`. -This set of values is repeated for each layer: application, client library, http client library. What is set by `setClientName` method comes first in the list. +The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os` and `hc` if the pairs do not set these keys. For example: ```java showLineNumbers -client.setClientName("my-app-01/1.0"); +import com.clickhouse.client.api.Client; + +import java.util.LinkedHashMap; +import java.util.Map; + +public Client createClientWithMetadata() { + Map metadata = new LinkedHashMap<>(); + metadata.put("fv", "flink/2.0.0"); + metadata.put("api", "table"); + + return new Client.Builder() + // ... + .setClientName("my-app-01/1.0", metadata) + .build(); +} ``` will result in the following `http_user_agent` value: ``` -my-app-01/1.0 clickhouse-java-v2/0.9.6-SNAPSHOT (Linux; jvm:17.0.17) Apache-HttpClient/5.4.4 +my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) ``` -Application can set own http header `User-Agent` to identify itself. But part `clickhouse-java-v2/0.9.6-SNAPSHOT` will be appended to the end of the header. +Use `setClientName` to identify the application. Do not set the `User-Agent` header for this: a custom `User-Agent` header replaces +the client name, and the client still appends its own part `clickhouse-java-v2/0.12.0 (...)` to the end of the header. ### Operation Identification {#operation-identification} diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 0d23aae0f..d24a75e97 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -146,10 +146,14 @@ conn.setClientInfo(com.clickhouse.jdbc.ClientInfoProperties.APPLICATION_NAME.get Both ways will result in the following `http_user_agent` value in the query log: ``` -my-app-01/1.0 jdbc-v2/0.9.7 clickhouse-java-v2/0.9.6 (Linux; jvm:17.0.17) Apache-HttpClient/5.4.4 +my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) ``` +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property. For example, +`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4)`. + **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. +Use the client name to identify the application. Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. ### Operation Identification {#operation-identification} diff --git a/docs/features.md b/docs/features.md index a9c6e405a..ca416e24e 100644 --- a/docs/features.md +++ b/docs/features.md @@ -39,6 +39,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with a `null` or empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: @@ -97,7 +98,7 @@ Compatibility-sensitive traits: - Nested columns: Un-flattened `Nested(f1 T1, ..., fN TN)` columns (tables created with `flatten_nested = 0`) are exposed as JDBC `ARRAY` whose element type is `Tuple(f1 T1, ..., fN TN)`. They can be inserted through `Connection#createArrayOf`/`setArray` or `setObject` (a Java array of tuples) and read back through `getArray`/`getObject`; `java.sql.Array#getResultSet()` iterates the nested rows as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple. An `Array(Nested(...))` column is read as a JDBC `ARRAY` whose elements are the nested values, each an array of tuples (`Object[]` of `Object[]`); its `getResultSet()` returns each element as a `java.sql.Array` equal to a top-level `Nested(...)` value. - Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape. For ClickHouse `26.8+` the variant also carries `MultiPoint`. - MultiPoint type mapping: For ClickHouse `26.8+`, JDBC exposes `MultiPoint` as `ARRAY` with type name `MultiPoint`, returns `double[][]` from `getObject()` and a `java.sql.Array` from `getArray()`, and accepts a nested `Array` (`createArrayOf("Array(Point)", ...)`) for prepared-statement inserts, the same as `Ring` and `LineString`. -- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. +- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name. The `client_id_metadata` connection property is forwarded to the client and adds key/value pairs to the `User-Agent` header (see the `client-v2` `User-Agent` entry). - Wrapper support: Implements standard JDBC `Wrapper` and `unwrap` behavior on major JDBC objects. - Packaging and runtime compatibility: Ships as a JDBC 4.2 driver, depends on `client-v2`, and includes native-image metadata for GraalVM users. diff --git a/docs/integration-client.md b/docs/integration-client.md index 7e0e7342b..89219b385 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -92,6 +92,8 @@ In production environments, a single ClickHouse cluster often handles diverse wo Use a structured format such as `/` or `:/` (for example, `order-service/1.2.0` or `etl-worker:cdc/2.0.1`). +Always identify the application with the client name. Do not set the `User-Agent` header (for example, with `httpHeader("User-Agent", ...)`): a custom `User-Agent` header replaces the client name in the header. + **1. Client Builder** (static setup at client creation): ```java @@ -119,12 +121,40 @@ public void updateClientWorkload(Client client, String workloadOrTenant) { #### How it is observed on the server (`User-Agent` header) -The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version, operating system, JVM version, and underlying HTTP client details: +The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version), in the standard ClickHouse client format: + +```text +order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) +``` + +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `hc` if the map does not set these keys (set them to replace the values of the client): + +```java +import com.clickhouse.client.api.Client; + +import java.util.LinkedHashMap; +import java.util.Map; + +public Client createClientWithMetadata() { + Map metadata = new LinkedHashMap<>(); + metadata.put("fv", "flink/2.0.0"); + metadata.put("api", "table"); + + return new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("secret") + .setClientName("Flink-ClickHouse-Sink/0.2.0", metadata) + .build(); +} +``` ```text -order-service/1.2.0 clickhouse-java-v2/0.9.6 (Linux; jvm:17.0.2) Apache-HttpClient/5.4.4 +Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) ``` +The same pairs can be set with the `client_id_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. + > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting Java Client applications. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index bbdd38d30..8188f8883 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -152,12 +152,16 @@ public void executeWorkloadTask(Connection conn, String taskName) throws SQLExce #### How it is observed on the server (`User-Agent` header) -The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. HikariCP), client version, operating system, and JVM version: +The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, detected frameworks (e.g. Apache Spark), the client version, and a comment group with the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version): ```text -order-service/1.2.0 ClickHouse-JDBC/0.9.8 clickhouse-java-v2/0.9.8 (Linux; jvm:17.0.2) Apache-HttpClient/5.4.4 +order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) ``` +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4)`). + +Always identify the application with the client name (`client_name` or `ApplicationName`). Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. + > **CRITICAL SERVER OBSERVATION NOTE:** > In ClickHouse's `system.query_log` and `system.processes`, HTTP requests record this information in the **`http_user_agent`** column. The `client_name` column in `system.query_log` is populated **only** for native TCP protocol connections. Always query `http_user_agent` when troubleshooting JDBC applications. diff --git a/history/latest/3202.md b/history/latest/3202.md new file mode 100644 index 000000000..a2e6b65ff --- /dev/null +++ b/history/latest/3202.md @@ -0,0 +1,11 @@ +- **[client-v2, jdbc-v2]** Breaking change: the `User-Agent` header now uses the standard ClickHouse client format, + ` clickhouse-java-v2/ (lv:java/; os:; hc:ahc_)`: + `jvm:` is now `lv:java/`, the OS is the `os:` entry, and the trailing + `Apache-HttpClient/` is replaced by the `hc:ahc_` entry (the Apache HttpClient version that the + client is built with). Added `Client.Builder#setClientName(String, Map)` (property `client_id_metadata`) to add + key/value pairs to the comment group. The pairs can set `lv`, `os` and `hc` to replace the values of the client. + Leading and trailing whitespace is removed from the client name and from a custom `User-Agent` value. A blank value + set for one operation counts as not set. The client logs a warning if the `User-Agent` value of the client + configuration is longer than 200 characters. + Queries that parse `http_user_agent` in `system.query_log` must use the new format. + (https://github.com/ClickHouse/clickhouse-java/issues/3202) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 782ef8229..93930459c 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -8,6 +8,7 @@ import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.ServerException; import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.internal.Versions; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.data.ClickHouseFormat; import com.fasterxml.jackson.databind.JsonNode; @@ -349,7 +350,7 @@ public void setAndGetClientInfoTest(String clientName) throws Exception { } } - private void executeQueryAndVerifyUserAgent(Statement stmt, String clientName) throws Exception { + private String executeQueryAndVerifyUserAgent(Statement stmt, String clientName) throws Exception { final String testQuery = "SELECT '" + UUID.randomUUID() + "'"; stmt.execute(testQuery); String queryId = ((StatementImpl)stmt).getLastQueryId(); @@ -375,6 +376,46 @@ private void executeQueryAndVerifyUserAgent(Statement stmt, String clientName) t } Assert.assertTrue(userAgent.contains(Client.CLIENT_USER_AGENT), "Expected to contain '" + Client.CLIENT_USER_AGENT + "' but value was '" + userAgent + "'"); Assert.assertTrue(userAgent.contains(Driver.DRIVER_CLIENT_NAME), "Expected to contain '" + Driver.DRIVER_CLIENT_NAME + "' but value was '" + userAgent + "'"); + return userAgent; + } + + @Test(groups = { "integration" }, dataProvider = "clientMetadataInUserAgentDP") + public void testClientMetadataInUserAgent(boolean setInUrl) throws Exception { + if (isCloud()) { + throw new SkipException("Cloud does not support flushing query_log"); + } + + final String metadata = "fv=flink/2.0.0,api=table"; + Properties info = new Properties(); + info.setProperty("user", "default"); + info.setProperty("password", ClickHouseServerForTest.getPassword()); + info.setProperty(ClientConfigProperties.DATABASE.getKey(), ClickHouseServerForTest.getDatabase()); + String url = getEndpointString(); + if (setInUrl) { + url += "?" + ClientConfigProperties.CLIENT_ID_METADATA.getKey() + "=" + metadata; + } else { + info.setProperty(ClientConfigProperties.CLIENT_ID_METADATA.getKey(), metadata); + } + + String expectedSuffix = "clickhouse-java-v2/" + Client.clientVersion + " (fv:flink/2.0.0; api:table; lv:java/" + + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; hc:ahc_" + + Versions.APACHE_HTTP_CLIENT + ")"; + try (Connection localConnection = new ConnectionImpl(url, info); + Statement stmt = localConnection.createStatement()) { + String userAgent = executeQueryAndVerifyUserAgent(stmt, null); + Assert.assertTrue(userAgent.endsWith(expectedSuffix), + "Expected to end with '" + expectedSuffix + "' but value was '" + userAgent + "'"); + + localConnection.setClientInfo(ClientInfoProperties.APPLICATION_NAME.getKey(), "my-app/1.0"); + userAgent = executeQueryAndVerifyUserAgent(stmt, "my-app/1.0"); + Assert.assertTrue(userAgent.endsWith(expectedSuffix), + "Expected to end with '" + expectedSuffix + "' but value was '" + userAgent + "'"); + } + } + + @DataProvider(name = "clientMetadataInUserAgentDP") + public static Object[][] clientMetadataInUserAgentDP() { + return new Object[][] {{false}, {true}}; } @Test(groups = { "integration" }) diff --git a/pom.xml b/pom.xml index b70722404..521a55d18 100644 --- a/pom.xml +++ b/pom.xml @@ -139,6 +139,7 @@ 3.3.1 1.6.13 3.5.3 + 3.0.0 3.2.0 2.18.0 3.3.1 @@ -473,6 +474,11 @@ build-helper-maven-plugin ${helper-plugin.version} + + org.codehaus.mojo + templating-maven-plugin + ${templating-plugin.version} + org.codehaus.mojo flatten-maven-plugin