[client-v2] Send the User-Agent in the standard client format and add client metadata

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -1004,6 +1004,8 @@ public Builder allowBinaryReaderToReuseBuffers(boolean reuse) {
          * Defines list of headers that should be sent with each request. The Client will use a header value
          * defined in {@code headers} instead of any other.
          * Operation settings may override these headers.
+         * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify
+         * the application, use {@link #setClientName(String)} instead of the {@code User-Agent} header.
          *
          * @see InsertSettings#httpHeaders(Map)
          * @see QuerySettings#httpHeaders(Map)
@@ -1139,20 +1141,54 @@ public Builder useHTTPBasicAuth(boolean useBasicAuth) {
         }
 
         /**
-         * Sets additional information about calling application. This string will be passed to server as a client name.
-         * In case of HTTP protocol it will be passed as a {@code User-Agent} header.
-         * Warn: If custom value of User-Agent header is set it will override this value for HTTP transport
-         * Client name is used by server to identify client application when investigating {@code system.query_log}. In case of HTTP
-         * transport this value will be in the {@code system.query_log.http_user_agent} column. Currently only HTTP transport is used.
+         * Sets the client name, which identifies the application to the server. The client sends it in the
+         * {@code User-Agent} header, before its own part {@code clickhouse-java-v2/<version> (...)}. The server records
+         * the header in the {@code http_user_agent} column of {@code system.query_log}.
+         * <p>
+         * To identify the application, set the client name. Do not set the {@code User-Agent} header with
+         * {@link #httpHeader(String, String)}: a custom {@code User-Agent} header replaces the client name.
+         * <p>
+         * This method does not change the client metadata. To also add key/value pairs to the {@code User-Agent}
+         * header, use {@link #setClientName(String, Map)}.
          *
-         * @param clientName - client application display name.
+         * @param clientName - client application name, for example {@code my-app/1.0}
          * @return same instance of the builder
          */
         public Builder setClientName(String clientName) {
             this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), clientName);
             return this;
         }
 
+        /**
+         * Sets the client name and the client metadata, which identify the application to the server. See
+         * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the
+         * {@code User-Agent} header, in the iteration order of {@code metadata}, before its own {@code lv} and
+         * {@code os} entries. For example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0}
+         * and {@code api=table} give
+         * {@code my-app/1.0 clickhouse-java-v2/<version> (fv:flink/2.0.0; api:table; lv:java/<java.version>; os:<os.name>)}.
+         * <p>
+         * Keys and values are trimmed, and entries with an empty key or value are ignored. Keys and values must be
+         * printable ASCII, a key must not contain spaces or any of {@code :;(),=}, a value must not contain any of
+         * {@code ;()}, and the keys {@code lv} and {@code os} are reserved; otherwise {@link #build()} throws
+         * {@link ClientMisconfigurationException}. The metadata is a client-level setting: it is kept when the client
+         * name changes with {@link Client#updateClientName(String)}.
+         *
+         * @param clientName - client application name, for example {@code my-app/1.0}
+         * @param metadata - key/value pairs; {@code null} or empty removes the previously set metadata
+         * @return same instance of the builder
+         * @see ClientConfigProperties#CLIENT_METADATA
+         */
+        public Builder setClientName(String clientName, Map<String, String> metadata) {
+            setClientName(clientName);
+            if (metadata == null || metadata.isEmpty()) {
+                this.configuration.remove(ClientConfigProperties.CLIENT_METADATA.getKey());
+            } else {
+                this.configuration.put(ClientConfigProperties.CLIENT_METADATA.getKey(),
+                        ClientConfigProperties.fromKeyValuePairs(metadata));
+            }
+            return this;
+        }
+
         /**
          * Sets client options from provided map. Values are copied as is
          * @param options - map of client options

client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java

@@ -4,6 +4,7 @@
 import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader;
 import com.clickhouse.client.api.enums.SSLMode;
 import com.clickhouse.client.api.internal.CompressedBlockOutputStream;
+import com.clickhouse.client.api.internal.WorkloadIdentitySupport;
 import com.clickhouse.data.ClickHouseDataType;
 import com.clickhouse.data.ClickHouseFormat;
 import org.slf4j.Logger;
@@ -257,6 +258,24 @@ public Object parseValue(String value) {
                     .collect(Collectors.toList());
         }
     },
+
+    /**
+     * Key/value pairs, written as {@code key1=value1,key2=value2}, that the client adds to the comment group of the
+     * {@code User-Agent} header. A {@code ,} or {@code \} in a value is escaped with {@code \}. This is a client-level
+     * setting: it is not read from operation settings. See {@link Client.Builder#setClientName(String, Map)} for the
+     * format and the rules for keys and values.
+     */
+    CLIENT_METADATA("client_metadata", Map.class) {
+        @Override
+        @SuppressWarnings("unchecked")
+        public Object parseValue(String value) {
+            Map<String, String> metadata = (Map<String, String>) super.parseValue(value);
+            if (metadata != null) {
+                metadata.forEach(WorkloadIdentitySupport::checkClientMetadataEntry);
+            }
+            return metadata;
+        }
+    },
     ;
 
     private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class);
@@ -506,6 +525,36 @@ public static Map<String, String> toKeyValuePairs(String str) {
 
 
 
+    /**
+     * Inverse of {@link #toKeyValuePairs(String)}: escapes {@code \}, {@code ,} and {@code =} with {@code \} and
+     * skips entries with a {@code null} key or value.
+     */
+    static String fromKeyValuePairs(Map<String, String> map) {
+        StringBuilder sb = new StringBuilder();
+        for (Map.Entry<String, String> entry : map.entrySet()) {
+            if (entry.getKey() == null || entry.getValue() == null) {
+                continue;
+            }
+            if (sb.length() > 0) {
+                sb.append(',');
+            }
+            appendEscaped(sb, entry.getKey()).append('=');
+            appendEscaped(sb, entry.getValue());
+        }
+        return sb.toString();
+    }
+
+    private static StringBuilder appendEscaped(StringBuilder sb, String str) {
+        for (int i = 0; i < str.length(); i++) {
+            char ch = str.charAt(i);
+            if (ch == '\\' || ch == ',' || ch == '=') {
+                sb.append('\\');
+            }
+            sb.append(ch);
+        }
+        return sb;
+    }
+
     public static String mapToString(Map<?,?> map, Function<Object, String> valueConverter) {
         StringBuilder sb = new StringBuilder();
         for (Map.Entry<?, ?> entry : map.entrySet()) {

client-v2/src/main/java/com/clickhouse/client/api/internal/CommonSettings.java

@@ -178,6 +178,8 @@ public String getDatabase() {
     /**
      * Defines list of headers that should be sent with current request. The Client will use a header value
      * defined in {@code headers} instead of any other.
+     * The {@code User-Agent} header is different: the client adds its own part to the end of the value. To identify
+     * the application, use {@link Client.Builder#setClientName(String)} instead of the {@code User-Agent} header.
      *
      * @see Client.Builder#httpHeaders(Map)
      * @param key - header name.

client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java

@@ -1,7 +1,6 @@
 package com.clickhouse.client.api.internal;
 
 import com.clickhouse.client.api.ClickHouseException;
-import com.clickhouse.client.api.Client;
 import com.clickhouse.client.api.ClientConfigProperties;
 import com.clickhouse.client.api.ClientException;
 import com.clickhouse.client.api.ClientFaultCause;
@@ -105,7 +104,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
@@ -135,7 +133,7 @@ public class HttpAPIClientHelper {
 
     private final Set<ClientFaultCause> defaultRetryCauses;
 
-    private final String defaultUserAgent;
+    private final WorkloadIdentitySupport workloadIdentitySupport;
     private final Object metricsRegistry;
 
     ConnPoolControl<?> poolControl;
@@ -171,7 +169,7 @@ public HttpAPIClientHelper(Map<String, Object> configuration, Object metricsRegi
             defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None);
         }
 
-        this.defaultUserAgent = buildDefaultUserAgent();
+        this.workloadIdentitySupport = new WorkloadIdentitySupport(configuration);
     }
 
     /**
@@ -1194,58 +1192,8 @@ private void correctUserAgentHeader(HttpRequest request, Map<String, Object> req
         Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT);
         request.removeHeaders(HttpHeaders.USER_AGENT);
 
-        String clientName = ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig);
-        String userAgentValue = defaultUserAgent;
-        if (userAgentHeader == null && clientName != null && !clientName.isEmpty()) {
-            userAgentValue = clientName + " " + defaultUserAgent;
-        } else if (userAgentHeader != null) {
-            userAgentValue = userAgentHeader.getValue() + " " + defaultUserAgent;
-        }
-        request.setHeader(HttpHeaders.USER_AGENT, userAgentValue);
-    }
-
-    private String buildDefaultUserAgent() {
-        StringBuilder userAgent = new StringBuilder();
-        userAgent.append(Client.CLIENT_USER_AGENT);
-
-        String clientVersion = Client.clientVersion;
-
-        userAgent.append(clientVersion);
-
-        userAgent.append(" (");
-        userAgent.append(System.getProperty("os.name"));
-        userAgent.append("; ");
-        userAgent.append("jvm:").append(System.getProperty("java.version"));
-        userAgent.append("; ");
-
-        userAgent.setLength(userAgent.length() - 2);
-        userAgent.append(')');
-
-        try {
-            String httpClientVersion = this.httpClient.getClass().getPackage().getImplementationVersion();
-            if (Objects.equals(this.httpClient.getClass().getPackage().getImplementationTitle(), this.getClass().getPackage().getImplementationTitle())) {
-                // shaded jar - all packages have same implementation title
-                httpClientVersion = "unknown";
-                try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("client-v2-version.properties")) {
-                    Properties p = new Properties();
-                    p.load(in);
-
-                    String tmp = p.getProperty("apache.http.client.version");
-                    if (tmp != null && !tmp.isEmpty() && !tmp.equals("${apache.httpclient.version}")) {
-                        httpClientVersion = tmp;
-                    }
-                } catch (Exception e) {
-                    LOG.debug("Failed to read HTTP client version from client-v2-version.properties", e);
-                }
-            }
-            userAgent.append(" ")
-                    .append("Apache-HttpClient")
-                    .append('/')
-                    .append(httpClientVersion);
-        } catch (Exception e) {
-            LOG.warn("failed to construct http client version string", e);
-        }
-        return userAgent.toString();
+        request.setHeader(HttpHeaders.USER_AGENT, workloadIdentitySupport.getUserAgent(
+                userAgentHeader == null ? null : userAgentHeader.getValue(), requestConfig));
     }
 
     public void close() {

client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java

@@ -0,0 +1,95 @@
+package com.clickhouse.client.api.internal;
+
+import com.clickhouse.client.api.Client;
+import com.clickhouse.client.api.ClientConfigProperties;
+import com.clickhouse.client.api.ClientMisconfigurationException;
+
+import java.util.Map;
+
+/**
+ * Builds the identification of the client and of the application that uses it. The identification is the value of
+ * the {@code User-Agent} header, in the standard ClickHouse client format:
+ * {@code <application> clickhouse-java-v2/<version> (<key>:<value>; ...; lv:java/<java.version>; os:<os.name>)}.
+ * <p>
+ * The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), and the key/value pairs are
+ * the client metadata ({@link ClientConfigProperties#CLIENT_METADATA}). A transport sends the value of
+ * {@link #getUserAgent(String, Map)} with every request, so the identification is the same for every transport.
+ */
+public class WorkloadIdentitySupport {
+
+    private static final String RESERVED_KEY_LANGUAGE_VERSION = "lv";
+
+    private static final String RESERVED_KEY_OS = "os";
+
+    private final String clientUserAgent;
+
+    /**
+     * @param configuration - parsed client configuration
+     */
+    public WorkloadIdentitySupport(Map<String, Object> configuration) {
+        this.clientUserAgent = buildClientUserAgent(ClientConfigProperties.CLIENT_METADATA.getOrDefault(configuration));
+    }
+
+    /**
+     * Returns the value of the {@code User-Agent} header for a request: the application part, then the client part.
+     * The application part is {@code customUserAgent} if the application set its own {@code User-Agent} header, and
+     * the client name otherwise. A blank value counts as not set, and a blank application part is left out.
+     *
+     * @param customUserAgent - value of the {@code User-Agent} header that the application set, or {@code null}
+     * @param requestConfig - settings of the request
+     * @return value of the {@code User-Agent} header
+     */
+    public String getUserAgent(String customUserAgent, Map<String, Object> requestConfig) {
+        String application = isBlank(customUserAgent)
+                ? ClientConfigProperties.CLIENT_NAME.<String>getOrDefault(requestConfig) : customUserAgent;
+        return isBlank(application) ? clientUserAgent : application + " " + clientUserAgent;
+    }
+
+    private static boolean isBlank(String str) {
+        return str == null || str.trim().isEmpty();
+    }
+
+    /**
+     * Checks that a client metadata entry does not break the format of the {@code User-Agent} header.
+     *
+     * @param key - trimmed, not empty key
+     * @param value - trimmed, not empty value
+     * @throws ClientMisconfigurationException if the key is reserved, or the key or the value contains a character
+     *         that is not allowed
+     */
+    public static void checkClientMetadataEntry(String key, String value) {
+        if (key.equalsIgnoreCase(RESERVED_KEY_LANGUAGE_VERSION) || key.equalsIgnoreCase(RESERVED_KEY_OS)) {
+            throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey()
+                    + "' entry '" + key + "': the key is reserved by the client");
+        }
+        if (!isClientMetadataText(key, " :;(),=") || !isClientMetadataText(value, ";()")) {
+            throw new ClientMisconfigurationException("Invalid '" + ClientConfigProperties.CLIENT_METADATA.getKey()
+                    + "' entry '" + key + "=" + value + "': keys and values must be printable ASCII, a key must not"
+                    + " contain spaces or any of ':;(),=', and a value must not contain any of ';()'");
+        }
+    }
+
+    private static boolean isClientMetadataText(String str, String forbiddenChars) {
+        for (int i = 0; i < str.length(); i++) {
+            char ch = str.charAt(i);
+            if (ch < 0x20 || ch > 0x7E || forbiddenChars.indexOf(ch) >= 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private static String buildClientUserAgent(Map<String, String> clientMetadata) {
+        StringBuilder userAgent = new StringBuilder();
+        userAgent.append(Client.CLIENT_USER_AGENT).append(Client.clientVersion).append(" (");
+        if (clientMetadata != null) {
+            for (Map.Entry<String, String> entry : clientMetadata.entrySet()) {
+                userAgent.append(entry.getKey()).append(':').append(entry.getValue()).append("; ");
+            }
+        }
+        userAgent.append(RESERVED_KEY_LANGUAGE_VERSION).append(":java/").append(System.getProperty("java.version"))
+                .append("; ").append(RESERVED_KEY_OS).append(':').append(System.getProperty("os.name"))
+                .append(')');
+        return userAgent.toString();
+    }
+}

client-v2/src/main/resources/client-v2-version.properties

@@ -1,3 +1,2 @@
 version=${revision}
-build.date=${build_timestamp}
-apache.http.client.version=${apache.httpclient.version}
+build.date=${build_timestamp}

client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java

@@ -1201,9 +1201,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA
     @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider")
     public static Object[][] testUserAgentHasCompleteProductName_dataProvider() {
         return new Object[][] {
-                { "", Pattern.compile("clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$") },
-                { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")},
-                { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2\\/.+ \\(.+\\) Apache-HttpClient\\/[\\d\\.]+$")}};
+                { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)") },
+                { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")},
+                { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+\\)")}};
     }
 
     @Test(dataProvider = "testClientNameDataProvider")