From a6cbd8f7c265f9a00a5aed0a38050c6b1171ea13 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:10:43 +0000 Subject: [PATCH 1/4] feat(jdbc-v2): read DatabaseMetaData with SHOW / DESCRIBE statements getSchemas(), getTables() and getColumns() now use SHOW DATABASES, SHOW FULL TABLES and DESCRIBE TABLE instead of system.databases, system.tables and system.columns. SHOW TABLES FROM also lists the tables that the server hides from system.tables by default (tables of DataLakeCatalog and remote databases). The new driver property jdbc_metadata_use_show_statements (default true) selects the implementation; false keeps the system table queries. The result sets have the same columns and values, except that getTables() returns null in REMARKS and TYPE_SCHEM. getColumns() skips a table that fails with UNKNOWN_TABLE, UNKNOWN_DATABASE, ACCESS_DENIED or DATALAKE_DATABASE_ERROR. Implements: https://github.com/ClickHouse/clickhouse-java/issues/2907 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 + docs/features.md | 2 + docs/integration-jdbc.md | 1 + .../com/clickhouse/jdbc/DriverProperties.java | 11 + .../jdbc/internal/DetachedResultSet.java | 16 + .../jdbc/metadata/DatabaseMetaDataImpl.java | 324 ++++++++++++++++++ .../DatabaseMetaDataImplLikeMatcherTest.java | 43 +++ .../jdbc/metadata/DatabaseMetaDataTest.java | 194 +++++++++++ 8 files changed, 605 insertions(+) create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e558e56a..6d39da8b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,11 @@ - **[r2dbc]** Pre `1.0.0` is not supported anymore because R2DBC API reached stable `1.0.0` version. +- **[jdbc-v2]** `DatabaseMetaData#getTables` now returns `null` in `REMARKS` (table comment) and `TYPE_SCHEM` by + default, because metadata is read with `SHOW` statements (see the `jdbc_metadata_use_show_statements` entry in New + Features). Set `jdbc_metadata_use_show_statements=false` to get the table comments. + (https://github.com/ClickHouse/clickhouse-java/issues/2907) + ### New Features - **[migration-helpers]** Added `migration-helpers` module containing `ConfigurationMigrationHelper` and @@ -168,6 +173,15 @@ as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple. (https://github.com/ClickHouse/clickhouse-java/issues/2477) - **[client-v2, jdbc-v2]** Added logging on previously-silent error and diagnostic paths (no functional or public-API change). (https://github.com/ClickHouse/clickhouse-java/issues/2969) +- **[jdbc-v2]** `DatabaseMetaData#getSchemas`, `#getTables` and `#getColumns` now read metadata with `SHOW DATABASES`, + `SHOW TABLES` and `DESCRIBE TABLE` instead of the `system.databases`, `system.tables` and `system.columns` tables. + The server does not show some tables in the system tables by default (for example, tables of `DataLakeCatalog` + databases), so tools that use `DatabaseMetaData` did not find them. The new driver property + `jdbc_metadata_use_show_statements` (default `true`) selects the implementation; set it to `false` to use the system + tables. With the `SHOW` statements, `getTables()` returns `null` in `REMARKS` (table comment) and `TYPE_SCHEM`; + `getColumns()` sends one `DESCRIBE TABLE` query for each table that matches and skips a table that was dropped + meanwhile, that the user cannot describe, or whose data lake metadata cannot be read. All other columns and values + are the same. (https://github.com/ClickHouse/clickhouse-java/issues/2907) ### Bug Fixes diff --git a/docs/features.md b/docs/features.md index a41f5f936..80fd29e4d 100644 --- a/docs/features.md +++ b/docs/features.md @@ -88,6 +88,7 @@ Compatibility-sensitive traits: - Binary string reads: `ResultSet#getBytes(int|String)` and `ResultSet#getBinaryStream(int|String)` return the raw bytes of a `String`/`FixedString` column. Combined with the `binary_string_support` connection property, non-UTF-8/binary content stored in `String` columns round-trips byte-for-byte; `NULL` values report `null` with `wasNull()` set. `ResultSet#getObject(...)` never exposes the internal `StringValue` holder for these columns: `getObject(column, byte[].class)` returns the raw bytes, while `getObject(column, Object.class)` and the no-type `getObject(column)` overloads return a decoded `String`. - Result-set metadata: Exposes JDBC `ResultSetMetaData` backed by ClickHouse column schema. - Database metadata: Implements JDBC `DatabaseMetaData` for ClickHouse catalogs, schemas, tables, columns, and related capability reporting. +- Metadata with `SHOW` statements: By default `DatabaseMetaData#getSchemas`, `#getTables` and `#getColumns` use `SHOW DATABASES`, `SHOW FULL TABLES` and `DESCRIBE TABLE`, so they also return tables that the server does not show in the system tables by default (for example, tables of `DataLakeCatalog` databases). The driver property `jdbc_metadata_use_show_statements=false` selects the `system.databases`, `system.tables` and `system.columns` queries. - Parameter metadata: Reports prepared-statement parameter counts. - Type mapping and conversions: Maps ClickHouse types to JDBC types and Java classes, including date/time handling and `java.time` support. - QBit type mapping: For ClickHouse `25.10+`, JDBC exposes the experimental `QBit(element_type, dimension)` type as `ARRAY`, returning the vector as a `java.sql.Array` of the element type from `getObject()`/`getArray()`. Supported element types are `BFloat16` and `Float32` (both `java.lang.Float`) and `Float64` (`java.lang.Double`). The `allow_experimental_qbit_type` server setting is required to create a `QBit` column. @@ -103,6 +104,7 @@ Compatibility-sensitive traits: Compatibility-sensitive traits: - Prepared statements are client-side SQL rendering, not server-side prepared statements. Changes to literal encoding or placeholder parsing are externally visible behavior. +- `DatabaseMetaData#getSchemas`, `#getTables` and `#getColumns` return the same result-set columns and values with and without `jdbc_metadata_use_show_statements`, with two exceptions for the `SHOW` statements: `getTables()` returns `null` in `REMARKS` and `TYPE_SCHEM`, because `SHOW TABLES` returns no table comment and no database engine. `getColumns()` with the `SHOW` statements sends one `DESCRIBE TABLE` query for each table that matches, and skips a database or table that fails with `UNKNOWN_TABLE`, `UNKNOWN_DATABASE`, `ACCESS_DENIED` or `DATALAKE_DATABASE_ERROR` (for example, a table with column-level grants only, which `system.columns` shows with the granted columns). - JDBC `cluster_name` is compatibility-sensitive for cancellation behavior: when configured, `Statement.cancel()` issues `KILL QUERY ON CLUSTER ''`, and when omitted it falls back to a local `KILL QUERY`. - String parameters are escaped with backslash-based escaping: backslashes are doubled and single quotes are backslash-escaped before values are wrapped in single quotes. - `?` placeholder detection is SQL-aware and should not treat question marks inside quoted strings, quoted identifiers, comments, casts, or similar syntax as bind parameters. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index 32348c8ed..bbdd38d30 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -833,6 +833,7 @@ Key JDBC-specific properties (see [`DriverProperties`](../jdbc-v2/src/main/java/ | `jdbc_type_mappings` | — | Custom ClickHouse → Java type overrides | | `default_query_settings` | — | Default settings for all queries | | `jdbc_metrics_recorder` | — | Custom `MetricsRecorder` implementation class name | +| `jdbc_metadata_use_show_statements` | `true` | `DatabaseMetaData` schemas, tables and columns via `SHOW` / `DESCRIBE` instead of `system.*` tables | ## Observability & Monitoring diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index db4963fee..4b27626d0 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -160,6 +160,17 @@ public enum DriverProperties { */ METRICS_RECORDER("jdbc_metrics_recorder", null), + /** + * Makes {@link java.sql.DatabaseMetaData#getSchemas()}, {@link java.sql.DatabaseMetaData#getSchemas(String, String)}, + * {@link java.sql.DatabaseMetaData#getTables} and {@link java.sql.DatabaseMetaData#getColumns} read metadata with + * {@code SHOW DATABASES}, {@code SHOW TABLES} and {@code DESCRIBE TABLE} instead of {@code system.databases}, + * {@code system.tables} and {@code system.columns}. These statements also return tables that the system tables + * hide by default, for example tables of {@code DataLakeCatalog} databases. With these statements + * {@code getTables()} returns {@code null} in {@code REMARKS} and {@code TYPE_SCHEM}. + * Default: true + */ + METADATA_USE_SHOW_STATEMENTS("jdbc_metadata_use_show_statements", String.valueOf(Boolean.TRUE)), + ; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/DetachedResultSet.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/DetachedResultSet.java index 5c13569b1..5141210ff 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/DetachedResultSet.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/DetachedResultSet.java @@ -1,7 +1,9 @@ package com.clickhouse.jdbc.internal; +import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.jdbc.JdbcV2Wrapper; import com.clickhouse.jdbc.ResultSetImpl; +import com.clickhouse.jdbc.metadata.ResultSetMetaDataImpl; import java.io.InputStream; import java.io.Reader; @@ -97,6 +99,20 @@ public static DetachedResultSet createFromResultSet(ResultSet resultSet, Calenda return new DetachedResultSet(records, metaData, defaultCalendar); } + /** + * Creates a result set from records built by the driver. + * + * @param records - rows, each maps a column name to its value + * @param columns - columns of the result set + * @param defaultCalendar - calendar used for date and time values + * @return result set with the records + * @throws SQLException - if the result set cannot be created + */ + public static DetachedResultSet createFromRecords(List> records, List columns, + Calendar defaultCalendar) throws SQLException { + return new DetachedResultSet(records, new ResultSetMetaDataImpl(columns, "", "", "", null, null), defaultCalendar); + } + @Override public boolean next() throws SQLException { ensureOpen(); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java index 84bb1886c..27595d023 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java @@ -21,6 +21,7 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.RowIdLifetime; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -31,10 +32,16 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wrapper { @@ -78,6 +85,8 @@ public String getTypeName() { private String jdbcUrl; + private final boolean useShowStatements; + /** * Creates an instance of DatabaseMetaData for the given connection. * @@ -93,6 +102,7 @@ public DatabaseMetaDataImpl(ConnectionImpl connection, boolean useCatalogs, Stri this.useCatalogs = useCatalogs; this.catalogPlaceholder = useCatalogs ? "'local' " : "''"; this.jdbcUrl = url; + this.useShowStatements = connection.getJdbcConfig().isFlagSet(DriverProperties.METADATA_USE_SHOW_STATEMENTS); } private Statement createStatement() throws SQLException { @@ -977,6 +987,9 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam // Get engines that map to the requested table types Set requestedTypes = (types == null || types.length == 0) ? TABLE_TYPES : Arrays.stream(types).collect(Collectors.toSet()) ; + if (useShowStatements) { + return getTablesWithShowStatements(schemaPattern, tableNamePattern, requestedTypes); + } Set engines = getEnginesForTableTypes(requestedTypes); // Build engine filter conditions @@ -1048,6 +1061,9 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam @Override public ResultSet getSchemas() throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name + if (useShowStatements) { + return getSchemasWithShowStatements(null); + } try { return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases ORDER BY name"); } catch (Exception e) { @@ -1093,6 +1109,9 @@ public ResultSet getTableTypes() throws SQLException { @SuppressWarnings({"squid:S2095", "squid:S2077"}) public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name + if (useShowStatements) { + return getColumnsWithShowStatements(schemaPattern, tableNamePattern, columnNamePattern); + } final String sql = "SELECT " + catalogPlaceholder + " AS TABLE_CAT, " + "database AS TABLE_SCHEM, " + @@ -1163,6 +1182,308 @@ private static String generateSqlTypeSizes(String columnName) { private static final List>> GET_COLUMNS_RS_MUTATORS = Collections.singletonList(DATA_TYPE_VALUE_FUNCTION); + // Result sets of the methods that use SHOW statements have the same columns as the result sets of the system table + // queries. Only REMARKS and TYPE_SCHEM of getTables() are nullable: SHOW TABLES returns no table comment and no + // database engine. + private static final List GET_SCHEMAS_COLUMNS = ClickHouseColumn.parse( + "TABLE_SCHEM String, TABLE_CATALOG String"); + + private static final List GET_TABLES_COLUMNS = ClickHouseColumn.parse( + "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, TABLE_TYPE String, REMARKS Nullable(String), " + + "TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME Nullable(String), " + + "SELF_REFERENCING_COL_NAME Nullable(String), REF_GENERATION Nullable(String)"); + + private static final List GET_COLUMNS_COLUMNS = ClickHouseColumn.parse( + "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, COLUMN_NAME String, DATA_TYPE Int32, " + + "TYPE_NAME String, COLUMN_SIZE Nullable(Int32), BUFFER_LENGTH Int32, DECIMAL_DIGITS Nullable(Int32), " + + "NUM_PREC_RADIX Nullable(Int32), NULLABLE Int32, REMARKS String, COLUMN_DEF String, SQL_DATA_TYPE Int32, " + + "SQL_DATETIME_SUB Int32, CHAR_OCTET_LENGTH Nullable(Int32), ORDINAL_POSITION Int32, IS_NULLABLE String, " + + "SCOPE_CATALOG Nullable(String), SCOPE_SCHEMA Nullable(String), SCOPE_TABLE Nullable(String), " + + "SOURCE_DATA_TYPE Nullable(Int16), IS_AUTOINCREMENT String, IS_GENERATEDCOLUMN String"); + + private static final Set INTEGER_TYPES = EnumSet.of(ClickHouseDataType.Bool, + ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, + ClickHouseDataType.Int128, ClickHouseDataType.Int256, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, + ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.UInt128, ClickHouseDataType.UInt256); + + private static final Set DECIMAL_TYPES = EnumSet.of(ClickHouseDataType.Decimal, + ClickHouseDataType.Decimal32, ClickHouseDataType.Decimal64, ClickHouseDataType.Decimal128, + ClickHouseDataType.Decimal256); + + private static final Map TYPE_NAME_TO_BYTE_LENGTH = Arrays.stream(ClickHouseDataType.values()) + .filter(type -> type.getByteLength() > 0) + .collect(Collectors.toMap(ClickHouseDataType::name, ClickHouseDataType::getByteLength)); + + // UNKNOWN_TABLE, UNKNOWN_DATABASE, ACCESS_DENIED, DATALAKE_DATABASE_ERROR + private static final Set SKIPPED_OBJECT_ERROR_CODES = new HashSet<>(Arrays.asList(60, 81, 497, 736)); + + // DESCRIBE indents named tuple elements on new lines when print_pretty_type_names is enabled. The setting is not + // changed in the query because readonly=1 users cannot change settings. + private static final Pattern PRETTY_TYPE_OPENING = Pattern.compile("\\(\n *"); + private static final Pattern PRETTY_TYPE_SEPARATOR = Pattern.compile(",\n *"); + + private ResultSet getSchemasWithShowStatements(String schemaPattern) throws SQLException { + List> records = new ArrayList<>(); + try { + for (String database : showDatabases(schemaPattern)) { + Map record = new HashMap<>(); + record.put("TABLE_SCHEM", database); + record.put("TABLE_CATALOG", ""); + records.add(record); + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_SCHEMAS_COLUMNS, connection.getDefaultCalendar()); + } + + private ResultSet getTablesWithShowStatements(String schemaPattern, String tableNamePattern, + Set requestedTypes) throws SQLException { + // Like the system.tables query: temporary tables are not returned (they have no row in system.databases), and + // when no known table type is requested, the types do not filter the tables. + boolean filterByType = requestedTypes.stream().anyMatch(TABLE_TYPES::contains); + List> records = new ArrayList<>(); + try { + for (String[] table : showTables(schemaPattern, tableNamePattern, false)) { + Map record = new HashMap<>(); + record.put("TABLE_CAT", ""); + record.put("TABLE_SCHEM", table[0]); + record.put("TABLE_NAME", table[1]); + record.put(TABLE_TYPE_COL_IN_GET_TABLES, table[2]); + record.put("REMARKS", null); + record.put("TYPE_CAT", null); + record.put("TYPE_SCHEM", null); + record.put("TYPE_NAME", null); + record.put("SELF_REFERENCING_COL_NAME", null); + record.put("REF_GENERATION", null); + TABLE_TYPE_MUTATOR.accept(record); + if (!filterByType || requestedTypes.contains(record.get(TABLE_TYPE_COL_IN_GET_TABLES))) { + records.add(record); + } + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_TABLES_COLUMNS, connection.getDefaultCalendar()); + } + + private ResultSet getColumnsWithShowStatements(String schemaPattern, String tableNamePattern, + String columnNamePattern) throws SQLException { + Predicate columnNameMatcher = likeMatcher(columnNamePattern); + List> records = new ArrayList<>(); + try (Statement stmt = createStatement()) { + for (String[] table : showTables(schemaPattern, tableNamePattern, true)) { + String tableRef = table[0].isEmpty() ? quoteIdentifier(table[1]) + : quoteIdentifier(table[0]) + "." + quoteIdentifier(table[1]); + List> tableRecords = new ArrayList<>(); + try (ResultSet rs = stmt.executeQuery("DESCRIBE TABLE " + tableRef)) { + Set describeColumns = new HashSet<>(); + ResultSetMetaData describeMetaData = rs.getMetaData(); + for (int i = 1; i <= describeMetaData.getColumnCount(); i++) { + describeColumns.add(describeMetaData.getColumnLabel(i)); + } + int position = 0; + while (rs.next()) { + if ((describeColumns.contains("is_subcolumn") && rs.getInt("is_subcolumn") != 0) + || (describeColumns.contains("is_virtual") && rs.getInt("is_virtual") != 0)) { + continue; + } + position++; + String columnName = rs.getString("name"); + if (columnNameMatcher.test(columnName)) { + tableRecords.add(createColumnRecord(table[0], table[1], columnName, position, + rs.getString("type"), + describeColumns.contains("default_expression") ? rs.getString("default_expression") : "", + describeColumns.contains("comment") ? rs.getString("comment") : "")); + } + } + } catch (SQLException e) { + if (!isSkippedObjectError(e)) { + throw e; + } + log.debug("Skipped metadata of table {}", tableRef, e); + continue; + } + records.addAll(tableRecords); + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_COLUMNS_COLUMNS, connection.getDefaultCalendar()); + } + + private List showDatabases(String schemaPattern) throws SQLException { + List databases = new ArrayList<>(); + if (schemaPattern != null && schemaPattern.isEmpty()) { + return databases; // SHOW ignores an empty LIKE pattern, but it matches only an empty name + } + try (Statement stmt = createStatement(); + ResultSet rs = stmt.executeQuery("SHOW DATABASES LIKE " + likeLiteral(schemaPattern))) { + while (rs.next()) { + databases.add(rs.getString(1)); + } + } + return databases; + } + + /** + * Returns {database, name, engine} of the tables that match the patterns. Temporary tables have an empty database, + * the same as in system.tables, and are returned only when {@code includeTemporary} is set. + */ + private List showTables(String schemaPattern, String tableNamePattern, boolean includeTemporary) + throws SQLException { + List tables = new ArrayList<>(); + if (tableNamePattern != null && tableNamePattern.isEmpty()) { + return tables; // SHOW ignores an empty LIKE pattern, but it matches only an empty name + } + String tableNameLike = " LIKE " + likeLiteral(tableNamePattern); + try (Statement stmt = createStatement()) { + if (includeTemporary && likeMatcher(schemaPattern).test("")) { + tables.addAll(showTables(stmt, "", "SHOW FULL TEMPORARY TABLES" + tableNameLike)); + } + for (String database : showDatabases(schemaPattern)) { + try { + tables.addAll(showTables(stmt, database, + "SHOW FULL TABLES FROM " + quoteIdentifier(database) + tableNameLike)); + } catch (SQLException e) { + if (!isSkippedObjectError(e)) { + throw e; + } + log.debug("Skipped metadata of database {}", database, e); + } + } + } + return tables; + } + + private static List showTables(Statement stmt, String database, String sql) throws SQLException { + List tables = new ArrayList<>(); + try (ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + tables.add(new String[]{database, rs.getString("name"), rs.getString("engine")}); + } + } + return tables; + } + + private static Map createColumnRecord(String database, String table, String column, int position, + String describedType, String defaultExpression, + String comment) { + String type = PRETTY_TYPE_SEPARATOR.matcher(PRETTY_TYPE_OPENING.matcher(describedType).replaceAll("(")) + .replaceAll(", "); + Map record = new HashMap<>(); + record.put("TABLE_CAT", ""); + record.put("TABLE_SCHEM", database); + record.put("TABLE_NAME", table); + record.put("COLUMN_NAME", column); + record.put("TYPE_NAME", type); + putColumnSizes(record, type); + record.put("BUFFER_LENGTH", 0); + record.put("NULLABLE", type.contains("Nullable(") ? typeNullable : typeNoNulls); + record.put("REMARKS", comment); + record.put("COLUMN_DEF", defaultExpression); + record.put("SQL_DATA_TYPE", 0); + record.put("SQL_DATETIME_SUB", 0); + record.put("ORDINAL_POSITION", position); + record.put("IS_NULLABLE", type.toUpperCase(Locale.ROOT).contains("NULLABLE") ? "YES" : "NO"); + record.put("SCOPE_CATALOG", null); + record.put("SCOPE_SCHEMA", null); + record.put("SCOPE_TABLE", null); + record.put("SOURCE_DATA_TYPE", null); + record.put("IS_AUTOINCREMENT", "NO"); + record.put("IS_GENERATEDCOLUMN", "NO"); + DATA_TYPE_VALUE_FUNCTION.accept(record); + return record; + } + + /** + * Sets COLUMN_SIZE, DECIMAL_DIGITS, NUM_PREC_RADIX and CHAR_OCTET_LENGTH with the values that the system.columns + * query returns. system.columns unwraps only Nullable (and the custom name of SimpleAggregateFunction), and reports + * the bit width of integers, the precision and scale of decimals and the length of FixedString. + */ + private static void putColumnSizes(Map record, String type) { + Integer precision = null; + Integer radix = null; + Integer scale = null; + Integer octetLength = null; + try { + ClickHouseColumn column = ClickHouseColumn.of("", type); + if (column.getDataType() == ClickHouseDataType.SimpleAggregateFunction) { + column = column.getNestedColumns().get(0); + } + ClickHouseDataType dataType = column.getDataType(); + if (!column.isLowCardinality()) { + if (dataType == ClickHouseDataType.FixedString) { + octetLength = column.getPrecision(); + } else if (INTEGER_TYPES.contains(dataType)) { + precision = dataType.getByteLength() * Byte.SIZE; + radix = 2; + scale = 0; + } else if (DECIMAL_TYPES.contains(dataType)) { + precision = column.getPrecision(); + radix = 10; + scale = column.getScale(); + } + } + } catch (Exception e) { + log.debug("Failed to parse column type: {}", type, e); + } + Integer byteLength = TYPE_NAME_TO_BYTE_LENGTH.get(type); + record.put("COLUMN_SIZE", octetLength != null ? octetLength + : byteLength != null ? byteLength : precision != null ? precision : 0); + record.put("DECIMAL_DIGITS", scale == null || scale == 0 ? null : scale); + record.put("NUM_PREC_RADIX", radix); + record.put("CHAR_OCTET_LENGTH", octetLength); + } + + // Escaped like a PreparedStatement string parameter, so '\' reaches LIKE as the escape character. + private static String likeLiteral(String pattern) { + return "'" + SQLUtils.escapeSingleQuotes(pattern == null ? "%" : pattern) + "'"; + } + + private static String quoteIdentifier(String name) { + return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; + } + + /** + * Returns true for errors of a single database or table that must not fail the whole metadata call: the object + * was dropped after it was listed, the user can list it but cannot describe it (for example, with column-level + * grants), or a data lake catalog cannot read its metadata. + */ + private static boolean isSkippedObjectError(SQLException e) { + return SKIPPED_OBJECT_ERROR_CODES.contains(e.getErrorCode()); + } + + /** + * Returns a matcher with the semantics of the ClickHouse LIKE operator: {@code %} matches any sequence of + * characters, {@code _} matches one character and {@code \} escapes the next character. A {@code null} pattern + * matches everything. + */ + static Predicate likeMatcher(String pattern) { + if (pattern == null) { + return value -> true; + } + StringBuilder regex = new StringBuilder(); + StringBuilder literal = new StringBuilder(); + for (int i = 0; i < pattern.length(); i++) { + char c = pattern.charAt(i); + if (c == '%' || c == '_') { + if (literal.length() > 0) { + regex.append(Pattern.quote(literal.toString())); + literal.setLength(0); + } + regex.append(c == '%' ? ".*" : "."); + } else { + literal.append(c == '\\' && i + 1 < pattern.length() ? pattern.charAt(++i) : c); + } + } + if (literal.length() > 0) { + regex.append(Pattern.quote(literal.toString())); + } + Pattern compiled = Pattern.compile(regex.toString(), Pattern.DOTALL); + return value -> compiled.matcher(value).matches(); + } + @Override public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) throws SQLException { //Return an empty result set with the required columns @@ -1796,6 +2117,9 @@ public RowIdLifetime getRowIdLifetime() throws SQLException { @Override public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name + if (useShowStatements) { + return getSchemasWithShowStatements(schemaPattern); + } try { return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases " + "WHERE name LIKE '" + (schemaPattern == null ? "%" : schemaPattern) + "'"); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java new file mode 100644 index 000000000..0e6d7b51b --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java @@ -0,0 +1,43 @@ +package com.clickhouse.jdbc.metadata; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class DatabaseMetaDataImplLikeMatcherTest { + + @DataProvider(name = "likePatterns") + public Object[][] likePatterns() { + return new Object[][] { + {null, "any_name", true}, + {null, "", true}, + {"", "", true}, + {"", "a", false}, + {"%", "", true}, + {"%", "a\nb", true}, + {"abc", "abc", true}, + {"abc", "ABC", false}, + {"abc", "abcd", false}, + {"a_c", "abc", true}, + {"a_c", "ac", false}, + {"a%c", "ac", true}, + {"a%c", "a.b.c", true}, + {"a%c", "acd", false}, + {"a\\_c", "a_c", true}, + {"a\\_c", "abc", false}, + {"100\\%", "100%", true}, + {"100\\%", "1000", false}, + {"a\\\\b", "a\\b", true}, + {"a.c", "abc", false}, + {"(x)[y]*", "(x)[y]*", true}, + {"_", "😀", true}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "likePatterns") + public void testLikeMatcher(String pattern, String value, boolean expected) { + assertEquals(DatabaseMetaDataImpl.likeMatcher(pattern).test(value), expected, + "'" + value + "' LIKE '" + pattern + "'"); + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index 08dd650ae..e8432a83d 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -10,6 +10,7 @@ import com.clickhouse.jdbc.internal.JdbcUtils; import org.testng.Assert; import org.testng.SkipException; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.ByteArrayOutputStream; @@ -25,10 +26,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static java.sql.RowIdLifetime.ROWID_UNSUPPORTED; @@ -624,6 +627,197 @@ public void testGetSchemas() throws Exception { } } + private interface MetadataCall { + ResultSet call(DatabaseMetaData metaData) throws SQLException; + } + + private static final String SHOW_STATEMENTS_TABLE = "metadata_show_statements"; + + private static final String SHOW_STATEMENTS_ODD_TABLE = SHOW_STATEMENTS_TABLE + " odd.'\"\\`name"; + + private static String quoteIdentifier(String name) { + return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; + } + + @DataProvider(name = "metadataCallsWithShowStatements") + public Object[][] metadataCallsWithShowStatements() { + final String db = getDatabase(); + final String table = SHOW_STATEMENTS_TABLE; + final String tables = SHOW_STATEMENTS_TABLE + "%"; + return new Object[][] { + {(MetadataCall) md -> md.getColumns(null, db, table, null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, "d%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, "%\\_t%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, tables, null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db + "%", "metadata\\_show\\_statements%", null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, ""), false, false}, + {(MetadataCall) md -> md.getTables(null, db, tables, null), true, true}, + {(MetadataCall) md -> md.getTables(null, db + "%", "metadata\\_show\\_statements%", null), true, true}, + {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"VIEW", "MATERIALIZED VIEW"}), true, true}, + {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"UNKNOWN"}), true, true}, + {(MetadataCall) md -> md.getTables(null, "system", "%", new String[]{"SYSTEM TABLE"}), true, true}, + {(MetadataCall) md -> md.getTables(null, db, "", null), true, false}, + {(MetadataCall) md -> md.getSchemas(null, db), false, true}, + {(MetadataCall) md -> md.getSchemas(null, "syst%"), false, true}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "metadataCallsWithShowStatements") + public void testShowStatementsReturnSameMetadataAsSystemTables(MetadataCall call, boolean isGetTables, + boolean expectRows) throws Exception { + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_mv"); + stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_view"); + stmt.executeUpdate("DROP TABLE IF EXISTS " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("DROP TABLE IF EXISTS " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE)); + stmt.executeUpdate("CREATE TABLE " + SHOW_STATEMENTS_TABLE + " (" + + "id Int32 COMMENT 'row id', u8 UInt8, i128 Int128, u256 UInt256, b Bool, nb Nullable(Bool), " + + "f32 Float32, d Decimal(10, 2) DEFAULT 1.5, d0 Nullable(Decimal(5, 0)), d64 Decimal64(4), " + + "s String, fs FixedString(4), nfs Nullable(FixedString(3)), lfs LowCardinality(FixedString(3)), " + + "ls LowCardinality(Nullable(String)), e Enum8('a' = 1), dt DateTime64(3, 'UTC'), dd Date, " + + "arr Array(Nullable(Int8)), m Map(String, Tuple(x Int32, y String)), " + + "nested_tuple Tuple(a Tuple(b Int32, c Nullable(String)), e UInt8), j JSON(a.b UInt32), " + + "sa SimpleAggregateFunction(sum, UInt64), " + + "nsa SimpleAggregateFunction(anyLast, Nullable(Decimal(9, 3))), " + + "mat Int64 MATERIALIZED id * 2, al String ALIAS toString(id), eph Int32 EPHEMERAL" + + ") ENGINE = MergeTree ORDER BY id COMMENT 'table comment'"); + stmt.executeUpdate("CREATE VIEW " + SHOW_STATEMENTS_TABLE + "_view AS SELECT id, d FROM " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("CREATE MATERIALIZED VIEW " + SHOW_STATEMENTS_TABLE + "_mv ENGINE = MergeTree ORDER BY id " + + "AS SELECT id, s FROM " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("CREATE TABLE " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE) + " (id Int32, " + + quoteIdentifier("odd \"col\"\\") + " String) ENGINE = MergeTree ORDER BY id"); + } + + Properties systemTablesProps = new Properties(); + systemTablesProps.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), "false"); + try (Connection showConn = getJdbcConnection(); Connection systemConn = getJdbcConnection(systemTablesProps); + ResultSet showRs = call.call(showConn.getMetaData()); + ResultSet systemRs = call.call(systemConn.getMetaData())) { + Set ignoredColumns = isGetTables + ? new HashSet<>(Arrays.asList("REMARKS", "TYPE_SCHEM")) : Collections.emptySet(); + + assertEquals(columnsOf(showRs.getMetaData(), ignoredColumns), columnsOf(systemRs.getMetaData(), ignoredColumns)); + List> showRows = rowsOf(showRs, ignoredColumns); + List> systemRows = rowsOf(systemRs, ignoredColumns); + assertEquals(!systemRows.isEmpty(), expectRows); + if (isGetTables) { + showRows.sort(Comparator.comparing(Object::toString)); + systemRows.sort(Comparator.comparing(Object::toString)); + } + assertEquals(showRows, systemRows); + } + } + + private static List columnsOf(ResultSetMetaData metaData, Set ignoredColumns) throws SQLException { + List columns = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String label = metaData.getColumnLabel(i); + columns.add(label + ":" + metaData.getColumnType(i) + (ignoredColumns.contains(label) ? "" + : ":" + metaData.getColumnTypeName(i) + ":" + metaData.isNullable(i))); + } + return columns; + } + + private static List> rowsOf(ResultSet rs, Set ignoredColumns) throws SQLException { + ResultSetMetaData metaData = rs.getMetaData(); + List> rows = new ArrayList<>(); + while (rs.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + if (!ignoredColumns.contains(metaData.getColumnLabel(i))) { + row.add(rs.getObject(i)); + } + } + rows.add(row); + } + return rows; + } + + @DataProvider(name = "showStatementsFlagValues") + public Object[][] showStatementsFlagValues() { + return new Object[][] { + {null, null}, + {"true", null}, + {"false", "flag table comment"}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "showStatementsFlagValues") + public void testTableRemarksDependOnShowStatementsFlag(String flag, String expectedRemarks) throws Exception { + final String tableName = "metadata_show_statements_flag"; + Properties props = new Properties(); + if (flag != null) { + props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + } + try (Connection conn = getJdbcConnection(props)) { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); + stmt.executeUpdate("CREATE TABLE " + tableName + " (id Int32) ENGINE = MergeTree ORDER BY id " + + "COMMENT 'flag table comment'"); + } + try (ResultSet rs = conn.getMetaData().getTables(null, getDatabase(), tableName, null)) { + assertTrue(rs.next()); + assertEquals(rs.getString("TABLE_NAME"), tableName); + assertEquals(rs.getString("TABLE_TYPE"), "TABLE"); + assertEquals(rs.getString("REMARKS"), expectedRemarks); + assertFalse(rs.next()); + } + } + } + + @DataProvider(name = "showStatementsFlagForRestrictedUser") + public Object[][] showStatementsFlagForRestrictedUser() { + return new Object[][] { + {"true", Arrays.asList("metadata_show_statements_granted.id:Int32", + "metadata_show_statements_granted.t:Tuple(a Int32, b String)")}, + {"false", Arrays.asList("metadata_show_statements_granted.id:Int32", + "metadata_show_statements_granted.t:Tuple(a Int32, b String)", + "metadata_show_statements_restricted.id:Int32")}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "showStatementsFlagForRestrictedUser") + public void testGetColumnsForReadonlyUserWithColumnGrants(String flag, List expectedColumns) throws Exception { + if (isCloud()) { + throw new SkipException("Should not be tested in cloud because creates users"); + } + final String db = getDatabase(); + final String user = "metadata_show_statements_user"; + final String password = UUID.randomUUID() + "Aa1!"; + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_granted"); + stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_restricted"); + stmt.executeUpdate("CREATE TABLE metadata_show_statements_granted (id Int32, t Tuple(a Int32, b String)) " + + "ENGINE = MergeTree ORDER BY id"); + stmt.executeUpdate("CREATE TABLE metadata_show_statements_restricted (id Int32, secret String) " + + "ENGINE = MergeTree ORDER BY id"); + stmt.executeUpdate("DROP USER IF EXISTS " + user); + stmt.executeUpdate("CREATE USER " + user + " IDENTIFIED WITH plaintext_password BY '" + password + + "' SETTINGS readonly = 1"); + stmt.executeUpdate("GRANT SELECT ON " + db + ".metadata_show_statements_granted TO " + user); + stmt.executeUpdate("GRANT SELECT(id) ON " + db + ".metadata_show_statements_restricted TO " + user); + } + try { + Properties props = new Properties(); + props.setProperty("user", user); + props.setProperty("password", password); + props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + List columns = new ArrayList<>(); + try (Connection conn = getJdbcConnection(props); + ResultSet rs = conn.getMetaData().getColumns(null, db, "metadata_show_statements_%ed", null)) { + while (rs.next()) { + columns.add(rs.getString("TABLE_NAME") + "." + rs.getString("COLUMN_NAME") + ":" + + rs.getString("TYPE_NAME")); + } + } + assertEquals(columns, expectedColumns); + } finally { + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP USER IF EXISTS " + user); + } + } + } + @Test public void testSchemaTerm() throws Exception { From 64013a901bf41fe9bc3fcc081af6fee395cf652d Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:48:39 +0000 Subject: [PATCH 2/4] Address review feedback: pin SHOW metadata order and escaping - Document that the SHOW path returns rows in the JDBC order: the server sorts SHOW DATABASES / SHOW TABLES by name and DESCRIBE returns the columns in table order. Add a test for the order of getSchemas() with the flag on and off. - Mark the two methods that build SHOW / DESCRIBE queries with @SuppressWarnings("squid:S2077"), as getColumns() does: the queries contain only values escaped by likeLiteral() and quoteIdentifier(). Add parity cases with quotes in the patterns, including "%' OR '1'='1". Co-Authored-By: Claude Opus 5.5 --- .../jdbc/metadata/DatabaseMetaDataImpl.java | 7 ++++++ .../jdbc/metadata/DatabaseMetaDataTest.java | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java index 27595d023..e3314c45c 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java @@ -1267,6 +1267,10 @@ private ResultSet getTablesWithShowStatements(String schemaPattern, String table return DetachedResultSet.createFromRecords(records, GET_TABLES_COLUMNS, connection.getDefaultCalendar()); } + // The rows are in the JDBC order (TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION) without a sort: the server sorts the + // result of SHOW DATABASES and SHOW TABLES by name, and DESCRIBE returns the columns in table order. + // The query contains only identifiers escaped by quoteIdentifier(). + @SuppressWarnings({"squid:S2077"}) private ResultSet getColumnsWithShowStatements(String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { Predicate columnNameMatcher = likeMatcher(columnNamePattern); @@ -1312,6 +1316,9 @@ private ResultSet getColumnsWithShowStatements(String schemaPattern, String tabl return DetachedResultSet.createFromRecords(records, GET_COLUMNS_COLUMNS, connection.getDefaultCalendar()); } + // Returns the names sorted by name (the server sorts the result of SHOW DATABASES). The query contains only the + // pattern escaped by likeLiteral(). + @SuppressWarnings({"squid:S2077"}) private List showDatabases(String schemaPattern) throws SQLException { List databases = new ArrayList<>(); if (schemaPattern != null && schemaPattern.isEmpty()) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index e8432a83d..f05fb06f8 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -651,6 +651,9 @@ public Object[][] metadataCallsWithShowStatements() { {(MetadataCall) md -> md.getColumns(null, db, tables, null), false, true}, {(MetadataCall) md -> md.getColumns(null, db + "%", "metadata\\_show\\_statements%", null), false, true}, {(MetadataCall) md -> md.getColumns(null, db, table, ""), false, false}, + {(MetadataCall) md -> md.getColumns(null, db, "metadata\\_show\\_statements odd.'%", "odd \"col\"%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, "%' OR '1'='1", "%", null), false, false}, + {(MetadataCall) md -> md.getTables(null, db, "metadata\\_show\\_statements odd.'%", null), true, true}, {(MetadataCall) md -> md.getTables(null, db, tables, null), true, true}, {(MetadataCall) md -> md.getTables(null, db + "%", "metadata\\_show\\_statements%", null), true, true}, {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"VIEW", "MATERIALIZED VIEW"}), true, true}, @@ -765,6 +768,27 @@ public void testTableRemarksDependOnShowStatementsFlag(String flag, String expec } } + @DataProvider(name = "showStatementsFlags") + public Object[][] showStatementsFlags() { + return new Object[][] {{"true"}, {"false"}}; + } + + @Test(groups = {"integration"}, dataProvider = "showStatementsFlags") + public void testGetSchemasOrderedBySchemaName(String flag) throws Exception { + Properties props = new Properties(); + props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + List schemas = new ArrayList<>(); + try (Connection conn = getJdbcConnection(props); ResultSet rs = conn.getMetaData().getSchemas()) { + while (rs.next()) { + schemas.add(rs.getString("TABLE_SCHEM")); + } + } + assertTrue(schemas.containsAll(Arrays.asList(getDatabase(), "system")), schemas.toString()); + List sortedSchemas = new ArrayList<>(schemas); + Collections.sort(sortedSchemas); + assertEquals(schemas, sortedSchemas); + } + @DataProvider(name = "showStatementsFlagForRestrictedUser") public Object[][] showStatementsFlagForRestrictedUser() { return new Object[][] { From f1a948ed0e75e08e5de380fe2afea8ca45f2d64e Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Wed, 30 Sep 2026 02:55:51 +0000 Subject: [PATCH 3/4] Address review feedback: move SHOW metadata to ShowStatementsMetaData Move the SHOW DATABASES / SHOW TABLES / DESCRIBE TABLE logic out of DatabaseMetaDataImpl into the internal class ShowStatementsMetaData. DatabaseMetaDataImpl creates one instance and forks in getSchemas(), getSchemas(String, String), getTables() and getColumns(): the flag selects ShowStatementsMetaData or the unchanged system table query, which is now in a private *Impl method. The SHOW statements are prepared statements. The LIKE patterns are parameters. The server accepts only an identifier after FROM and DESCRIBE TABLE, so database and table names stay escaped identifiers. Implements: https://github.com/ClickHouse/clickhouse-java/issues/2907 Co-Authored-By: Claude Opus 5.5 --- .../jdbc/metadata/DatabaseMetaDataImpl.java | 361 ++--------------- .../jdbc/metadata/ShowStatementsMetaData.java | 381 ++++++++++++++++++ .../jdbc/metadata/DatabaseMetaDataTest.java | 4 +- ...howStatementsMetaDataLikeMatcherTest.java} | 4 +- 4 files changed, 421 insertions(+), 329 deletions(-) create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java rename jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/{DatabaseMetaDataImplLikeMatcherTest.java => ShowStatementsMetaDataLikeMatcherTest.java} (90%) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java index e3314c45c..525c1e8af 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java @@ -21,7 +21,6 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.RowIdLifetime; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -32,16 +31,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.regex.Pattern; import java.util.stream.Collectors; public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wrapper { @@ -87,6 +80,8 @@ public String getTypeName() { private final boolean useShowStatements; + private final ShowStatementsMetaData showStatementsMetaData; + /** * Creates an instance of DatabaseMetaData for the given connection. * @@ -103,6 +98,7 @@ public DatabaseMetaDataImpl(ConnectionImpl connection, boolean useCatalogs, Stri this.catalogPlaceholder = useCatalogs ? "'local' " : "''"; this.jdbcUrl = url; this.useShowStatements = connection.getJdbcConfig().isFlagSet(DriverProperties.METADATA_USE_SHOW_STATEMENTS); + this.showStatementsMetaData = new ShowStatementsMetaData(connection); } private Statement createStatement() throws SQLException { @@ -114,6 +110,10 @@ private Statement createStatement() throws SQLException { } private PreparedStatement prepareStatement(String sql) throws SQLException { + return prepareStatement(connection, sql); + } + + static PreparedStatement prepareStatement(ConnectionImpl connection, String sql) throws SQLException { PreparedStatement stmt = connection.prepareStatement(sql); if (stmt instanceof StatementImpl) { ((StatementImpl) stmt).getLocalSettings().setFormat(ClickHouseFormat.RowBinaryWithNamesAndTypes); @@ -953,9 +953,9 @@ public static boolean isSystemTableEngine(String engine) { return engine != null && (engine.startsWith("System") || engine.startsWith("Async")); } - private static final String TABLE_TYPE_COL_IN_GET_TABLES = "TABLE_TYPE"; + static final String TABLE_TYPE_COL_IN_GET_TABLES = "TABLE_TYPE"; - private static final Consumer> TABLE_TYPE_MUTATOR = row -> { + static final Consumer> TABLE_TYPE_MUTATOR = row -> { String engine = (String) row.get(TABLE_TYPE_COL_IN_GET_TABLES); String tableType; @@ -985,11 +985,16 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam // TODO: when switch between catalog and schema is implemented, then TABLE_SCHEMA and TABLE_CAT should be populated accordingly // TODO: handle useCatalogs == true and return schema catalog name - // Get engines that map to the requested table types Set requestedTypes = (types == null || types.length == 0) ? TABLE_TYPES : Arrays.stream(types).collect(Collectors.toSet()) ; if (useShowStatements) { - return getTablesWithShowStatements(schemaPattern, tableNamePattern, requestedTypes); + return showStatementsMetaData.getTables(schemaPattern, tableNamePattern, requestedTypes); + } else { + return getTablesImpl(schemaPattern, tableNamePattern, requestedTypes); } + } + + private ResultSet getTablesImpl(String schemaPattern, String tableNamePattern, Set requestedTypes) throws SQLException { + // Get engines that map to the requested table types Set engines = getEnginesForTableTypes(requestedTypes); // Build engine filter conditions @@ -1062,8 +1067,13 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam public ResultSet getSchemas() throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name if (useShowStatements) { - return getSchemasWithShowStatements(null); + return showStatementsMetaData.getSchemas(null); + } else { + return getSchemasImpl(); } + } + + private ResultSet getSchemasImpl() throws SQLException { try { return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases ORDER BY name"); } catch (Exception e) { @@ -1106,12 +1116,17 @@ public ResultSet getTableTypes() throws SQLException { } @Override - @SuppressWarnings({"squid:S2095", "squid:S2077"}) public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name if (useShowStatements) { - return getColumnsWithShowStatements(schemaPattern, tableNamePattern, columnNamePattern); + return showStatementsMetaData.getColumns(schemaPattern, tableNamePattern, columnNamePattern); + } else { + return getColumnsImpl(schemaPattern, tableNamePattern, columnNamePattern); } + } + + @SuppressWarnings({"squid:S2095", "squid:S2077"}) + private ResultSet getColumnsImpl(String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { final String sql = "SELECT " + catalogPlaceholder + " AS TABLE_CAT, " + "database AS TABLE_SCHEM, " + @@ -1163,7 +1178,7 @@ private static String generateSqlTypeSizes(String columnName) { } - private static final Consumer> DATA_TYPE_VALUE_FUNCTION = row -> { + static final Consumer> DATA_TYPE_VALUE_FUNCTION = row -> { String typeName = (String) row.get("TYPE_NAME"); SQLType type = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(typeName); if (type == null) { @@ -1182,315 +1197,6 @@ private static String generateSqlTypeSizes(String columnName) { private static final List>> GET_COLUMNS_RS_MUTATORS = Collections.singletonList(DATA_TYPE_VALUE_FUNCTION); - // Result sets of the methods that use SHOW statements have the same columns as the result sets of the system table - // queries. Only REMARKS and TYPE_SCHEM of getTables() are nullable: SHOW TABLES returns no table comment and no - // database engine. - private static final List GET_SCHEMAS_COLUMNS = ClickHouseColumn.parse( - "TABLE_SCHEM String, TABLE_CATALOG String"); - - private static final List GET_TABLES_COLUMNS = ClickHouseColumn.parse( - "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, TABLE_TYPE String, REMARKS Nullable(String), " + - "TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME Nullable(String), " + - "SELF_REFERENCING_COL_NAME Nullable(String), REF_GENERATION Nullable(String)"); - - private static final List GET_COLUMNS_COLUMNS = ClickHouseColumn.parse( - "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, COLUMN_NAME String, DATA_TYPE Int32, " + - "TYPE_NAME String, COLUMN_SIZE Nullable(Int32), BUFFER_LENGTH Int32, DECIMAL_DIGITS Nullable(Int32), " + - "NUM_PREC_RADIX Nullable(Int32), NULLABLE Int32, REMARKS String, COLUMN_DEF String, SQL_DATA_TYPE Int32, " + - "SQL_DATETIME_SUB Int32, CHAR_OCTET_LENGTH Nullable(Int32), ORDINAL_POSITION Int32, IS_NULLABLE String, " + - "SCOPE_CATALOG Nullable(String), SCOPE_SCHEMA Nullable(String), SCOPE_TABLE Nullable(String), " + - "SOURCE_DATA_TYPE Nullable(Int16), IS_AUTOINCREMENT String, IS_GENERATEDCOLUMN String"); - - private static final Set INTEGER_TYPES = EnumSet.of(ClickHouseDataType.Bool, - ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, - ClickHouseDataType.Int128, ClickHouseDataType.Int256, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, - ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.UInt128, ClickHouseDataType.UInt256); - - private static final Set DECIMAL_TYPES = EnumSet.of(ClickHouseDataType.Decimal, - ClickHouseDataType.Decimal32, ClickHouseDataType.Decimal64, ClickHouseDataType.Decimal128, - ClickHouseDataType.Decimal256); - - private static final Map TYPE_NAME_TO_BYTE_LENGTH = Arrays.stream(ClickHouseDataType.values()) - .filter(type -> type.getByteLength() > 0) - .collect(Collectors.toMap(ClickHouseDataType::name, ClickHouseDataType::getByteLength)); - - // UNKNOWN_TABLE, UNKNOWN_DATABASE, ACCESS_DENIED, DATALAKE_DATABASE_ERROR - private static final Set SKIPPED_OBJECT_ERROR_CODES = new HashSet<>(Arrays.asList(60, 81, 497, 736)); - - // DESCRIBE indents named tuple elements on new lines when print_pretty_type_names is enabled. The setting is not - // changed in the query because readonly=1 users cannot change settings. - private static final Pattern PRETTY_TYPE_OPENING = Pattern.compile("\\(\n *"); - private static final Pattern PRETTY_TYPE_SEPARATOR = Pattern.compile(",\n *"); - - private ResultSet getSchemasWithShowStatements(String schemaPattern) throws SQLException { - List> records = new ArrayList<>(); - try { - for (String database : showDatabases(schemaPattern)) { - Map record = new HashMap<>(); - record.put("TABLE_SCHEM", database); - record.put("TABLE_CATALOG", ""); - records.add(record); - } - } catch (Exception e) { - throw ExceptionUtils.toSqlState(e); - } - return DetachedResultSet.createFromRecords(records, GET_SCHEMAS_COLUMNS, connection.getDefaultCalendar()); - } - - private ResultSet getTablesWithShowStatements(String schemaPattern, String tableNamePattern, - Set requestedTypes) throws SQLException { - // Like the system.tables query: temporary tables are not returned (they have no row in system.databases), and - // when no known table type is requested, the types do not filter the tables. - boolean filterByType = requestedTypes.stream().anyMatch(TABLE_TYPES::contains); - List> records = new ArrayList<>(); - try { - for (String[] table : showTables(schemaPattern, tableNamePattern, false)) { - Map record = new HashMap<>(); - record.put("TABLE_CAT", ""); - record.put("TABLE_SCHEM", table[0]); - record.put("TABLE_NAME", table[1]); - record.put(TABLE_TYPE_COL_IN_GET_TABLES, table[2]); - record.put("REMARKS", null); - record.put("TYPE_CAT", null); - record.put("TYPE_SCHEM", null); - record.put("TYPE_NAME", null); - record.put("SELF_REFERENCING_COL_NAME", null); - record.put("REF_GENERATION", null); - TABLE_TYPE_MUTATOR.accept(record); - if (!filterByType || requestedTypes.contains(record.get(TABLE_TYPE_COL_IN_GET_TABLES))) { - records.add(record); - } - } - } catch (Exception e) { - throw ExceptionUtils.toSqlState(e); - } - return DetachedResultSet.createFromRecords(records, GET_TABLES_COLUMNS, connection.getDefaultCalendar()); - } - - // The rows are in the JDBC order (TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION) without a sort: the server sorts the - // result of SHOW DATABASES and SHOW TABLES by name, and DESCRIBE returns the columns in table order. - // The query contains only identifiers escaped by quoteIdentifier(). - @SuppressWarnings({"squid:S2077"}) - private ResultSet getColumnsWithShowStatements(String schemaPattern, String tableNamePattern, - String columnNamePattern) throws SQLException { - Predicate columnNameMatcher = likeMatcher(columnNamePattern); - List> records = new ArrayList<>(); - try (Statement stmt = createStatement()) { - for (String[] table : showTables(schemaPattern, tableNamePattern, true)) { - String tableRef = table[0].isEmpty() ? quoteIdentifier(table[1]) - : quoteIdentifier(table[0]) + "." + quoteIdentifier(table[1]); - List> tableRecords = new ArrayList<>(); - try (ResultSet rs = stmt.executeQuery("DESCRIBE TABLE " + tableRef)) { - Set describeColumns = new HashSet<>(); - ResultSetMetaData describeMetaData = rs.getMetaData(); - for (int i = 1; i <= describeMetaData.getColumnCount(); i++) { - describeColumns.add(describeMetaData.getColumnLabel(i)); - } - int position = 0; - while (rs.next()) { - if ((describeColumns.contains("is_subcolumn") && rs.getInt("is_subcolumn") != 0) - || (describeColumns.contains("is_virtual") && rs.getInt("is_virtual") != 0)) { - continue; - } - position++; - String columnName = rs.getString("name"); - if (columnNameMatcher.test(columnName)) { - tableRecords.add(createColumnRecord(table[0], table[1], columnName, position, - rs.getString("type"), - describeColumns.contains("default_expression") ? rs.getString("default_expression") : "", - describeColumns.contains("comment") ? rs.getString("comment") : "")); - } - } - } catch (SQLException e) { - if (!isSkippedObjectError(e)) { - throw e; - } - log.debug("Skipped metadata of table {}", tableRef, e); - continue; - } - records.addAll(tableRecords); - } - } catch (Exception e) { - throw ExceptionUtils.toSqlState(e); - } - return DetachedResultSet.createFromRecords(records, GET_COLUMNS_COLUMNS, connection.getDefaultCalendar()); - } - - // Returns the names sorted by name (the server sorts the result of SHOW DATABASES). The query contains only the - // pattern escaped by likeLiteral(). - @SuppressWarnings({"squid:S2077"}) - private List showDatabases(String schemaPattern) throws SQLException { - List databases = new ArrayList<>(); - if (schemaPattern != null && schemaPattern.isEmpty()) { - return databases; // SHOW ignores an empty LIKE pattern, but it matches only an empty name - } - try (Statement stmt = createStatement(); - ResultSet rs = stmt.executeQuery("SHOW DATABASES LIKE " + likeLiteral(schemaPattern))) { - while (rs.next()) { - databases.add(rs.getString(1)); - } - } - return databases; - } - - /** - * Returns {database, name, engine} of the tables that match the patterns. Temporary tables have an empty database, - * the same as in system.tables, and are returned only when {@code includeTemporary} is set. - */ - private List showTables(String schemaPattern, String tableNamePattern, boolean includeTemporary) - throws SQLException { - List tables = new ArrayList<>(); - if (tableNamePattern != null && tableNamePattern.isEmpty()) { - return tables; // SHOW ignores an empty LIKE pattern, but it matches only an empty name - } - String tableNameLike = " LIKE " + likeLiteral(tableNamePattern); - try (Statement stmt = createStatement()) { - if (includeTemporary && likeMatcher(schemaPattern).test("")) { - tables.addAll(showTables(stmt, "", "SHOW FULL TEMPORARY TABLES" + tableNameLike)); - } - for (String database : showDatabases(schemaPattern)) { - try { - tables.addAll(showTables(stmt, database, - "SHOW FULL TABLES FROM " + quoteIdentifier(database) + tableNameLike)); - } catch (SQLException e) { - if (!isSkippedObjectError(e)) { - throw e; - } - log.debug("Skipped metadata of database {}", database, e); - } - } - } - return tables; - } - - private static List showTables(Statement stmt, String database, String sql) throws SQLException { - List tables = new ArrayList<>(); - try (ResultSet rs = stmt.executeQuery(sql)) { - while (rs.next()) { - tables.add(new String[]{database, rs.getString("name"), rs.getString("engine")}); - } - } - return tables; - } - - private static Map createColumnRecord(String database, String table, String column, int position, - String describedType, String defaultExpression, - String comment) { - String type = PRETTY_TYPE_SEPARATOR.matcher(PRETTY_TYPE_OPENING.matcher(describedType).replaceAll("(")) - .replaceAll(", "); - Map record = new HashMap<>(); - record.put("TABLE_CAT", ""); - record.put("TABLE_SCHEM", database); - record.put("TABLE_NAME", table); - record.put("COLUMN_NAME", column); - record.put("TYPE_NAME", type); - putColumnSizes(record, type); - record.put("BUFFER_LENGTH", 0); - record.put("NULLABLE", type.contains("Nullable(") ? typeNullable : typeNoNulls); - record.put("REMARKS", comment); - record.put("COLUMN_DEF", defaultExpression); - record.put("SQL_DATA_TYPE", 0); - record.put("SQL_DATETIME_SUB", 0); - record.put("ORDINAL_POSITION", position); - record.put("IS_NULLABLE", type.toUpperCase(Locale.ROOT).contains("NULLABLE") ? "YES" : "NO"); - record.put("SCOPE_CATALOG", null); - record.put("SCOPE_SCHEMA", null); - record.put("SCOPE_TABLE", null); - record.put("SOURCE_DATA_TYPE", null); - record.put("IS_AUTOINCREMENT", "NO"); - record.put("IS_GENERATEDCOLUMN", "NO"); - DATA_TYPE_VALUE_FUNCTION.accept(record); - return record; - } - - /** - * Sets COLUMN_SIZE, DECIMAL_DIGITS, NUM_PREC_RADIX and CHAR_OCTET_LENGTH with the values that the system.columns - * query returns. system.columns unwraps only Nullable (and the custom name of SimpleAggregateFunction), and reports - * the bit width of integers, the precision and scale of decimals and the length of FixedString. - */ - private static void putColumnSizes(Map record, String type) { - Integer precision = null; - Integer radix = null; - Integer scale = null; - Integer octetLength = null; - try { - ClickHouseColumn column = ClickHouseColumn.of("", type); - if (column.getDataType() == ClickHouseDataType.SimpleAggregateFunction) { - column = column.getNestedColumns().get(0); - } - ClickHouseDataType dataType = column.getDataType(); - if (!column.isLowCardinality()) { - if (dataType == ClickHouseDataType.FixedString) { - octetLength = column.getPrecision(); - } else if (INTEGER_TYPES.contains(dataType)) { - precision = dataType.getByteLength() * Byte.SIZE; - radix = 2; - scale = 0; - } else if (DECIMAL_TYPES.contains(dataType)) { - precision = column.getPrecision(); - radix = 10; - scale = column.getScale(); - } - } - } catch (Exception e) { - log.debug("Failed to parse column type: {}", type, e); - } - Integer byteLength = TYPE_NAME_TO_BYTE_LENGTH.get(type); - record.put("COLUMN_SIZE", octetLength != null ? octetLength - : byteLength != null ? byteLength : precision != null ? precision : 0); - record.put("DECIMAL_DIGITS", scale == null || scale == 0 ? null : scale); - record.put("NUM_PREC_RADIX", radix); - record.put("CHAR_OCTET_LENGTH", octetLength); - } - - // Escaped like a PreparedStatement string parameter, so '\' reaches LIKE as the escape character. - private static String likeLiteral(String pattern) { - return "'" + SQLUtils.escapeSingleQuotes(pattern == null ? "%" : pattern) + "'"; - } - - private static String quoteIdentifier(String name) { - return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; - } - - /** - * Returns true for errors of a single database or table that must not fail the whole metadata call: the object - * was dropped after it was listed, the user can list it but cannot describe it (for example, with column-level - * grants), or a data lake catalog cannot read its metadata. - */ - private static boolean isSkippedObjectError(SQLException e) { - return SKIPPED_OBJECT_ERROR_CODES.contains(e.getErrorCode()); - } - - /** - * Returns a matcher with the semantics of the ClickHouse LIKE operator: {@code %} matches any sequence of - * characters, {@code _} matches one character and {@code \} escapes the next character. A {@code null} pattern - * matches everything. - */ - static Predicate likeMatcher(String pattern) { - if (pattern == null) { - return value -> true; - } - StringBuilder regex = new StringBuilder(); - StringBuilder literal = new StringBuilder(); - for (int i = 0; i < pattern.length(); i++) { - char c = pattern.charAt(i); - if (c == '%' || c == '_') { - if (literal.length() > 0) { - regex.append(Pattern.quote(literal.toString())); - literal.setLength(0); - } - regex.append(c == '%' ? ".*" : "."); - } else { - literal.append(c == '\\' && i + 1 < pattern.length() ? pattern.charAt(++i) : c); - } - } - if (literal.length() > 0) { - regex.append(Pattern.quote(literal.toString())); - } - Pattern compiled = Pattern.compile(regex.toString(), Pattern.DOTALL); - return value -> compiled.matcher(value).matches(); - } - @Override public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) throws SQLException { //Return an empty result set with the required columns @@ -2125,8 +1831,13 @@ public RowIdLifetime getRowIdLifetime() throws SQLException { public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name if (useShowStatements) { - return getSchemasWithShowStatements(schemaPattern); + return showStatementsMetaData.getSchemas(schemaPattern); + } else { + return getSchemasImpl(schemaPattern); } + } + + private ResultSet getSchemasImpl(String schemaPattern) throws SQLException { try { return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases " + "WHERE name LIKE '" + (schemaPattern == null ? "%" : schemaPattern) + "'"); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java new file mode 100644 index 000000000..a98932561 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java @@ -0,0 +1,381 @@ +package com.clickhouse.jdbc.metadata; + +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.jdbc.ConnectionImpl; +import com.clickhouse.jdbc.internal.DetachedResultSet; +import com.clickhouse.jdbc.internal.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static java.sql.DatabaseMetaData.typeNoNulls; +import static java.sql.DatabaseMetaData.typeNullable; + +/** + * Reads the metadata of {@link DatabaseMetaDataImpl} with {@code SHOW DATABASES}, {@code SHOW TABLES} and + * {@code DESCRIBE TABLE} instead of the system tables. These statements also return the databases and tables that the + * system tables hide by default, for example the tables of {@code DataLakeCatalog} databases. + * Used when {@link com.clickhouse.jdbc.DriverProperties#METADATA_USE_SHOW_STATEMENTS} is set. + *

+ * The result sets have the same columns and values as the result sets of the system table queries, except that + * {@code getTables()} returns {@code null} in {@code REMARKS} and {@code TYPE_SCHEM}: {@code SHOW TABLES} returns no + * table comment and no database engine. + */ +final class ShowStatementsMetaData { + private static final Logger log = LoggerFactory.getLogger(ShowStatementsMetaData.class); + + private static final List GET_SCHEMAS_COLUMNS = ClickHouseColumn.parse( + "TABLE_SCHEM String, TABLE_CATALOG String"); + + private static final List GET_TABLES_COLUMNS = ClickHouseColumn.parse( + "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, TABLE_TYPE String, REMARKS Nullable(String), " + + "TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME Nullable(String), " + + "SELF_REFERENCING_COL_NAME Nullable(String), REF_GENERATION Nullable(String)"); + + private static final List GET_COLUMNS_COLUMNS = ClickHouseColumn.parse( + "TABLE_CAT String, TABLE_SCHEM String, TABLE_NAME String, COLUMN_NAME String, DATA_TYPE Int32, " + + "TYPE_NAME String, COLUMN_SIZE Nullable(Int32), BUFFER_LENGTH Int32, DECIMAL_DIGITS Nullable(Int32), " + + "NUM_PREC_RADIX Nullable(Int32), NULLABLE Int32, REMARKS String, COLUMN_DEF String, SQL_DATA_TYPE Int32, " + + "SQL_DATETIME_SUB Int32, CHAR_OCTET_LENGTH Nullable(Int32), ORDINAL_POSITION Int32, IS_NULLABLE String, " + + "SCOPE_CATALOG Nullable(String), SCOPE_SCHEMA Nullable(String), SCOPE_TABLE Nullable(String), " + + "SOURCE_DATA_TYPE Nullable(Int16), IS_AUTOINCREMENT String, IS_GENERATEDCOLUMN String"); + + private static final Set INTEGER_TYPES = EnumSet.of(ClickHouseDataType.Bool, + ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, + ClickHouseDataType.Int128, ClickHouseDataType.Int256, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, + ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.UInt128, ClickHouseDataType.UInt256); + + private static final Set DECIMAL_TYPES = EnumSet.of(ClickHouseDataType.Decimal, + ClickHouseDataType.Decimal32, ClickHouseDataType.Decimal64, ClickHouseDataType.Decimal128, + ClickHouseDataType.Decimal256); + + private static final Map TYPE_NAME_TO_BYTE_LENGTH = Arrays.stream(ClickHouseDataType.values()) + .filter(type -> type.getByteLength() > 0) + .collect(Collectors.toMap(ClickHouseDataType::name, ClickHouseDataType::getByteLength)); + + // UNKNOWN_TABLE, UNKNOWN_DATABASE, ACCESS_DENIED, DATALAKE_DATABASE_ERROR + private static final Set SKIPPED_OBJECT_ERROR_CODES = new HashSet<>(Arrays.asList(60, 81, 497, 736)); + + // DESCRIBE indents named tuple elements on new lines when print_pretty_type_names is enabled. The setting is not + // changed in the query because readonly=1 users cannot change settings. + private static final Pattern PRETTY_TYPE_OPENING = Pattern.compile("\\(\n *"); + private static final Pattern PRETTY_TYPE_SEPARATOR = Pattern.compile(",\n *"); + + private final ConnectionImpl connection; + + ShowStatementsMetaData(ConnectionImpl connection) { + this.connection = connection; + } + + /** + * Returns the rows of {@link java.sql.DatabaseMetaData#getSchemas(String, String)}. + */ + ResultSet getSchemas(String schemaPattern) throws SQLException { + List> records = new ArrayList<>(); + try { + for (String database : showDatabases(schemaPattern)) { + Map record = new HashMap<>(); + record.put("TABLE_SCHEM", database); + record.put("TABLE_CATALOG", ""); + records.add(record); + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_SCHEMAS_COLUMNS, connection.getDefaultCalendar()); + } + + /** + * Returns the rows of {@link java.sql.DatabaseMetaData#getTables(String, String, String, String[])}. + */ + ResultSet getTables(String schemaPattern, String tableNamePattern, Set requestedTypes) throws SQLException { + // Like the system.tables query: temporary tables are not returned (they have no row in system.databases), and + // when no known table type is requested, the types do not filter the tables. + boolean filterByType = requestedTypes.stream().anyMatch(DatabaseMetaDataImpl.TABLE_TYPES::contains); + List> records = new ArrayList<>(); + try { + for (String[] table : showTables(schemaPattern, tableNamePattern, false)) { + Map record = new HashMap<>(); + record.put("TABLE_CAT", ""); + record.put("TABLE_SCHEM", table[0]); + record.put("TABLE_NAME", table[1]); + record.put(DatabaseMetaDataImpl.TABLE_TYPE_COL_IN_GET_TABLES, table[2]); + record.put("REMARKS", null); + record.put("TYPE_CAT", null); + record.put("TYPE_SCHEM", null); + record.put("TYPE_NAME", null); + record.put("SELF_REFERENCING_COL_NAME", null); + record.put("REF_GENERATION", null); + DatabaseMetaDataImpl.TABLE_TYPE_MUTATOR.accept(record); + if (!filterByType || requestedTypes.contains(record.get(DatabaseMetaDataImpl.TABLE_TYPE_COL_IN_GET_TABLES))) { + records.add(record); + } + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_TABLES_COLUMNS, connection.getDefaultCalendar()); + } + + /** + * Returns the rows of {@link java.sql.DatabaseMetaData#getColumns(String, String, String, String)}. + * The rows are in the JDBC order (TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION) without a sort: the server sorts the + * result of SHOW DATABASES and SHOW TABLES by name, and DESCRIBE returns the columns in table order. + */ + ResultSet getColumns(String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + Predicate columnNameMatcher = likeMatcher(columnNamePattern); + List> records = new ArrayList<>(); + try { + for (String[] table : showTables(schemaPattern, tableNamePattern, true)) { + try { + records.addAll(describeTable(table[0], table[1], columnNameMatcher)); + } catch (SQLException e) { + if (!isSkippedObjectError(e)) { + throw e; + } + log.debug("Skipped metadata of table {}", table[0].isEmpty() ? table[1] : table[0] + "." + table[1], e); + } + } + } catch (Exception e) { + throw ExceptionUtils.toSqlState(e); + } + return DetachedResultSet.createFromRecords(records, GET_COLUMNS_COLUMNS, connection.getDefaultCalendar()); + } + + /** + * Returns the names of the databases that match the pattern, sorted by name (the server sorts the result of + * SHOW DATABASES). + */ + private List showDatabases(String schemaPattern) throws SQLException { + List databases = new ArrayList<>(); + if (schemaPattern != null && schemaPattern.isEmpty()) { + return databases; // SHOW ignores an empty LIKE pattern, but it matches only an empty name + } + try (PreparedStatement stmt = DatabaseMetaDataImpl.prepareStatement(connection, "SHOW DATABASES LIKE ?")) { + stmt.setString(1, likePattern(schemaPattern)); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + databases.add(rs.getString(1)); + } + } + } + return databases; + } + + /** + * Returns {database, name, engine} of the tables that match the patterns. Temporary tables have an empty database, + * the same as in system.tables, and are returned only when {@code includeTemporary} is set. + */ + private List showTables(String schemaPattern, String tableNamePattern, boolean includeTemporary) + throws SQLException { + List tables = new ArrayList<>(); + if (tableNamePattern != null && tableNamePattern.isEmpty()) { + return tables; // SHOW ignores an empty LIKE pattern, but it matches only an empty name + } + if (includeTemporary && likeMatcher(schemaPattern).test("")) { + tables.addAll(showTablesOfDatabase("", tableNamePattern)); + } + for (String database : showDatabases(schemaPattern)) { + try { + tables.addAll(showTablesOfDatabase(database, tableNamePattern)); + } catch (SQLException e) { + if (!isSkippedObjectError(e)) { + throw e; + } + log.debug("Skipped metadata of database {}", database, e); + } + } + return tables; + } + + /** + * Returns {database, name, engine} of the tables of one database, or of the temporary tables when the database is + * empty. The server accepts only an identifier after {@code FROM}, so the database name is escaped by + * {@link #quoteIdentifier(String)} and only the pattern is a parameter. + */ + @SuppressWarnings({"squid:S2077"}) + private List showTablesOfDatabase(String database, String tableNamePattern) throws SQLException { + String sql = database.isEmpty() ? "SHOW FULL TEMPORARY TABLES LIKE ?" + : "SHOW FULL TABLES FROM " + quoteIdentifier(database) + " LIKE ?"; + List tables = new ArrayList<>(); + try (PreparedStatement stmt = DatabaseMetaDataImpl.prepareStatement(connection, sql)) { + stmt.setString(1, likePattern(tableNamePattern)); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + tables.add(new String[]{database, rs.getString("name"), rs.getString("engine")}); + } + } + } + return tables; + } + + /** + * Returns the rows of {@code getColumns()} for the columns of one table that match the column name matcher. The + * server accepts only an identifier after {@code DESCRIBE TABLE}, so the names are escaped by + * {@link #quoteIdentifier(String)}. A temporary table has an empty database. + */ + @SuppressWarnings({"squid:S2077"}) + private List> describeTable(String database, String table, Predicate columnNameMatcher) + throws SQLException { + String tableRef = database.isEmpty() ? quoteIdentifier(table) + : quoteIdentifier(database) + "." + quoteIdentifier(table); + List> records = new ArrayList<>(); + try (PreparedStatement stmt = DatabaseMetaDataImpl.prepareStatement(connection, "DESCRIBE TABLE " + tableRef); + ResultSet rs = stmt.executeQuery()) { + Set describeColumns = new HashSet<>(); + ResultSetMetaData describeMetaData = rs.getMetaData(); + for (int i = 1; i <= describeMetaData.getColumnCount(); i++) { + describeColumns.add(describeMetaData.getColumnLabel(i)); + } + int position = 0; + while (rs.next()) { + if ((describeColumns.contains("is_subcolumn") && rs.getInt("is_subcolumn") != 0) + || (describeColumns.contains("is_virtual") && rs.getInt("is_virtual") != 0)) { + continue; + } + position++; + String columnName = rs.getString("name"); + if (columnNameMatcher.test(columnName)) { + records.add(createColumnRecord(database, table, columnName, position, rs.getString("type"), + describeColumns.contains("default_expression") ? rs.getString("default_expression") : "", + describeColumns.contains("comment") ? rs.getString("comment") : "")); + } + } + } + return records; + } + + private static Map createColumnRecord(String database, String table, String column, int position, + String describedType, String defaultExpression, + String comment) { + String type = PRETTY_TYPE_SEPARATOR.matcher(PRETTY_TYPE_OPENING.matcher(describedType).replaceAll("(")) + .replaceAll(", "); + Map record = new HashMap<>(); + record.put("TABLE_CAT", ""); + record.put("TABLE_SCHEM", database); + record.put("TABLE_NAME", table); + record.put("COLUMN_NAME", column); + record.put("TYPE_NAME", type); + putColumnSizes(record, type); + record.put("BUFFER_LENGTH", 0); + record.put("NULLABLE", type.contains("Nullable(") ? typeNullable : typeNoNulls); + record.put("REMARKS", comment); + record.put("COLUMN_DEF", defaultExpression); + record.put("SQL_DATA_TYPE", 0); + record.put("SQL_DATETIME_SUB", 0); + record.put("ORDINAL_POSITION", position); + record.put("IS_NULLABLE", type.toUpperCase(Locale.ROOT).contains("NULLABLE") ? "YES" : "NO"); + record.put("SCOPE_CATALOG", null); + record.put("SCOPE_SCHEMA", null); + record.put("SCOPE_TABLE", null); + record.put("SOURCE_DATA_TYPE", null); + record.put("IS_AUTOINCREMENT", "NO"); + record.put("IS_GENERATEDCOLUMN", "NO"); + DatabaseMetaDataImpl.DATA_TYPE_VALUE_FUNCTION.accept(record); + return record; + } + + /** + * Sets COLUMN_SIZE, DECIMAL_DIGITS, NUM_PREC_RADIX and CHAR_OCTET_LENGTH with the values that the system.columns + * query returns. system.columns unwraps only Nullable (and the custom name of SimpleAggregateFunction), and reports + * the bit width of integers, the precision and scale of decimals and the length of FixedString. + */ + private static void putColumnSizes(Map record, String type) { + Integer precision = null; + Integer radix = null; + Integer scale = null; + Integer octetLength = null; + try { + ClickHouseColumn column = ClickHouseColumn.of("", type); + if (column.getDataType() == ClickHouseDataType.SimpleAggregateFunction) { + column = column.getNestedColumns().get(0); + } + ClickHouseDataType dataType = column.getDataType(); + if (!column.isLowCardinality()) { + if (dataType == ClickHouseDataType.FixedString) { + octetLength = column.getPrecision(); + } else if (INTEGER_TYPES.contains(dataType)) { + precision = dataType.getByteLength() * Byte.SIZE; + radix = 2; + scale = 0; + } else if (DECIMAL_TYPES.contains(dataType)) { + precision = column.getPrecision(); + radix = 10; + scale = column.getScale(); + } + } + } catch (Exception e) { + log.debug("Failed to parse column type: {}", type, e); + } + Integer byteLength = TYPE_NAME_TO_BYTE_LENGTH.get(type); + record.put("COLUMN_SIZE", octetLength != null ? octetLength + : byteLength != null ? byteLength : precision != null ? precision : 0); + record.put("DECIMAL_DIGITS", scale == null || scale == 0 ? null : scale); + record.put("NUM_PREC_RADIX", radix); + record.put("CHAR_OCTET_LENGTH", octetLength); + } + + private static String likePattern(String pattern) { + return pattern == null ? "%" : pattern; + } + + private static String quoteIdentifier(String name) { + return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; + } + + /** + * Returns true for errors of a single database or table that must not fail the whole metadata call: the object + * was dropped after it was listed, the user can list it but cannot describe it (for example, with column-level + * grants), or a data lake catalog cannot read its metadata. + */ + private static boolean isSkippedObjectError(SQLException e) { + return SKIPPED_OBJECT_ERROR_CODES.contains(e.getErrorCode()); + } + + /** + * Returns a matcher with the semantics of the ClickHouse LIKE operator: {@code %} matches any sequence of + * characters, {@code _} matches one character and {@code \} escapes the next character. A {@code null} pattern + * matches everything. + */ + static Predicate likeMatcher(String pattern) { + if (pattern == null) { + return value -> true; + } + StringBuilder regex = new StringBuilder(); + StringBuilder literal = new StringBuilder(); + for (int i = 0; i < pattern.length(); i++) { + char c = pattern.charAt(i); + if (c == '%' || c == '_') { + if (literal.length() > 0) { + regex.append(Pattern.quote(literal.toString())); + literal.setLength(0); + } + regex.append(c == '%' ? ".*" : "."); + } else { + literal.append(c == '\\' && i + 1 < pattern.length() ? pattern.charAt(++i) : c); + } + } + if (literal.length() > 0) { + regex.append(Pattern.quote(literal.toString())); + } + Pattern compiled = Pattern.compile(regex.toString(), Pattern.DOTALL); + return value -> compiled.matcher(value).matches(); + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index f05fb06f8..5020d7226 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -633,7 +633,7 @@ private interface MetadataCall { private static final String SHOW_STATEMENTS_TABLE = "metadata_show_statements"; - private static final String SHOW_STATEMENTS_ODD_TABLE = SHOW_STATEMENTS_TABLE + " odd.'\"\\`name"; + private static final String SHOW_STATEMENTS_ODD_TABLE = SHOW_STATEMENTS_TABLE + " odd.'\"\\`?name"; private static String quoteIdentifier(String name) { return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; @@ -688,7 +688,7 @@ public void testShowStatementsReturnSameMetadataAsSystemTables(MetadataCall call stmt.executeUpdate("CREATE MATERIALIZED VIEW " + SHOW_STATEMENTS_TABLE + "_mv ENGINE = MergeTree ORDER BY id " + "AS SELECT id, s FROM " + SHOW_STATEMENTS_TABLE); stmt.executeUpdate("CREATE TABLE " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE) + " (id Int32, " + - quoteIdentifier("odd \"col\"\\") + " String) ENGINE = MergeTree ORDER BY id"); + quoteIdentifier("odd \"col\"?\\") + " String) ENGINE = MergeTree ORDER BY id"); } Properties systemTablesProps = new Properties(); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java similarity index 90% rename from jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java rename to jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java index 0e6d7b51b..d1abd63dd 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImplLikeMatcherTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java @@ -5,7 +5,7 @@ import static org.testng.Assert.assertEquals; -public class DatabaseMetaDataImplLikeMatcherTest { +public class ShowStatementsMetaDataLikeMatcherTest { @DataProvider(name = "likePatterns") public Object[][] likePatterns() { @@ -37,7 +37,7 @@ public Object[][] likePatterns() { @Test(groups = {"unit"}, dataProvider = "likePatterns") public void testLikeMatcher(String pattern, String value, boolean expected) { - assertEquals(DatabaseMetaDataImpl.likeMatcher(pattern).test(value), expected, + assertEquals(ShowStatementsMetaData.likeMatcher(pattern).test(value), expected, "'" + value + "' LIKE '" + pattern + "'"); } } From 05ef73c0fd77ebb9de662c41d02301f78bd65b44 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:55:22 +0000 Subject: [PATCH 4/4] test(jdbc-v2): run DatabaseMetaDataTest with SHOW statements in a subclass DatabaseMetaDataTest now disables jdbc_metadata_use_show_statements, so it covers the system table queries. ShowStatementDatabaseMetaDataTest extends it and enables the flag, so the whole suite also runs on the SHOW/DESCRIBE path. The show-vs-system parity, flag and readonly user tests move to the new class. ShowStatementDatabaseMetaDataWithEmptyFormatTest runs the SHOW suite with an empty format, the same as DatabaseMetaDataWithEmptyFormatTest does for the system tables. The LIKE matcher unit test becomes a database test: it checks each case with the server's LIKE and through getColumns(). likeMatcher() is now private. testGetColumnsOfTemporaryTable covers the schema patterns that select temporary tables. Co-Authored-By: Claude Opus 5.5 --- .../jdbc/metadata/ShowStatementsMetaData.java | 2 +- .../jdbc/metadata/DatabaseMetaDataTest.java | 231 ++----------- .../ShowStatementDatabaseMetaDataTest.java | 304 ++++++++++++++++++ ...ntDatabaseMetaDataWithEmptyFormatTest.java | 35 ++ ...ShowStatementsMetaDataLikeMatcherTest.java | 43 --- 5 files changed, 375 insertions(+), 240 deletions(-) create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataTest.java create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataWithEmptyFormatTest.java delete mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java index a98932561..5fd4f2eb2 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaData.java @@ -354,7 +354,7 @@ private static boolean isSkippedObjectError(SQLException e) { * characters, {@code _} matches one character and {@code \} escapes the next character. A {@code null} pattern * matches everything. */ - static Predicate likeMatcher(String pattern) { + private static Predicate likeMatcher(String pattern) { if (pattern == null) { return value -> true; } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index f812c4c0f..54ab62e64 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Properties; @@ -44,6 +43,19 @@ @Test(groups = { "integration" }) public class DatabaseMetaDataTest extends JdbcIntegrationTest { + + /** + * Disables {@link DriverProperties#METADATA_USE_SHOW_STATEMENTS} unless the caller sets it, so this suite reads + * metadata from the system tables. {@link ShowStatementDatabaseMetaDataTest} runs the same suite with the flag + * enabled. + */ + @Override + public Connection getJdbcConnection(Properties properties) throws SQLException { + Properties props = properties == null ? new Properties() : (Properties) properties.clone(); + props.putIfAbsent(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), String.valueOf(false)); + return super.getJdbcConnection(props); + } + @Test(groups = { "integration" }) public void testGetColumns() throws Exception { try (Connection conn = getJdbcConnection()) { @@ -710,158 +722,10 @@ public void testGetSchemas() throws Exception { } } - private interface MetadataCall { - ResultSet call(DatabaseMetaData metaData) throws SQLException; - } - - private static final String SHOW_STATEMENTS_TABLE = "metadata_show_statements"; - - private static final String SHOW_STATEMENTS_ODD_TABLE = SHOW_STATEMENTS_TABLE + " odd.'\"\\`?name"; - - private static String quoteIdentifier(String name) { - return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; - } - - @DataProvider(name = "metadataCallsWithShowStatements") - public Object[][] metadataCallsWithShowStatements() { - final String db = getDatabase(); - final String table = SHOW_STATEMENTS_TABLE; - final String tables = SHOW_STATEMENTS_TABLE + "%"; - return new Object[][] { - {(MetadataCall) md -> md.getColumns(null, db, table, null), false, true}, - {(MetadataCall) md -> md.getColumns(null, db, table, "d%"), false, true}, - {(MetadataCall) md -> md.getColumns(null, db, table, "%\\_t%"), false, true}, - {(MetadataCall) md -> md.getColumns(null, db, tables, null), false, true}, - {(MetadataCall) md -> md.getColumns(null, db + "%", "metadata\\_show\\_statements%", null), false, true}, - {(MetadataCall) md -> md.getColumns(null, db, table, ""), false, false}, - {(MetadataCall) md -> md.getColumns(null, db, "metadata\\_show\\_statements odd.'%", "odd \"col\"%"), false, true}, - {(MetadataCall) md -> md.getColumns(null, "%' OR '1'='1", "%", null), false, false}, - {(MetadataCall) md -> md.getTables(null, db, "metadata\\_show\\_statements odd.'%", null), true, true}, - {(MetadataCall) md -> md.getTables(null, db, tables, null), true, true}, - {(MetadataCall) md -> md.getTables(null, db + "%", "metadata\\_show\\_statements%", null), true, true}, - {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"VIEW", "MATERIALIZED VIEW"}), true, true}, - {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"UNKNOWN"}), true, true}, - {(MetadataCall) md -> md.getTables(null, "system", "%", new String[]{"SYSTEM TABLE"}), true, true}, - {(MetadataCall) md -> md.getTables(null, db, "", null), true, false}, - {(MetadataCall) md -> md.getSchemas(null, db), false, true}, - {(MetadataCall) md -> md.getSchemas(null, "syst%"), false, true}, - }; - } - - @Test(groups = {"integration"}, dataProvider = "metadataCallsWithShowStatements") - public void testShowStatementsReturnSameMetadataAsSystemTables(MetadataCall call, boolean isGetTables, - boolean expectRows) throws Exception { - try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { - stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_mv"); - stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_view"); - stmt.executeUpdate("DROP TABLE IF EXISTS " + SHOW_STATEMENTS_TABLE); - stmt.executeUpdate("DROP TABLE IF EXISTS " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE)); - stmt.executeUpdate("CREATE TABLE " + SHOW_STATEMENTS_TABLE + " (" + - "id Int32 COMMENT 'row id', u8 UInt8, i128 Int128, u256 UInt256, b Bool, nb Nullable(Bool), " + - "f32 Float32, d Decimal(10, 2) DEFAULT 1.5, d0 Nullable(Decimal(5, 0)), d64 Decimal64(4), " + - "s String, fs FixedString(4), nfs Nullable(FixedString(3)), lfs LowCardinality(FixedString(3)), " + - "ls LowCardinality(Nullable(String)), e Enum8('a' = 1), dt DateTime64(3, 'UTC'), dd Date, " + - "arr Array(Nullable(Int8)), m Map(String, Tuple(x Int32, y String)), " + - "nested_tuple Tuple(a Tuple(b Int32, c Nullable(String)), e UInt8), j JSON(a.b UInt32), " + - "sa SimpleAggregateFunction(sum, UInt64), " + - "nsa SimpleAggregateFunction(anyLast, Nullable(Decimal(9, 3))), " + - "mat Int64 MATERIALIZED id * 2, al String ALIAS toString(id), eph Int32 EPHEMERAL" + - ") ENGINE = MergeTree ORDER BY id COMMENT 'table comment'"); - stmt.executeUpdate("CREATE VIEW " + SHOW_STATEMENTS_TABLE + "_view AS SELECT id, d FROM " + SHOW_STATEMENTS_TABLE); - stmt.executeUpdate("CREATE MATERIALIZED VIEW " + SHOW_STATEMENTS_TABLE + "_mv ENGINE = MergeTree ORDER BY id " + - "AS SELECT id, s FROM " + SHOW_STATEMENTS_TABLE); - stmt.executeUpdate("CREATE TABLE " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE) + " (id Int32, " + - quoteIdentifier("odd \"col\"?\\") + " String) ENGINE = MergeTree ORDER BY id"); - } - - Properties systemTablesProps = new Properties(); - systemTablesProps.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), "false"); - try (Connection showConn = getJdbcConnection(); Connection systemConn = getJdbcConnection(systemTablesProps); - ResultSet showRs = call.call(showConn.getMetaData()); - ResultSet systemRs = call.call(systemConn.getMetaData())) { - Set ignoredColumns = isGetTables - ? new HashSet<>(Arrays.asList("REMARKS", "TYPE_SCHEM")) : Collections.emptySet(); - - assertEquals(columnsOf(showRs.getMetaData(), ignoredColumns), columnsOf(systemRs.getMetaData(), ignoredColumns)); - List> showRows = rowsOf(showRs, ignoredColumns); - List> systemRows = rowsOf(systemRs, ignoredColumns); - assertEquals(!systemRows.isEmpty(), expectRows); - if (isGetTables) { - showRows.sort(Comparator.comparing(Object::toString)); - systemRows.sort(Comparator.comparing(Object::toString)); - } - assertEquals(showRows, systemRows); - } - } - - private static List columnsOf(ResultSetMetaData metaData, Set ignoredColumns) throws SQLException { - List columns = new ArrayList<>(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String label = metaData.getColumnLabel(i); - columns.add(label + ":" + metaData.getColumnType(i) + (ignoredColumns.contains(label) ? "" - : ":" + metaData.getColumnTypeName(i) + ":" + metaData.isNullable(i))); - } - return columns; - } - - private static List> rowsOf(ResultSet rs, Set ignoredColumns) throws SQLException { - ResultSetMetaData metaData = rs.getMetaData(); - List> rows = new ArrayList<>(); - while (rs.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - if (!ignoredColumns.contains(metaData.getColumnLabel(i))) { - row.add(rs.getObject(i)); - } - } - rows.add(row); - } - return rows; - } - - @DataProvider(name = "showStatementsFlagValues") - public Object[][] showStatementsFlagValues() { - return new Object[][] { - {null, null}, - {"true", null}, - {"false", "flag table comment"}, - }; - } - - @Test(groups = {"integration"}, dataProvider = "showStatementsFlagValues") - public void testTableRemarksDependOnShowStatementsFlag(String flag, String expectedRemarks) throws Exception { - final String tableName = "metadata_show_statements_flag"; - Properties props = new Properties(); - if (flag != null) { - props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); - } - try (Connection conn = getJdbcConnection(props)) { - try (Statement stmt = conn.createStatement()) { - stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); - stmt.executeUpdate("CREATE TABLE " + tableName + " (id Int32) ENGINE = MergeTree ORDER BY id " + - "COMMENT 'flag table comment'"); - } - try (ResultSet rs = conn.getMetaData().getTables(null, getDatabase(), tableName, null)) { - assertTrue(rs.next()); - assertEquals(rs.getString("TABLE_NAME"), tableName); - assertEquals(rs.getString("TABLE_TYPE"), "TABLE"); - assertEquals(rs.getString("REMARKS"), expectedRemarks); - assertFalse(rs.next()); - } - } - } - - @DataProvider(name = "showStatementsFlags") - public Object[][] showStatementsFlags() { - return new Object[][] {{"true"}, {"false"}}; - } - - @Test(groups = {"integration"}, dataProvider = "showStatementsFlags") - public void testGetSchemasOrderedBySchemaName(String flag) throws Exception { - Properties props = new Properties(); - props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + @Test(groups = {"integration"}) + public void testGetSchemasOrderedBySchemaName() throws Exception { List schemas = new ArrayList<>(); - try (Connection conn = getJdbcConnection(props); ResultSet rs = conn.getMetaData().getSchemas()) { + try (Connection conn = getJdbcConnection(); ResultSet rs = conn.getMetaData().getSchemas()) { while (rs.next()) { schemas.add(rs.getString("TABLE_SCHEM")); } @@ -872,55 +736,30 @@ public void testGetSchemasOrderedBySchemaName(String flag) throws Exception { assertEquals(schemas, sortedSchemas); } - @DataProvider(name = "showStatementsFlagForRestrictedUser") - public Object[][] showStatementsFlagForRestrictedUser() { - return new Object[][] { - {"true", Arrays.asList("metadata_show_statements_granted.id:Int32", - "metadata_show_statements_granted.t:Tuple(a Int32, b String)")}, - {"false", Arrays.asList("metadata_show_statements_granted.id:Int32", - "metadata_show_statements_granted.t:Tuple(a Int32, b String)", - "metadata_show_statements_restricted.id:Int32")}, - }; + @DataProvider(name = "temporaryTableSchemaPatterns") + public Object[][] temporaryTableSchemaPatterns() { + return new Object[][] {{null, true}, {"", true}, {"%", true}, {getDatabase(), false}}; } - @Test(groups = {"integration"}, dataProvider = "showStatementsFlagForRestrictedUser") - public void testGetColumnsForReadonlyUserWithColumnGrants(String flag, List expectedColumns) throws Exception { + @Test(groups = {"integration"}, dataProvider = "temporaryTableSchemaPatterns") + public void testGetColumnsOfTemporaryTable(String schemaPattern, boolean expected) throws Exception { if (isCloud()) { - throw new SkipException("Should not be tested in cloud because creates users"); - } - final String db = getDatabase(); - final String user = "metadata_show_statements_user"; - final String password = UUID.randomUUID() + "Aa1!"; - try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { - stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_granted"); - stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_restricted"); - stmt.executeUpdate("CREATE TABLE metadata_show_statements_granted (id Int32, t Tuple(a Int32, b String)) " + - "ENGINE = MergeTree ORDER BY id"); - stmt.executeUpdate("CREATE TABLE metadata_show_statements_restricted (id Int32, secret String) " + - "ENGINE = MergeTree ORDER BY id"); - stmt.executeUpdate("DROP USER IF EXISTS " + user); - stmt.executeUpdate("CREATE USER " + user + " IDENTIFIED WITH plaintext_password BY '" + password + - "' SETTINGS readonly = 1"); - stmt.executeUpdate("GRANT SELECT ON " + db + ".metadata_show_statements_granted TO " + user); - stmt.executeUpdate("GRANT SELECT(id) ON " + db + ".metadata_show_statements_restricted TO " + user); + throw new SkipException("HTTP sessions require server affinity"); } - try { - Properties props = new Properties(); - props.setProperty("user", user); - props.setProperty("password", password); - props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); - List columns = new ArrayList<>(); - try (Connection conn = getJdbcConnection(props); - ResultSet rs = conn.getMetaData().getColumns(null, db, "metadata_show_statements_%ed", null)) { - while (rs.next()) { - columns.add(rs.getString("TABLE_NAME") + "." + rs.getString("COLUMN_NAME") + ":" + - rs.getString("TYPE_NAME")); - } + final String tableName = "metadata_temporary_table"; + Properties props = new Properties(); + props.setProperty(ClientConfigProperties.serverSetting("session_id"), tableName + "_" + UUID.randomUUID()); + try (Connection conn = getJdbcConnection(props)) { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TEMPORARY TABLE " + tableName + " (id Int32)"); } - assertEquals(columns, expectedColumns); - } finally { - try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { - stmt.executeUpdate("DROP USER IF EXISTS " + user); + try (ResultSet rs = conn.getMetaData().getColumns(null, schemaPattern, tableName, null)) { + assertEquals(rs.next(), expected); + if (expected) { + assertEquals(rs.getString("TABLE_SCHEM"), ""); + assertEquals(rs.getString("COLUMN_NAME"), "id"); + assertFalse(rs.next()); + } } } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataTest.java new file mode 100644 index 000000000..d0bf8f220 --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataTest.java @@ -0,0 +1,304 @@ +package com.clickhouse.jdbc.metadata; + +import com.clickhouse.client.ClickHouseServerForTest; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.jdbc.ConnectionImpl; +import com.clickhouse.jdbc.DriverProperties; +import org.testng.SkipException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +/** + * Runs the whole {@link DatabaseMetaDataTest} suite with {@link DriverProperties#METADATA_USE_SHOW_STATEMENTS} + * enabled, so {@code getSchemas()}, {@code getTables()} and {@code getColumns()} read metadata with + * {@code SHOW DATABASES}, {@code SHOW TABLES} and {@code DESCRIBE TABLE}. The tests of this class compare the results + * with the system table queries and cover the flag. + */ +@Test(groups = { "integration" }) +public class ShowStatementDatabaseMetaDataTest extends DatabaseMetaDataTest { + + @Override + public Connection getJdbcConnection(Properties properties) throws SQLException { + Properties props = properties == null ? new Properties() : (Properties) properties.clone(); + props.putIfAbsent(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), String.valueOf(true)); + return super.getJdbcConnection(props); + } + + private Connection getJdbcConnectionWithDefaultFlag() throws SQLException { + Properties info = new Properties(); + info.setProperty("user", "default"); + info.setProperty("password", ClickHouseServerForTest.getPassword()); + info.setProperty(ClientConfigProperties.DATABASE.getKey(), getDatabase()); + return new ConnectionImpl(getEndpointString(), info); + } + + private interface MetadataCall { + ResultSet call(DatabaseMetaData metaData) throws SQLException; + } + + private static final String SHOW_STATEMENTS_TABLE = "metadata_show_statements"; + + private static final String SHOW_STATEMENTS_ODD_TABLE = SHOW_STATEMENTS_TABLE + " odd.'\"\\`?name"; + + private static String quoteIdentifier(String name) { + return '`' + name.replace("\\", "\\\\").replace("`", "\\`") + '`'; + } + + @DataProvider(name = "metadataCallsWithShowStatements") + public Object[][] metadataCallsWithShowStatements() { + final String db = getDatabase(); + final String table = SHOW_STATEMENTS_TABLE; + final String tables = SHOW_STATEMENTS_TABLE + "%"; + return new Object[][] { + {(MetadataCall) md -> md.getColumns(null, db, table, null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, "d%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, "%\\_t%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, tables, null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db + "%", "metadata\\_show\\_statements%", null), false, true}, + {(MetadataCall) md -> md.getColumns(null, db, table, ""), false, false}, + {(MetadataCall) md -> md.getColumns(null, db, "metadata\\_show\\_statements odd.'%", "odd \"col\"%"), false, true}, + {(MetadataCall) md -> md.getColumns(null, "%' OR '1'='1", "%", null), false, false}, + {(MetadataCall) md -> md.getTables(null, db, "metadata\\_show\\_statements odd.'%", null), true, true}, + {(MetadataCall) md -> md.getTables(null, db, tables, null), true, true}, + {(MetadataCall) md -> md.getTables(null, db + "%", "metadata\\_show\\_statements%", null), true, true}, + {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"VIEW", "MATERIALIZED VIEW"}), true, true}, + {(MetadataCall) md -> md.getTables(null, db, tables, new String[]{"UNKNOWN"}), true, true}, + {(MetadataCall) md -> md.getTables(null, "system", "%", new String[]{"SYSTEM TABLE"}), true, true}, + {(MetadataCall) md -> md.getTables(null, db, "", null), true, false}, + {(MetadataCall) md -> md.getSchemas(null, db), false, true}, + {(MetadataCall) md -> md.getSchemas(null, "syst%"), false, true}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "metadataCallsWithShowStatements") + public void testShowStatementsReturnSameMetadataAsSystemTables(MetadataCall call, boolean isGetTables, + boolean expectRows) throws Exception { + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_mv"); + stmt.executeUpdate("DROP VIEW IF EXISTS " + SHOW_STATEMENTS_TABLE + "_view"); + stmt.executeUpdate("DROP TABLE IF EXISTS " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("DROP TABLE IF EXISTS " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE)); + stmt.executeUpdate("CREATE TABLE " + SHOW_STATEMENTS_TABLE + " (" + + "id Int32 COMMENT 'row id', u8 UInt8, i128 Int128, u256 UInt256, b Bool, nb Nullable(Bool), " + + "f32 Float32, d Decimal(10, 2) DEFAULT 1.5, d0 Nullable(Decimal(5, 0)), d64 Decimal64(4), " + + "s String, fs FixedString(4), nfs Nullable(FixedString(3)), lfs LowCardinality(FixedString(3)), " + + "ls LowCardinality(Nullable(String)), e Enum8('a' = 1), dt DateTime64(3, 'UTC'), dd Date, " + + "arr Array(Nullable(Int8)), m Map(String, Tuple(x Int32, y String)), " + + "nested_tuple Tuple(a Tuple(b Int32, c Nullable(String)), e UInt8), j JSON(a.b UInt32), " + + "sa SimpleAggregateFunction(sum, UInt64), " + + "nsa SimpleAggregateFunction(anyLast, Nullable(Decimal(9, 3))), " + + "mat Int64 MATERIALIZED id * 2, al String ALIAS toString(id), eph Int32 EPHEMERAL" + + ") ENGINE = MergeTree ORDER BY id COMMENT 'table comment'"); + stmt.executeUpdate("CREATE VIEW " + SHOW_STATEMENTS_TABLE + "_view AS SELECT id, d FROM " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("CREATE MATERIALIZED VIEW " + SHOW_STATEMENTS_TABLE + "_mv ENGINE = MergeTree ORDER BY id " + + "AS SELECT id, s FROM " + SHOW_STATEMENTS_TABLE); + stmt.executeUpdate("CREATE TABLE " + quoteIdentifier(SHOW_STATEMENTS_ODD_TABLE) + " (id Int32, " + + quoteIdentifier("odd \"col\"?\\") + " String) ENGINE = MergeTree ORDER BY id"); + } + + Properties systemTablesProps = new Properties(); + systemTablesProps.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), "false"); + try (Connection showConn = getJdbcConnection(); Connection systemConn = getJdbcConnection(systemTablesProps); + ResultSet showRs = call.call(showConn.getMetaData()); + ResultSet systemRs = call.call(systemConn.getMetaData())) { + Set ignoredColumns = isGetTables + ? new HashSet<>(Arrays.asList("REMARKS", "TYPE_SCHEM")) : Collections.emptySet(); + + assertEquals(columnsOf(showRs.getMetaData(), ignoredColumns), columnsOf(systemRs.getMetaData(), ignoredColumns)); + List> showRows = rowsOf(showRs, ignoredColumns); + List> systemRows = rowsOf(systemRs, ignoredColumns); + assertEquals(!systemRows.isEmpty(), expectRows); + if (isGetTables) { + showRows.sort(Comparator.comparing(Object::toString)); + systemRows.sort(Comparator.comparing(Object::toString)); + } + assertEquals(showRows, systemRows); + } + } + + private static List columnsOf(ResultSetMetaData metaData, Set ignoredColumns) throws SQLException { + List columns = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String label = metaData.getColumnLabel(i); + columns.add(label + ":" + metaData.getColumnType(i) + (ignoredColumns.contains(label) ? "" + : ":" + metaData.getColumnTypeName(i) + ":" + metaData.isNullable(i))); + } + return columns; + } + + private static List> rowsOf(ResultSet rs, Set ignoredColumns) throws SQLException { + ResultSetMetaData metaData = rs.getMetaData(); + List> rows = new ArrayList<>(); + while (rs.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + if (!ignoredColumns.contains(metaData.getColumnLabel(i))) { + row.add(rs.getObject(i)); + } + } + rows.add(row); + } + return rows; + } + + @DataProvider(name = "showStatementsFlagValues") + public Object[][] showStatementsFlagValues() { + return new Object[][] { + {null, null}, + {"true", null}, + {"false", "flag table comment"}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "showStatementsFlagValues") + public void testTableRemarksDependOnShowStatementsFlag(String flag, String expectedRemarks) throws Exception { + final String tableName = "metadata_show_statements_flag"; + Properties props = new Properties(); + if (flag != null) { + props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + } + try (Connection conn = flag == null ? getJdbcConnectionWithDefaultFlag() : getJdbcConnection(props)) { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); + stmt.executeUpdate("CREATE TABLE " + tableName + " (id Int32) ENGINE = MergeTree ORDER BY id " + + "COMMENT 'flag table comment'"); + } + try (ResultSet rs = conn.getMetaData().getTables(null, getDatabase(), tableName, null)) { + assertTrue(rs.next()); + assertEquals(rs.getString("TABLE_NAME"), tableName); + assertEquals(rs.getString("TABLE_TYPE"), "TABLE"); + assertEquals(rs.getString("REMARKS"), expectedRemarks); + assertFalse(rs.next()); + } + } + } + + @DataProvider(name = "showStatementsFlagForRestrictedUser") + public Object[][] showStatementsFlagForRestrictedUser() { + return new Object[][] { + {"true", Arrays.asList("metadata_show_statements_granted.id:Int32", + "metadata_show_statements_granted.t:Tuple(a Int32, b String)")}, + {"false", Arrays.asList("metadata_show_statements_granted.id:Int32", + "metadata_show_statements_granted.t:Tuple(a Int32, b String)", + "metadata_show_statements_restricted.id:Int32")}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "showStatementsFlagForRestrictedUser") + public void testGetColumnsForReadonlyUserWithColumnGrants(String flag, List expectedColumns) throws Exception { + if (isCloud()) { + throw new SkipException("Should not be tested in cloud because creates users"); + } + final String db = getDatabase(); + final String user = "metadata_show_statements_user"; + final String password = UUID.randomUUID() + "Aa1!"; + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_granted"); + stmt.executeUpdate("DROP TABLE IF EXISTS metadata_show_statements_restricted"); + stmt.executeUpdate("CREATE TABLE metadata_show_statements_granted (id Int32, t Tuple(a Int32, b String)) " + + "ENGINE = MergeTree ORDER BY id"); + stmt.executeUpdate("CREATE TABLE metadata_show_statements_restricted (id Int32, secret String) " + + "ENGINE = MergeTree ORDER BY id"); + stmt.executeUpdate("DROP USER IF EXISTS " + user); + stmt.executeUpdate("CREATE USER " + user + " IDENTIFIED WITH plaintext_password BY '" + password + + "' SETTINGS readonly = 1"); + stmt.executeUpdate("GRANT SELECT ON " + db + ".metadata_show_statements_granted TO " + user); + stmt.executeUpdate("GRANT SELECT(id) ON " + db + ".metadata_show_statements_restricted TO " + user); + } + try { + Properties props = new Properties(); + props.setProperty("user", user); + props.setProperty("password", password); + props.setProperty(DriverProperties.METADATA_USE_SHOW_STATEMENTS.getKey(), flag); + List columns = new ArrayList<>(); + try (Connection conn = getJdbcConnection(props); + ResultSet rs = conn.getMetaData().getColumns(null, db, "metadata_show_statements_%ed", null)) { + while (rs.next()) { + columns.add(rs.getString("TABLE_NAME") + "." + rs.getString("COLUMN_NAME") + ":" + + rs.getString("TYPE_NAME")); + } + } + assertEquals(columns, expectedColumns); + } finally { + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP USER IF EXISTS " + user); + } + } + } + + @DataProvider(name = "columnNameLikePatterns") + public Object[][] columnNameLikePatterns() { + return new Object[][] { + {null, "any_name", true}, + {"", "a", false}, + {"%", "a\nb", true}, + {"abc", "abc", true}, + {"abc", "ABC", false}, + {"abc", "abcd", false}, + {"a_c", "abc", true}, + {"a_c", "ac", false}, + {"a%c", "ac", true}, + {"a%c", "a.b.c", true}, + {"a%c", "acd", false}, + {"a\\_c", "a_c", true}, + {"a\\_c", "abc", false}, + {"100\\%", "100%", true}, + {"100\\%", "1000", false}, + {"a\\\\b", "a\\b", true}, + {"a.c", "abc", false}, + {"(x)[y]*", "(x)[y]*", true}, + {"_", "😀", true}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "columnNameLikePatterns") + public void testGetColumnsMatchesColumnNamePatternLikeServer(String pattern, String columnName, boolean expected) + throws Exception { + final String tableName = "metadata_show_statements_like"; + try (Connection conn = getJdbcConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); + stmt.executeUpdate("CREATE TABLE " + tableName + " (" + quoteIdentifier(columnName) + " String) " + + "ENGINE = Memory"); + } + if (pattern != null) { + try (PreparedStatement stmt = conn.prepareStatement("SELECT ? LIKE ?")) { + stmt.setString(1, columnName); + stmt.setString(2, pattern); + try (ResultSet rs = stmt.executeQuery()) { + assertTrue(rs.next()); + assertEquals(rs.getBoolean(1), expected, "server: '" + columnName + "' LIKE '" + pattern + "'"); + } + } + } + try (ResultSet rs = conn.getMetaData().getColumns(null, getDatabase(), tableName, pattern)) { + assertEquals(rs.next(), expected, "getColumns: '" + columnName + "' LIKE '" + pattern + "'"); + if (expected) { + assertEquals(rs.getString("COLUMN_NAME"), columnName); + assertFalse(rs.next()); + } + } + } + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataWithEmptyFormatTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataWithEmptyFormatTest.java new file mode 100644 index 000000000..cb97e394d --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementDatabaseMetaDataWithEmptyFormatTest.java @@ -0,0 +1,35 @@ +package com.clickhouse.jdbc.metadata; + +import com.clickhouse.client.api.ClientConfigProperties; +import org.testng.SkipException; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +/** + * Runs the whole {@link ShowStatementDatabaseMetaDataTest} suite over connections configured with an empty + * {@code format} property, the same as {@link DatabaseMetaDataWithEmptyFormatTest} does for the system table queries. + */ +@Test(groups = { "integration" }) +public class ShowStatementDatabaseMetaDataWithEmptyFormatTest extends ShowStatementDatabaseMetaDataTest { + + @Override + public Connection getJdbcConnection(Properties properties) throws SQLException { + Properties props = properties == null ? new Properties() : (Properties) properties.clone(); + props.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); + return super.getJdbcConnection(props); + } + + @Override + public void testAllTableEnginesFromSystemTableEnginesAreMapped() { + throw new SkipException("Reads through a plain Statement, which cannot consume TabSeparated"); + } + + @Override + @Test(groups = { "integration" }, dataProvider = "columnNameLikePatterns") + public void testGetColumnsMatchesColumnNamePatternLikeServer(String pattern, String columnName, boolean expected) { + throw new SkipException("Reads through a plain PreparedStatement, which cannot consume TabSeparated"); + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java deleted file mode 100644 index d1abd63dd..000000000 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/ShowStatementsMetaDataLikeMatcherTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.clickhouse.jdbc.metadata; - -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import static org.testng.Assert.assertEquals; - -public class ShowStatementsMetaDataLikeMatcherTest { - - @DataProvider(name = "likePatterns") - public Object[][] likePatterns() { - return new Object[][] { - {null, "any_name", true}, - {null, "", true}, - {"", "", true}, - {"", "a", false}, - {"%", "", true}, - {"%", "a\nb", true}, - {"abc", "abc", true}, - {"abc", "ABC", false}, - {"abc", "abcd", false}, - {"a_c", "abc", true}, - {"a_c", "ac", false}, - {"a%c", "ac", true}, - {"a%c", "a.b.c", true}, - {"a%c", "acd", false}, - {"a\\_c", "a_c", true}, - {"a\\_c", "abc", false}, - {"100\\%", "100%", true}, - {"100\\%", "1000", false}, - {"a\\\\b", "a\\b", true}, - {"a.c", "abc", false}, - {"(x)[y]*", "(x)[y]*", true}, - {"_", "😀", true}, - }; - } - - @Test(groups = {"unit"}, dataProvider = "likePatterns") - public void testLikeMatcher(String pattern, String value, boolean expected) { - assertEquals(ShowStatementsMetaData.likeMatcher(pattern).test(value), expected, - "'" + value + "' LIKE '" + pattern + "'"); - } -}