From ffedc5b740cfe4e4d0ca1a0faeb204407efb39d3 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:22:07 +0200 Subject: [PATCH 1/6] Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301) Changes of unmerged https://github.com/Altinity/ClickHouse/pull/2301 at e44403cb1372d1bc010a4f0f80f0f301afc80798 Conflicts committed as-is; the next commit resolves them: - src/DataTypes/DataTypeAggregateFunction.cpp (both modified) - src/Formats/FormatSettings.h (both modified) - src/Processors/Formats/Impl/Parquet/Decoding.cpp (both modified) - src/Processors/Formats/Impl/Parquet/SchemaConverter.h (both modified) - src/Processors/Formats/Impl/Parquet/Write.h (both modified) - src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp (both modified) - src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp (both modified) --- .../table-engines/integrations/iceberg.mdx | 62 +++++ docs/reference/formats/Parquet/Parquet.mdx | 35 +++ src/Core/Settings.cpp | 10 + src/Core/SettingsChangesHistory.cpp | 1 + src/DataTypes/DataTypeAggregateFunction.cpp | 220 +++++++++++++++++- src/DataTypes/DataTypeAggregateFunction.h | 16 +- .../gtest_annotated_type_matches_derived.cpp | 128 ++++++++++ .../gtest_ast_has_aggregate_function_type.cpp | 42 ++++ src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 4 + .../Formats/Impl/Parquet/Decoding.cpp | 33 +++ .../Formats/Impl/Parquet/Decoding.h | 15 ++ .../Formats/Impl/Parquet/PrepareForWrite.cpp | 43 ++++ .../Formats/Impl/Parquet/SchemaConverter.cpp | 132 ++++++++++- .../Formats/Impl/Parquet/SchemaConverter.h | 9 + .../Formats/Impl/Parquet/ThriftUtil.h | 3 + src/Processors/Formats/Impl/Parquet/Write.cpp | 26 +++ src/Processors/Formats/Impl/Parquet/Write.h | 4 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 4 + .../Impl/ParquetV3BlockInputFormat.cpp | 8 + ...portReplicatedMergeTreePartitionManifest.h | 7 + .../ExportPartitionTaskScheduler.cpp | 8 +- .../MergeTree/ExportPartitionUtils.cpp | 3 + .../MergeTree/MergeTreePartitionExportTask.h | 4 + .../tests/gtest_export_partition_ordering.cpp | 89 +++++++ .../DataLakes/Iceberg/Constant.h | 1 + .../DataLakes/Iceberg/DataFileStatistics.cpp | 38 ++- .../DataLakes/Iceberg/DataFileStatistics.h | 5 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 100 ++++++-- .../DataLakes/Iceberg/IcebergMetadata.h | 27 ++- .../DataLakes/Iceberg/MetadataGenerator.cpp | 88 +++++++ .../DataLakes/Iceberg/SchemaProcessor.cpp | 93 +++++++- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 5 + .../gtest_iceberg_metadata_generator.cpp | 106 ++++++++- .../tests/gtest_iceberg_schema_processor.cpp | 129 ++++++++++ .../tests/gtest_iceberg_type_mapping.cpp | 63 +++++ src/Storages/StorageMergeTree.cpp | 3 + src/Storages/StorageReplicatedMergeTree.cpp | 3 + .../helpers/export_partition_helpers.py | 16 +- .../test.py | 49 ++++ .../test_schema_match.py | 44 ++++ .../test_aggregate_function_states.py | 210 +++++++++++++++++ ...parquet_aggregate_function_state.reference | 87 +++++++ .../04673_parquet_aggregate_function_state.sh | 219 +++++++++++++++++ 44 files changed, 2141 insertions(+), 53 deletions(-) create mode 100644 src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp create mode 100644 src/DataTypes/tests/gtest_ast_has_aggregate_function_type.cpp create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py create mode 100644 tests/queries/0_stateless/04673_parquet_aggregate_function_state.reference create mode 100755 tests/queries/0_stateless/04673_parquet_aggregate_function_state.sh diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index 7a05dab165b5d..c39f0ba1bdf0c 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -106,6 +106,68 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `map` | `Map` | | `struct` | `Tuple` | +### Aggregate function states {#aggregate-function-states} + +Iceberg has no aggregate-state type, so ClickHouse stores the two aggregate-state types as ordinary +Iceberg values and records the ClickHouse type name in a `clickhouse.type` key on the schema field: + +| ClickHouse type | Iceberg type | Stored as | +|---|---|---| +| `AggregateFunction(f, T...)` | `binary` | The serialized state, the same bytes `f` writes with the `-State` combinator | +| `SimpleAggregateFunction(f, T)` | whatever `T` maps to | An ordinary value of type `T` | + +Other query engines ignore the key and see a plain `binary` (or `T`-typed) column. + +Creating such a table requires +[`allow_experimental_aggregate_function_states_in_open_formats`](/operations/settings/settings#allow_experimental_aggregate_function_states_in_open_formats), +and so does every query that reads a column whose `clickhouse.type` names an `AggregateFunction` - an +`INSERT` or an `ALTER TABLE ... EXPORT PART` as much as a `SELECT`. A state is an opaque blob handed +to the deserializer of the aggregate function the table's metadata names, so with the setting enabled +it is that metadata, not the query, which chooses the deserializer; keep it disabled for tables from +untrusted sources, where such a field is then rejected rather than read as `String`. A +`SimpleAggregateFunction` column is not gated on read, holding ordinary values of its storage type. + +The setting is read from the query that parses the schema, so a session `SET` or a `SETTINGS` clause +on that query supplies it. The `iceberg*` table functions build a fresh storage object per query, so +for them it takes effect per query; the table engine parses the schema once and keeps it, so the +query that first touches the table after `ATTACH` decides, and that outcome holds - including for +queries that do not set it - until `DETACH TABLE` or a server restart. + +Writing goes through the ordinary Parquet writer, which takes the same setting from its format +settings. An object storage table engine freezes its format settings at `CREATE TABLE` - the server +settings plus that query's `SETTINGS` clause, session settings ignored - so for `INSERT` the setting +must also be in that clause; given only on the `INSERT`, it does not reach the writer: + +```sql +CREATE TABLE agg (k UInt32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64)) +ENGINE = IcebergLocal('/path/to/table/') +PARTITION BY k +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; + +-- The states merge exactly as they do in an AggregatingMergeTree table. +SELECT k, uniqMerge(u), sum(s) FROM agg GROUP BY k +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; +``` + +`ALTER TABLE ... EXPORT PART` and `ALTER TABLE ... EXPORT PARTITION` from an `AggregatingMergeTree` +table into such an Iceberg table work as well, so a partition of pre-aggregated states can be moved +into the lake without finalizing it. Both take the setting off the `ALTER` query itself: + +```sql +ALTER TABLE mt EXPORT PART 'all_1_1_0' TO TABLE agg +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; + +-- EXPORT PARTITION, which only ReplicatedMergeTree implements, records the value in its manifest, +-- so every replica executing the task applies it in place of its own profile. +ALTER TABLE rmt EXPORT PARTITION ID '1' TO TABLE agg +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; +``` + +The Parquet footer of each data file carries the same information, so `DESCRIBE file('data.parquet')` +reports the aggregate types too; that path goes through Parquet schema inference, gated by the same +setting. See +[aggregate function states in Parquet](/interfaces/formats/Parquet#aggregate-function-states). + ## Schema evolution {#schema-evolution} ClickHouse supports reading Iceberg tables whose schema has evolved over time. This includes tables where columns have been added, removed, or reordered, as well as columns changed from required to nullable. Additionally, the following type casts are supported: diff --git a/docs/reference/formats/Parquet/Parquet.mdx b/docs/reference/formats/Parquet/Parquet.mdx index ef6cd249d41dc..45812fb206506 100644 --- a/docs/reference/formats/Parquet/Parquet.mdx +++ b/docs/reference/formats/Parquet/Parquet.mdx @@ -115,6 +115,41 @@ On write, top-level columns of type `Point`, `MultiPoint`, `LineString`, `Polygo Geometry columns must appear at the root of the schema or nested inside `Tuple` (`struct`); nesting them inside `Array` or `Map` is not supported. `Nullable` is not supported for geo columns either. +## Aggregate function states {#aggregate-function-states} + +Parquet has no aggregate-state type, so ClickHouse writes an [`AggregateFunction`](/sql-reference/data-types/aggregatefunction.md) column as a plain `BYTE_ARRAY` holding the serialized state - the same bytes the `-State` combinator produces - with no `STRING`/`UTF8` logical type, since a state is arbitrary binary data rather than text. A [`SimpleAggregateFunction(f, T)`](/sql-reference/data-types/simpleaggregatefunction.md) column is written as an ordinary value of type `T`. + +Neither type can be recovered from the Parquet schema alone: every `AggregateFunction` state is just a binary column, and a `SimpleAggregateFunction` is indistinguishable from its storage type. ClickHouse therefore records the type names in a `clickhouse.column_types` key in the file-level Parquet metadata, as a JSON object mapping column name to ClickHouse type name. The recorded name includes the state version, which is what pins the serialized layout across server versions. + +Both writing an `AggregateFunction` column and reconstructing one from that metadata on read are gated by [`allow_experimental_aggregate_function_states_in_open_formats`](/operations/settings/settings#allow_experimental_aggregate_function_states_in_open_formats), which is disabled by default. A state is an opaque blob passed to the deserializer of whichever aggregate function the file names, so with the setting enabled it is the file, not the query, choosing that deserializer; keep it disabled for files from untrusted sources, where such a file is then rejected rather than read as `String`. With it disabled, writing is refused with `UNKNOWN_TYPE`, exactly as in versions that did not support states in Parquet at all. `SimpleAggregateFunction` is gated in neither direction: it is stored as an ordinary value of its storage type and has always been written that way. + +With the setting enabled the states round-trip without any hint: + +```sql +SET allow_experimental_aggregate_function_states_in_open_formats = 1; + +INSERT INTO FUNCTION file('states.parquet') +SELECT k, uniqState(v) AS u, sumSimpleState(v) AS s FROM source GROUP BY k; + +DESCRIBE file('states.parquet'); +-- k UInt64 +-- u AggregateFunction(uniq, UInt64) +-- s SimpleAggregateFunction(sum, UInt64) + +SELECT k, uniqMerge(u), sum(s) FROM file('states.parquet') GROUP BY k; +``` + +An explicit structure overrides the recorded metadata and is unaffected by the setting, so a state column can also be read as `String` to get the raw serialized bytes: + +```sql +SELECT uniqMerge(CAST(u AS AggregateFunction(uniq, UInt64))) +FROM file('states.parquet', Parquet, 'u String'); +``` + +Reading fails rather than guessing if the recorded type does not describe the data actually in the file. Min/max statistics are never used for pruning a state column, because bounds over serialized states are meaningless. + +Other query engines see a plain binary (or `T`-typed) column and ignore the metadata key. The same mechanism backs [aggregate-state support in Iceberg tables](/engines/table-engines/integrations/iceberg#aggregate-function-states), whose data files are Parquet. + ## Example usage {#example-usage} ### Inserting data {#inserting-data} diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 438ceb7a543db..d7dea69f88d6f 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8133,6 +8133,16 @@ Query Iceberg table using the specific snapshot id. )", 0) \ DECLARE(Bool, allow_experimental_geo_types_in_iceberg, false, R"( Allow parsing Iceberg `geometry` and `geography` field types as ClickHouse `Geometry` (Variant) type. +)", 0) \ + DECLARE(Bool, allow_experimental_aggregate_function_states_in_open_formats, false, R"( +Allow `AggregateFunction` states in Parquet files and Iceberg tables, and `SimpleAggregateFunction` +columns in Iceberg tables. A state is stored as opaque binary, with its ClickHouse type recorded in +`clickhouse.column_types` (Parquet file metadata) or `clickhouse.type` (Iceberg schema field). + +Gates both writing such columns and reconstructing their types on read. When enabled, the data, not +the query, chooses the deserializer for the stored bytes, so keep it disabled for untrusted sources. +More [in Parquet](/interfaces/formats/Parquet#aggregate-function-states) and +[in Iceberg](/engines/table-engines/integrations/iceberg#aggregate-function-states). )", 0) \ DECLARE(Bool, show_data_lake_catalogs_in_system_tables, false, R"( Enables showing data lake catalogs in system tables. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a3142c12e5512..e25cd47ea2596 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -246,6 +246,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"datalake_ignore_unsupported_table_properties", false, false, "New setting: allow `CREATE TABLE` and `SHOW CREATE TABLE` in a `DataLakeCatalog` to omit unsupported table properties."}, {"object_storage_cluster_fallback_to_local_if_empty", false, false, "New setting"}, {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, + {"allow_experimental_aggregate_function_states_in_open_formats", false, false, "New setting gating aggregate function states in Parquet files and Iceberg tables. Disabled by default, so writing such a Parquet column keeps throwing `UNKNOWN_TYPE` and such an Iceberg column keeps being refused with `SUPPORT_IS_DISABLED` as in versions without the feature."}, }); addSettingsChanges(settings_changes_history, "26.6", { diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 74d8401b2208b..35e11af5154fd 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -6,11 +6,23 @@ #include #include +#include #include #include #include #include +<<<<<<< HEAD +======= +#include +#include +#include +#include +#include +#include +#include +#include +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include #include @@ -22,7 +34,9 @@ #include #include +#include #include +#include #include #include @@ -56,13 +70,19 @@ String DataTypeAggregateFunction::getFunctionName() const String DataTypeAggregateFunction::doGetName() const { - return getNameImpl(true); + return getNameImpl(true, false); } String DataTypeAggregateFunction::getNameWithoutVersion() const { - return getNameImpl(false); + return getNameImpl(false, false); +} + + +String DataTypeAggregateFunction::getNameForAnnotation() const +{ + return getNameImpl(true, true); } @@ -88,14 +108,23 @@ bool DataTypeAggregateFunction::isVersioned() const return function->isVersioned(); } +<<<<<<< HEAD String DataTypeAggregateFunction::getNameImpl(bool with_version) const +======= +void DataTypeAggregateFunction::updateVersionFromRevision(size_t revision, bool if_empty) const +{ + setVersion(function->getVersionFromRevision(revision), if_empty); +} + +String DataTypeAggregateFunction::getNameImpl(bool with_version, bool always_emit_version) const +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { WriteBufferFromOwnString stream; stream << "AggregateFunction("; - /// If aggregate function does not support versioning its version is 0 and is not printed. + /// Version 0 is normally omitted, but annotations must distinguish it from the default version. auto data_type_version = getVersion(); - if (with_version && data_type_version) + if (with_version && (data_type_version || (always_emit_version && isVersioned()))) stream << data_type_version << ", "; stream << function->getName(); @@ -537,4 +566,187 @@ bool hasAggregateFunctionType(const DataTypePtr & type) return result; } +bool astHasAggregateFunctionType(const ASTPtr & ast) +{ + std::string_view name; + if (const auto * data_type = ast->as()) + name = data_type->name; + else if (const auto * identifier = ast->as()) + name = identifier->name(); + + if (name == "AggregateFunction") + return true; + + for (const auto & child : ast->children) + if (astHasAggregateFunctionType(child)) + return true; + + return false; +} + +bool needsClickHouseTypeAnnotation(const DataTypePtr & type) +{ + auto result = false; + auto check = [&](const IDataType & t) + { + result |= WhichDataType(t).isAggregateFunction() + || typeid_cast(t.getCustomName()) != nullptr; + }; + + check(*type); + type->forEachChild(check); + return result; +} + +String getClickHouseTypeAnnotationName(const DataTypePtr & type) +{ + if (type->getCustomName()) + return type->getName(); + + switch (type->getTypeId()) + { + case TypeIndex::AggregateFunction: + return assert_cast(*type).getNameForAnnotation(); + case TypeIndex::Array: + return "Array(" + getClickHouseTypeAnnotationName(assert_cast(*type).getNestedType()) + ")"; + case TypeIndex::Map: + { + const auto & map_type = assert_cast(*type); + return "Map(" + getClickHouseTypeAnnotationName(map_type.getKeyType()) + ", " + + getClickHouseTypeAnnotationName(map_type.getValueType()) + ")"; + } + case TypeIndex::Tuple: + { + const auto & tuple_type = assert_cast(*type); + const auto & elements = tuple_type.getElements(); + const auto & names = tuple_type.getElementNames(); + WriteBufferFromOwnString stream; + stream << "Tuple("; + for (size_t i = 0; i < elements.size(); ++i) + { + if (i) + stream << ", "; + if (tuple_type.hasExplicitNames()) + stream << backQuoteIfNeed(names[i]) << ' '; + stream << getClickHouseTypeAnnotationName(elements[i]); + } + stream << ")"; + return stream.str(); + } + default: + return type->getName(); + } +} + +namespace +{ + +bool isFixedStringOfSize(const DataTypePtr & type, size_t size) +{ + const auto * fixed_string = typeid_cast(type.get()); + return fixed_string && fixed_string->getN() == size; +} + +bool isDateTime64WithScale(const DataTypePtr & type, UInt32 scale) +{ + const auto * date_time64 = typeid_cast(type.get()); + return date_time64 && date_time64->getScale() == scale; +} + +/// Mirrors the non-injective mappings in Parquet's `preparePrimitiveColumn`. +bool parquetWriterCouldProduce(const DataTypePtr & annotated, const DataTypePtr & derived) +{ + if (annotated->equals(*derived)) + return true; + + auto next_timestamp_unit = [](UInt32 scale) -> UInt32 + { + if (scale <= 3) + return 3; + if (scale <= 6) + return 6; + return 9; + }; + + switch (annotated->getTypeId()) + { + case TypeIndex::Date: + return WhichDataType(derived).isDate32() || WhichDataType(derived).isUInt16(); + case TypeIndex::DateTime: + return isDateTime64WithScale(derived, 3) || WhichDataType(derived).isUInt32(); + case TypeIndex::DateTime64: + return isDateTime64WithScale( + derived, next_timestamp_unit(assert_cast(*annotated).getScale())); + case TypeIndex::Time: + return isDateTime64WithScale(derived, 6); + case TypeIndex::Time64: + return isDateTime64WithScale( + derived, assert_cast(*annotated).getScale() <= 6 ? 6 : 9); + case TypeIndex::Enum8: + return isString(derived) || WhichDataType(derived).isInt8(); + case TypeIndex::Enum16: + return isString(derived) || WhichDataType(derived).isInt16(); + case TypeIndex::IPv4: + return WhichDataType(derived).isUInt32(); + case TypeIndex::IPv6: + case TypeIndex::UInt128: + case TypeIndex::Int128: + return isFixedStringOfSize(derived, 16); + case TypeIndex::UInt256: + case TypeIndex::Int256: + return isFixedStringOfSize(derived, 32); + case TypeIndex::FixedString: + return isString(derived); + case TypeIndex::Object: + return isString(derived); + default: + return false; + } +} + +} + +bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataTypePtr & derived_type, bool strict) +{ + auto annotated = removeLowCardinalityAndNullable(annotated_type); + auto derived = removeLowCardinalityAndNullable(derived_type); + + if (annotated->getTypeId() == TypeIndex::AggregateFunction) + return isString(derived); + + switch (annotated->getTypeId()) + { + case TypeIndex::Array: + { + const auto * derived_array = typeid_cast(derived.get()); + if (!derived_array) + return false; + return annotatedTypeMatchesDerived( + assert_cast(*annotated).getNestedType(), derived_array->getNestedType(), strict); + } + case TypeIndex::Tuple: + { + const auto & annotated_tuple = assert_cast(*annotated); + const auto * derived_tuple = typeid_cast(derived.get()); + if (!derived_tuple || derived_tuple->getElements().size() != annotated_tuple.getElements().size()) + return false; + for (size_t i = 0; i < annotated_tuple.getElements().size(); ++i) + if (!annotatedTypeMatchesDerived(annotated_tuple.getElement(i), derived_tuple->getElement(i), strict)) + return false; + return true; + } + case TypeIndex::Map: + { + const auto & annotated_map = assert_cast(*annotated); + const auto * derived_map = typeid_cast(derived.get()); + if (!derived_map) + return false; + return annotatedTypeMatchesDerived(annotated_map.getKeyType(), derived_map->getKeyType(), strict) + && annotatedTypeMatchesDerived(annotated_map.getValueType(), derived_map->getValueType(), strict); + } + default: + return !strict || parquetWriterCouldProduce(annotated, derived); + } +} + } diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h index 6774c89ae812b..f208be37dd824 100644 --- a/src/DataTypes/DataTypeAggregateFunction.h +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -24,7 +25,7 @@ class DataTypeAggregateFunction final : public IDataType Array parameters; std::optional version; - String getNameImpl(bool with_version) const; + String getNameImpl(bool with_version, bool always_emit_version) const; public: static constexpr bool is_parametric = true; @@ -43,6 +44,8 @@ class DataTypeAggregateFunction final : public IDataType String doGetName() const override; String getNameWithoutVersion() const; + /// Unlike `getName`, keeps an explicit version 0. + String getNameForAnnotation() const; const char * getFamilyName() const override { return "AggregateFunction"; } TypeIndex getTypeId() const override { return TypeIndex::AggregateFunction; } @@ -102,4 +105,15 @@ void pinCurrentStateVersionToAggregateFunctions(DataTypePtr & type); /// Checks type of any nested type is DataTypeAggregateFunction. bool hasAggregateFunctionType(const DataTypePtr & type); +/// Checks a parsed type name without resolving aggregate functions through `DataTypeFactory`. +bool astHasAggregateFunctionType(const ASTPtr & ast); + +bool needsClickHouseTypeAnnotation(const DataTypePtr & type); + +/// Returns an annotation name that preserves explicit aggregate-state versions. +String getClickHouseTypeAnnotationName(const DataTypePtr & type); + +/// Checks an annotation against the schema-derived type. Strict mode also verifies the Parquet mapping. +bool annotatedTypeMatchesDerived(const DataTypePtr & annotated, const DataTypePtr & derived, bool strict = false); + } diff --git a/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp b/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp new file mode 100644 index 0000000000000..3c1cd1a782424 --- /dev/null +++ b/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp @@ -0,0 +1,128 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +DataTypePtr typeFromString(const String & name) +{ + tryRegisterAggregateFunctions(); + return DataTypeFactory::instance().get(name); +} + +bool matches(const String & annotated, const String & derived) +{ + return annotatedTypeMatchesDerived(typeFromString(annotated), typeFromString(derived)); +} + +bool matchesStrictly(const String & annotated, const String & derived) +{ + return annotatedTypeMatchesDerived(typeFromString(annotated), typeFromString(derived), /*strict=*/ true); +} + +} + +TEST(AnnotatedTypeMatchesDerived, AggregateFunctionNeedsAStringPosition) +{ + EXPECT_TRUE(matches("AggregateFunction(uniq, UInt64)", "String")); + EXPECT_TRUE(matches("AggregateFunction(uniq, UInt64)", "Nullable(String)")); + + EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "Int64")); + EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "FixedString(16)")); + EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "Array(String)")); + + EXPECT_TRUE(matchesStrictly("AggregateFunction(uniq, UInt64)", "String")); + EXPECT_FALSE(matchesStrictly("AggregateFunction(uniq, UInt64)", "Int64")); +} + +TEST(AnnotatedTypeMatchesDerived, ContainersAreMatchedElementwise) +{ + EXPECT_TRUE(matches("Array(AggregateFunction(uniq, UInt64))", "Array(String)")); + EXPECT_TRUE(matches("Array(AggregateFunction(uniq, UInt64))", "Array(Nullable(String))")); + EXPECT_FALSE(matches("Array(AggregateFunction(uniq, UInt64))", "Array(Int64)")); + + EXPECT_TRUE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Map(String, String)")); + EXPECT_FALSE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Map(String, Int64)")); + + EXPECT_TRUE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a String, b Int64)")); + EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a Int64, b String)")); + EXPECT_FALSE(matches("Array(AggregateFunction(uniq, UInt64))", "String")); + EXPECT_FALSE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Array(Tuple(String, String))")); + EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "String")); + EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a String)")); + + EXPECT_TRUE(matches("SimpleAggregateFunction(sum, UInt64)", "Int64")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Nullable(String))", "Nullable(String)")); + EXPECT_TRUE(matches("Array(SimpleAggregateFunction(sum, UInt64))", "Array(Int64)")); +} + +TEST(AnnotatedTypeMatchesDerived, StrictModeRefusesAnAnnotationThatRetypesTheColumn) +{ + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(9))", "Int64")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "Int64")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(9))", "Int64")); + + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "Int32")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv4)", "Int32")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Float64)", "Int64")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, String)", "FixedString(16)")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 2)")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(3)")); + EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(9)")); + + EXPECT_FALSE(matchesStrictly( + "Tuple(a AggregateFunction(uniq, UInt64), b DateTime64(9))", "Tuple(a String, b Int64)")); + EXPECT_FALSE(matchesStrictly("Array(SimpleAggregateFunction(sum, UInt64))", "Array(Int64)")); + EXPECT_FALSE(matchesStrictly( + "Map(String, SimpleAggregateFunction(anyLast, DateTime64(9)))", "Map(String, Int64)")); +} + +TEST(AnnotatedTypeMatchesDerived, StrictModeAcceptsWhatTheParquetWriterProduces) +{ + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "UInt64")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, String)", "String")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, UUID)", "UUID")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 4)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, FixedString(16))", "FixedString(16)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date32)", "Date32")); + + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "Nullable(UInt64)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Nullable(String))", "String")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Nullable(String))", "Nullable(String)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Array(UInt64))", "Array(Nullable(UInt64))")); + EXPECT_TRUE(matchesStrictly( + "SimpleAggregateFunction(anyLast, Map(String, UInt64))", "Map(String, Nullable(UInt64))")); + EXPECT_TRUE(matchesStrictly( + "Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a Nullable(String), b Nullable(UInt64))")); + + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, LowCardinality(String))", "String")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, LowCardinality(String))", "Nullable(String)")); + EXPECT_TRUE(matchesStrictly( + "SimpleAggregateFunction(anyLast, LowCardinality(Nullable(String)))", "Nullable(String)")); + + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "Date32")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "UInt16")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime)", "DateTime64(3, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime)", "UInt32")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(9))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(0))", "DateTime64(3, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(7))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time)", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time64(3))", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time64(9))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "String")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "Int8")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "String")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "Int16")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv4)", "UInt32")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv6)", "FixedString(16)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Int128)", "FixedString(16)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, UInt256)", "FixedString(32)")); + EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, FixedString(16))", "String")); +} diff --git a/src/DataTypes/tests/gtest_ast_has_aggregate_function_type.cpp b/src/DataTypes/tests/gtest_ast_has_aggregate_function_type.cpp new file mode 100644 index 0000000000000..6c20941c88f15 --- /dev/null +++ b/src/DataTypes/tests/gtest_ast_has_aggregate_function_type.cpp @@ -0,0 +1,42 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +bool hasState(const String & type_name) +{ + ParserDataType parser; + ASTPtr ast = parseQuery( + parser, type_name.data(), type_name.data() + type_name.size(), "data type", + /*max_query_size=*/ 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + return astHasAggregateFunctionType(ast); +} + +} + +TEST(AstHasAggregateFunctionType, FindsOnlyAggregateFunctionTypeNames) +{ + for (const auto * name : { + "AggregateFunction(uniq, UInt64)", + "AggregateFunction(0, sumMap, Array(UInt8), Array(UInt32))", + "Array(AggregateFunction(no_such_aggregate_function, UInt64))", + "Map(String, AggregateFunction(uniq, UInt64))", + "Tuple(a UInt64, b Array(AggregateFunction(uniq, UInt64)))"}) + EXPECT_TRUE(hasState(name)) << name; + + for (const auto * name : { + "UInt64", + "SimpleAggregateFunction(sum, UInt64)", + "Array(SimpleAggregateFunction(max, String))", + "Tuple(a UInt64, b Array(Nullable(String)))", + "Enum8('AggregateFunction' = 1)", + "Tuple(`AggregateFunction` String)"}) + EXPECT_FALSE(hasState(name)) << name; +} diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index e01db68fd2175..f4a663f3b339f 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -54,6 +54,7 @@ FORMAT_FACTORY_SETTINGS(DECLARE_FORMAT_EXTERN, INITIALIZE_SETTING_EXTERN) extern const SettingsAggregateFunctionInputFormat aggregate_function_input_format; extern const SettingsBool allow_special_serialization_kinds_in_output_formats; extern const SettingsBool allow_experimental_nullable_tuple_type; + extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; extern SettingsGeoJSONUnsupportedGeometryHandling input_format_geojson_unsupported_geometry_handling; extern SettingsBool format_geojson_validate_geometry; @@ -265,6 +266,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.verify_checksums = settings[Setting::input_format_parquet_verify_checksums]; format_settings.parquet.local_time_as_utc = settings[Setting::input_format_parquet_local_time_as_utc]; format_settings.parquet.allow_geoparquet_parser = settings[Setting::input_format_parquet_allow_geoparquet_parser]; + format_settings.parquet.allow_aggregate_function_states = settings[Setting::allow_experimental_aggregate_function_states_in_open_formats]; format_settings.parquet.write_geometadata = settings[Setting::output_format_parquet_geometadata]; if (auto memory_limit = total_memory_tracker.getHardLimit(); memory_limit > 0) { diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 3afc683876923..91d0b7556dd30 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -420,7 +420,11 @@ struct FormatSettings double bloom_filter_bits_per_value = 10.5; size_t bloom_filter_flush_threshold_bytes = 1024 * 1024 * 128; bool allow_geoparquet_parser = true; +<<<<<<< HEAD bool spatial_filter_push_down = true; +======= + bool allow_aggregate_function_states = false; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) bool write_geometadata = true; size_t max_dictionary_size = 1024 * 1024; } parquet{}; diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.cpp b/src/Processors/Formats/Impl/Parquet/Decoding.cpp index ca7b8f5057a35..1d43a86dd8858 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.cpp +++ b/src/Processors/Formats/Impl/Parquet/Decoding.cpp @@ -1,10 +1,17 @@ #include #include +#include #include +#include +#include #include +<<<<<<< HEAD #include #include +======= +#include +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include @@ -1998,6 +2005,32 @@ void Int96Converter::convertColumn(std::span data, size_t num_values } } +void AggregateFunctionStateConverter::convertColumn(std::span chars, const UInt64 * offsets, size_t separator_bytes, size_t num_values, IColumn & col) const +{ + auto & column = assert_cast(col); + + /// Plain pages may follow a dictionary page whose states are shared through `src`. + column.ensureOwnership(); + + chassert(chars.size() >= offsets[num_values - 1]); + const FormatSettings format_settings; + for (ssize_t i = 0; i < ssize_t(num_values); ++i) + { + const char * ptr = chars.data() + offsets[i - 1]; + const size_t length = offsets[i] - offsets[i - 1] - separator_bytes; + ReadBufferFromMemory in(ptr, length); + serialization->deserializeBinary(column, in, format_settings); + if (!in.eof()) + { + column.popBack(1); + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Aggregate function state has {} trailing byte(s) after deserialization", + in.available()); + } + } +} + void GeoConverter::convertColumn(std::span chars, const UInt64 * offsets, size_t separator_bytes, size_t num_values, IColumn & col) const { col.reserve(col.size() + num_values); diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.h b/src/Processors/Formats/Impl/Parquet/Decoding.h index e0533fd295073..4aed66a4603e6 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.h +++ b/src/Processors/Formats/Impl/Parquet/Decoding.h @@ -4,6 +4,11 @@ #include #include +namespace DB +{ +class ISerialization; +} + namespace DB::ErrorCodes { extern const int INCORRECT_DATA; @@ -413,6 +418,16 @@ struct GeoConverter : public StringConverter void convertColumn(std::span chars, const UInt64 * offsets, size_t separator_bytes, size_t num_values, IColumn & col) const override; }; +struct AggregateFunctionStateConverter : public StringConverter +{ + std::shared_ptr serialization; + + explicit AggregateFunctionStateConverter(std::shared_ptr serialization_) + : serialization(std::move(serialization_)) {} + + void convertColumn(std::span chars, const UInt64 * offsets, size_t separator_bytes, size_t num_values, IColumn & col) const override; +}; + void decodeRepOrDefLevels(parq::Encoding::type encoding, UInt8 max, size_t num_values, std::span data, PaddedPODArray & out); diff --git a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp index ad23da3dc50d6..50fb435b071ef 100644 --- a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp +++ b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp @@ -27,6 +27,9 @@ #include #include #include +#include +#include +#include /// This file deals with schema conversion and with repetition and definition levels. @@ -908,6 +911,28 @@ void validateIcebergFieldIds( } } +void convertAggregateFunctionColumnToString(ColumnPtr & column, DataTypePtr & type) +{ + auto serialization = type->getDefaultSerialization(); + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.reserve(column->size()); + { + WriteBufferFromVector buffer(chars); + const FormatSettings format_settings; + for (size_t i = 0; i < column->size(); ++i) + { + serialization->serializeBinary(*column, i, buffer, format_settings); + offsets.push_back(buffer.count()); + } + buffer.finalize(); + } + + column = std::move(result); + type = std::make_shared(); +} + void prepareColumnRecursive( ColumnPtr column, DataTypePtr type, const std::string & name, const WriteOptions & options, ColumnChunkWriteStates & states, SchemaElements & schemas, const std::optional> & column_field_ids, @@ -934,6 +959,24 @@ void prepareColumnRecursive( preparePrimitiveColumn(column, nested_type, name, options, states, schemas, lookupLeafFieldId(column_field_ids, name)); break; } + case TypeIndex::AggregateFunction: + { + if (!options.allow_aggregate_function_states) + throw Exception( + ErrorCodes::UNKNOWN_TYPE, + "Internal type '{}' of column '{}' is not supported for conversion into Parquet data format. " + "Enable setting allow_experimental_aggregate_function_states_in_open_formats to write the " + "serialized states", + type->getFamilyName(), name); + + convertAggregateFunctionColumnToString(column, type); + /// Serialized states are binary even when Parquet strings use the UTF8 logical type. + WriteOptions binary_options = options; + binary_options.output_string_as_string = false; + preparePrimitiveColumn( + column, type, name, binary_options, states, schemas, lookupLeafFieldId(column_field_ids, name)); + break; + } default: preparePrimitiveColumn(column, type, name, options, states, schemas, lookupLeafFieldId(column_field_ids, name)); break; diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index bf5cd131f135e..192e8b3c793fc 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -19,6 +19,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include @@ -57,6 +62,38 @@ SchemaConverter::SchemaConverter( } } } + + /// An explicit structure takes precedence over type annotations in the footer. + if (!sample_block) + { + for (const auto & kv : file_metadata.key_value_metadata) + { + if (kv.key != clickhouse_column_types_key) + continue; + + try + { + Poco::JSON::Parser parser; + const auto object = parser.parse(kv.value).extract(); + for (const auto & name : object->getNames()) + annotated_column_type_names[name] = object->getValue(name); + } + catch (Exception & e) + { + e.addMessage("while parsing the `{}` key-value metadata of the parquet file", clickhouse_column_types_key); + throw; + } + catch (const Poco::Exception & e) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Cannot parse the `{}` key-value metadata of the parquet file: {}", + clickhouse_column_types_key, + e.displayText()); + } + break; + } + } } void SchemaConverter::checkHasColumns() @@ -127,6 +164,59 @@ void SchemaConverter::prepareForReading() } +DataTypePtr SchemaConverter::resolveAnnotatedType(const String & column_name, const String & type_name, const DataTypePtr & derived_type) const +{ + ASTPtr ast; + try + { + ParserDataType parser; + ast = parseQuery( + parser, type_name.data(), type_name.data() + type_name.size(), "data type", + /*max_query_size=*/ 0, options.format.max_parser_depth, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + } + catch (Exception & e) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet file records ClickHouse type {} for column {} in its `{}` metadata, which is not a " + "valid type name: {}", + type_name, column_name, clickhouse_column_types_key, e.message()); + } + + /// Check the gate before resolving a file-provided aggregate function name. + if (astHasAggregateFunctionType(ast) && !options.format.parquet.allow_aggregate_function_states) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet file records ClickHouse type {} for column {} in its `{}` metadata. Inferring " + "aggregate function states from parquet metadata is disabled: enable setting " + "allow_experimental_aggregate_function_states_in_open_formats to honour the " + "recorded type, or pass the structure explicitly", + type_name, column_name, clickhouse_column_types_key); + + DataTypePtr annotated_type; + try + { + annotated_type = DataTypeFactory::instance().get(ast); + } + catch (Exception & e) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet file records ClickHouse type {} for column {} in its `{}` metadata, which this " + "server cannot resolve: {}", + type_name, column_name, clickhouse_column_types_key, e.message()); + } + + if (!annotatedTypeMatchesDerived(annotated_type, derived_type, /*strict=*/ true)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet file records ClickHouse type {} for column {} in its `{}` metadata, but the " + "parquet schema for that column reads as {}", + annotated_type->getName(), column_name, clickhouse_column_types_key, derived_type->getName()); + + return annotated_type; +} + NamesAndTypesList SchemaConverter::inferSchema() { chassert(!sample_block); @@ -141,7 +231,27 @@ NamesAndTypesList SchemaConverter::inferSchema() if (node.output_idx.has_value()) { const OutputColumnInfo & col = output_columns.at(node.output_idx.value()); - res.emplace_back(col.name, col.output_type); + auto it = annotated_column_type_names.find(col.name); + if (it == annotated_column_type_names.end()) + { + res.emplace_back(col.name, col.output_type); + continue; + } + + DataTypePtr annotated_type; + try + { + annotated_type = resolveAnnotatedType(col.name, it->second, col.output_type); + } + catch (Exception & e) + { + if (options.format.parquet.skip_columns_with_unsupported_types_in_schema_inference + && (e.code() == ErrorCodes::INCORRECT_DATA || e.code() == ErrorCodes::NOT_IMPLEMENTED)) + continue; + throw; + } + + res.emplace_back(col.name, annotated_type); } } return res; @@ -443,11 +553,11 @@ bool SchemaConverter::processSubtreePrimitive(TraversalNode & node) } } - /// GeoParquet types like Point or Polygon can't be inside Nullable. - /// Geometry (Variant) is also not Nullable-compatible. + /// These types cannot be nested in `Nullable`. if (typeid_cast(inferred_type.get()) || typeid_cast(inferred_type.get()) - || typeid_cast(inferred_type.get())) + || typeid_cast(inferred_type.get()) + || typeid_cast(inferred_type.get())) { output_nullable = false; output_nullable_if_not_json = false; @@ -1064,6 +1174,20 @@ void SchemaConverter::processPrimitiveColumn( } } + if (typeid_cast(type_hint.get())) + { + if (type != parq::Type::BYTE_ARRAY) + throw Exception( + ErrorCodes::TYPE_MISMATCH, + "Column is requested as {} but its parquet physical type is {}, not BYTE_ARRAY", + type_hint->getName(), thriftToString(type)); + + out_inferred_type = type_hint; + out_decoder.allow_stats = false; + out_decoder.string_converter = std::make_shared(type_hint->getDefaultSerialization()); + return; + } + /// GeoParquet. /// Spec says "Geometry columns MUST be at the root of the schema", but we allow them to be /// nested in tuples etc, why not. (Though nesting in arrays/maps probably currently wouldn't diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h index fb7deeca8fafb..b8d92dfd30bf7 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h @@ -44,6 +44,7 @@ struct SchemaConverter /// The key is the parquet column name, without ColumnMapper. std::unordered_map geo_columns; +<<<<<<< HEAD /// If precomputed_geo_columns has a value it is used directly (including the empty-map case) /// and the constructor skips parsing the "geo" key-value metadata. This ensures that a /// failed parse caught by the caller (which leaves an empty map) does not cause @@ -51,6 +52,12 @@ struct SchemaConverter /// parse according to its own settings. SchemaConverter(const parq::FileMetaData &, const ReadOptions &, const Block *, std::optional> precomputed_geo_columns = std::nullopt); +======= + /// Type names from the `clickhouse.column_types` footer metadata. + std::unordered_map annotated_column_type_names; + + SchemaConverter(const parq::FileMetaData &, const ReadOptions &, const Block *); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) void prepareForReading(); NamesAndTypesList inferSchema(); @@ -134,6 +141,8 @@ struct SchemaConverter void checkHasColumns(); + DataTypePtr resolveAnnotatedType(const String & column_name, const String & type_name, const DataTypePtr & derived_type) const; + void processSubtree(TraversalNode & node); /// These functions are used by processSubtree for different kinds of SchemaElement. diff --git a/src/Processors/Formats/Impl/Parquet/ThriftUtil.h b/src/Processors/Formats/Impl/Parquet/ThriftUtil.h index 81292cb53dce2..b45401e76cca1 100644 --- a/src/Processors/Formats/Impl/Parquet/ThriftUtil.h +++ b/src/Processors/Formats/Impl/Parquet/ThriftUtil.h @@ -13,6 +13,9 @@ namespace DB::Parquet /// Namespace with structs generated from parquet.thrift namespace parq = parquet::format; +/// Maps top-level columns to ClickHouse types not represented by the Parquet schema. +constexpr const char * clickhouse_column_types_key = "clickhouse.column_types"; + /// All templates are explicitly instantiated, feel free to add more types. /// Returns number of bytes written. diff --git a/src/Processors/Formats/Impl/Parquet/Write.cpp b/src/Processors/Formats/Impl/Parquet/Write.cpp index 9b93f7599fb33..d53f1ab3447f5 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.cpp +++ b/src/Processors/Formats/Impl/Parquet/Write.cpp @@ -28,6 +28,8 @@ #include #include #include +#include +#include #if USE_SNAPPY #include @@ -1693,6 +1695,30 @@ void writeFileFooter(FileWriteState & file, } } + { + Poco::JSON::Object::Ptr column_types = new Poco::JSON::Object; + for (const auto & [column_name, type] : header.getNamesAndTypesList()) + { + if (!needsClickHouseTypeAnnotation(type)) + continue; + column_types->set(column_name, getClickHouseTypeAnnotationName(type)); + } + + if (column_types->size()) + { + std::ostringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss; + Poco::JSON::Stringifier::stringify(column_types, oss); + + parquet::format::KeyValue key_value; + key_value.__set_key(clickhouse_column_types_key); + key_value.__set_value(oss.str()); + + meta.key_value_metadata.push_back(std::move(key_value)); + meta.__isset.key_value_metadata = true; + } + } + size_t footer_size = serializeThriftStruct(meta, out); if (footer_size > INT32_MAX) diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index 0eef9bc59d482..2a7d2f60c4fc3 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -27,7 +27,11 @@ struct WriteOptions bool output_datetime_as_uint32 = false; bool output_date_as_uint16 = false; bool output_enum_as_byte_array = false; +<<<<<<< HEAD bool output_timestamp_adjusted_to_utc_from_timezone = false; +======= + bool allow_aggregate_function_states = false; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) /// Note: the meaning of some compression methods here is different from /// wrapReadBufferWithCompressionMethod: diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index eed31e78a7a51..fa2ddd8db999b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -55,7 +55,11 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea options.output_datetime_as_uint32 = format_settings.parquet.output_datetime_as_uint32; options.output_date_as_uint16 = format_settings.parquet.output_date_as_uint16; options.output_enum_as_byte_array = format_settings.parquet.output_enum_as_byte_array; +<<<<<<< HEAD options.output_timestamp_adjusted_to_utc_from_timezone = format_settings.parquet.output_timestamp_adjusted_to_utc_from_timezone; +======= + options.allow_aggregate_function_states = format_settings.parquet.allow_aggregate_function_states; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) options.data_page_size = format_settings.parquet.data_page_size; options.write_batch_size = format_settings.parquet.write_batch_size; options.write_page_index = format_settings.parquet.write_page_index; diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 8af873693eb52..c5d2679afbaf8 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -663,6 +663,7 @@ void registerParquetSchemaReader(FormatFactory & factory) [](const FormatSettings & settings) { return fmt::format( +<<<<<<< HEAD "schema_inference_make_columns_nullable={};schema_inference_make_json_columns_nullable={};" "enable_json_parsing={};max_parser_depth={};" "local_time_as_utc={};allow_geoparquet_parser={};skip_columns_with_unsupported_types={}", @@ -672,6 +673,13 @@ void registerParquetSchemaReader(FormatFactory & factory) settings.max_parser_depth, settings.parquet.local_time_as_utc, settings.parquet.allow_geoparquet_parser, +======= + "schema_inference_make_columns_nullable={};enable_json_parsing={};" + "allow_aggregate_function_states={};skip_columns_with_unsupported_types_in_schema_inference={}", + settings.schema_inference_make_columns_nullable, + settings.parquet.enable_json_parsing, + settings.parquet.allow_aggregate_function_states, +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) settings.parquet.skip_columns_with_unsupported_types_in_schema_inference); }); } diff --git a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h index dc4de190a9806..8da3e10b5b6fd 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h @@ -174,6 +174,7 @@ struct ExportReplicatedMergeTreePartitionManifest String filename_pattern; bool write_full_path_in_iceberg_metadata = false; bool allow_lossy_cast = false; + bool allow_aggregate_function_states_in_open_formats = false; String iceberg_metadata_json; /// Optional because of backwards compatibility @@ -222,6 +223,7 @@ struct ExportReplicatedMergeTreePartitionManifest json.set("task_timeout_seconds", task_timeout_seconds); json.set("write_full_path_in_iceberg_metadata", write_full_path_in_iceberg_metadata); json.set("allow_lossy_cast", allow_lossy_cast); + json.set("allow_aggregate_function_states_in_open_formats", allow_aggregate_function_states_in_open_formats); if (parquet_compression_method) json.set("parquet_compression_method", *parquet_compression_method); if (output_format_compression_level) @@ -303,6 +305,11 @@ struct ExportReplicatedMergeTreePartitionManifest /// on upgrade. New tasks always persist the initiator's actual choice. manifest.allow_lossy_cast = json->has("allow_lossy_cast") ? json->getValue("allow_lossy_cast") : true; + if (json->has("allow_aggregate_function_states_in_open_formats")) + { + manifest.allow_aggregate_function_states_in_open_formats = json->getValue("allow_aggregate_function_states_in_open_formats"); + } + /// Left unset (nullopt) for tasks created before these fields existed - such tasks were /// always scheduled under the old, strict column-matching check (a mismatch could never /// reach scheduling in the first place), so callers should treat an absent value as diff --git a/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp b/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp index f491c565fad14..d92ce6f2de64c 100644 --- a/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp +++ b/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp @@ -110,12 +110,16 @@ std::optional ExportPartitionTaskScheduler::run() const auto & manifest = entry.manifest; const auto key = entry.getCompositeKey(); - const auto database = storage.getContext()->resolveDatabase(manifest.destination_database); + + /// Catalog destinations must be resolved with the settings persisted by the initiating query. + const auto destination_resolution_context = ExportPartitionUtils::getContextCopyWithTaskSettings(storage.getContext(), manifest); + + const auto database = destination_resolution_context->resolveDatabase(manifest.destination_database); const auto & table = manifest.destination_table; const auto destination_storage_id = StorageID(QualifiedTableName {database, table}); - const auto destination_storage = DatabaseCatalog::instance().tryGetTable(destination_storage_id, storage.getContext()); + const auto destination_storage = DatabaseCatalog::instance().tryGetTable(destination_storage_id, destination_resolution_context); if (!destination_storage) { diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 41406b45c4fa3..2f05b974e531a 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -310,6 +310,9 @@ namespace ExportPartitionUtils /// schema drifts to a lossy target between scheduling and execution. context_copy->setSetting("export_merge_tree_part_allow_lossy_cast", manifest.allow_lossy_cast); + context_copy->setSetting( + "allow_experimental_aggregate_function_states_in_open_formats", manifest.allow_aggregate_function_states_in_open_formats); + if (manifest.iceberg_partition_timezone) { context_copy->setSetting("iceberg_partition_timezone", *manifest.iceberg_partition_timezone); diff --git a/src/Storages/MergeTree/MergeTreePartitionExportTask.h b/src/Storages/MergeTree/MergeTreePartitionExportTask.h index 8ce9958a0dbed..001e4ecaa208e 100644 --- a/src/Storages/MergeTree/MergeTreePartitionExportTask.h +++ b/src/Storages/MergeTree/MergeTreePartitionExportTask.h @@ -82,6 +82,7 @@ struct MergeTreePartitionExportTask String filename_pattern; bool write_full_path_in_iceberg_metadata = false; bool allow_lossy_cast = false; + bool allow_aggregate_function_states_in_open_formats = false; String iceberg_metadata_json; /// Optional for backwards compatibility with descriptors written before these @@ -189,6 +190,7 @@ struct MergeTreePartitionExportTask json.set("filename_pattern", filename_pattern); json.set("write_full_path_in_iceberg_metadata", write_full_path_in_iceberg_metadata); json.set("allow_lossy_cast", allow_lossy_cast); + json.set("allow_aggregate_function_states_in_open_formats", allow_aggregate_function_states_in_open_formats); if (!iceberg_metadata_json.empty()) json.set("iceberg_metadata_json", iceberg_metadata_json); if (parquet_compression_method) @@ -281,6 +283,8 @@ struct MergeTreePartitionExportTask task.filename_pattern = json->getValue("filename_pattern"); task.write_full_path_in_iceberg_metadata = json->getValue("write_full_path_in_iceberg_metadata"); task.allow_lossy_cast = json->getValue("allow_lossy_cast"); + if (json->has("allow_aggregate_function_states_in_open_formats")) + task.allow_aggregate_function_states_in_open_formats = json->getValue("allow_aggregate_function_states_in_open_formats"); if (json->has("iceberg_metadata_json")) task.iceberg_metadata_json = json->getValue("iceberg_metadata_json"); if (json->has("parquet_compression_method")) diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index e77ab1c3a639e..1fa1c365c24cb 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ namespace Setting { extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; + extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; } namespace ErrorCodes @@ -49,6 +51,29 @@ namespace manifest.filename_pattern = "{part_name}"; return manifest; } + + MergeTreePartitionExportTask makeValidPlainTask() + { + MergeTreePartitionExportTask task; + task.transaction_id = "tx1"; + task.query_id = "query1"; + task.partition_id = "2020"; + task.source_database = "db1"; + task.source_table = "source1"; + task.destination_database = "db1"; + task.destination_table = "table1"; + task.create_time = 1000; + task.parts.push_back({"part1", false, {}}); + task.task_timeout_seconds = 60; + task.max_threads = 1; + task.parallel_formatting = true; + task.parquet_parallel_encoding = true; + task.max_bytes_per_file = 1000000; + task.max_rows_per_file = 1000; + task.file_already_exists_policy = MergeTreePartExportManifest::FileAlreadyExistsPolicy::error; + task.filename_pattern = "{part_name}"; + return task; + } } class ExportPartitionOrderingTest : public ::testing::Test @@ -230,6 +255,70 @@ TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToW } } +TEST_F(ExportPartitionManifestBackCompatTest, MissingAggregateFunctionStateGatesParseAsDisabled) +{ + auto manifest = makeValidManifest(); + manifest.allow_aggregate_function_states_in_open_formats = true; + + Poco::JSON::Parser parser; + auto json = parser.parse(manifest.toJsonString()).extract(); + json->remove("allow_aggregate_function_states_in_open_formats"); + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + + auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(oss.str()); + EXPECT_FALSE(parsed.allow_aggregate_function_states_in_open_formats); +} + +TEST_F(ExportPartitionManifestBackCompatTest, AggregateFunctionStateGatesAppliedToWorkerContextForEveryValue) +{ + for (const bool value : {false, true}) + { + auto manifest = makeValidManifest(); + manifest.allow_aggregate_function_states_in_open_formats = value; + + auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); + + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats].value, + value) << "value=" << value; + } +} + +TEST_F(ExportPartitionManifestBackCompatTest, PlainTaskMissingAggregateFunctionStateGatesParseAsDisabled) +{ + auto task = makeValidPlainTask(); + task.allow_aggregate_function_states_in_open_formats = true; + + Poco::JSON::Parser parser; + auto json = parser.parse(task.toJsonString()).extract(); + json->remove("allow_aggregate_function_states_in_open_formats"); + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + + auto parsed = MergeTreePartitionExportTask::fromJsonString(oss.str()); + EXPECT_FALSE(parsed.allow_aggregate_function_states_in_open_formats); +} + +TEST_F(ExportPartitionManifestBackCompatTest, PlainTaskAggregateFunctionStateGatesAppliedToWorkerContextForEveryValue) +{ + for (const bool value : {false, true}) + { + auto task = makeValidPlainTask(); + task.allow_aggregate_function_states_in_open_formats = value; + + auto parsed = MergeTreePartitionExportTask::fromJsonString(task.toJsonString()); + auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, parsed); + + EXPECT_EQ(parsed.allow_aggregate_function_states_in_open_formats, value) << "value=" << value; + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats].value, + value) << "value=" << value; + } +} + TEST(ExportPartitionRetryClassification, MissingPartIsFatalOnlyOnPlainPath) { EXPECT_FALSE(ExportPartitionUtils::isNonRetryableExportError(ErrorCodes::NO_SUCH_DATA_PART)); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index 4f5ab2615b6e6..a8a45dee40c24 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -178,6 +178,7 @@ DEFINE_ICEBERG_FIELD_ALIAS(ref_min_snapshots_to_keep, min-snapshots-to-keep); DEFINE_ICEBERG_FIELD_ALIAS(ref_max_snapshot_age_ms, max-snapshot-age-ms); DEFINE_ICEBERG_FIELD_ALIAS(ref_max_ref_age_ms, max-ref-age-ms); DEFINE_ICEBERG_FIELD_ALIAS(clickhouse_export_partition_transaction_id, clickhouse.export-partition-transaction-id); +DEFINE_ICEBERG_FIELD_ALIAS(clickhouse_type, clickhouse.type); /// These are compound fields like `data_file.file_path`, we use prefix 'c_' to distinguish them. DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_path); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_format); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp index 88494c3233055..cbe3b84c9949b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,19 @@ static Range getExtremeRangeFromColumn(const ColumnPtr & column) return Range(min_val, true, max_val, true); } +static bool supportsExtremeRange(const IColumn & column) +{ + if (checkAndGetColumn(&column)) + return false; + + bool result = true; + column.forEachSubcolumnRecursively([&](const IColumn & subcolumn) + { + result &= checkAndGetColumn(&subcolumn) == nullptr; + }); + return result; +} + void DataFileStatistics::update(const Chunk & chunk) { if (!chunk.hasRows()) @@ -42,9 +56,12 @@ void DataFileStatistics::update(const Chunk & chunk) if (null_counts.empty()) { null_counts.resize(num_columns, 0); + track_ranges.resize(num_columns); for (size_t i = 0; i < num_columns; ++i) { - ranges.push_back(getExtremeRangeFromColumn(chunk.getColumns()[i])); + const auto & col = chunk.getColumns()[i]; + track_ranges[i] = supportsExtremeRange(*col); + ranges.push_back(track_ranges[i] ? getExtremeRangeFromColumn(col) : Range::createWholeUniverse()); } } @@ -58,7 +75,8 @@ void DataFileStatistics::update(const Chunk & chunk) for (UInt8 v : nullable_col->getNullMapData()) null_counts[i] += v; } - ranges[i] = uniteRanges(ranges[i], getExtremeRangeFromColumn(col)); + if (track_ranges[i]) + ranges[i] = uniteRanges(ranges[i], getExtremeRangeFromColumn(col)); } } @@ -127,6 +145,7 @@ void DataFileStatistics::merge(const DataFileStatistics & other) { null_counts = other.null_counts; ranges = other.ranges; + track_ranges = other.track_ranges; return; } @@ -134,7 +153,8 @@ void DataFileStatistics::merge(const DataFileStatistics & other) for (size_t i = 0; i < null_counts.size(); ++i) { null_counts[i] += other.null_counts[i]; - ranges[i] = uniteRanges(ranges[i], other.ranges[i]); + if (track_ranges[i]) + ranges[i] = uniteRanges(ranges[i], other.ranges[i]); } } @@ -175,8 +195,14 @@ std::vector> DataFileStatistics::getLowerBounds() const std::vector> result; for (size_t i = 0; i < ranges.size(); ++i) { +<<<<<<< HEAD if (!isExcluded(i)) result.push_back({field_ids[i], ranges[i].left}); +======= + if (!track_ranges[i]) + continue; + result.push_back({field_ids[i], ranges[i].left}); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return result; } @@ -186,8 +212,14 @@ std::vector> DataFileStatistics::getUpperBounds() const std::vector> result; for (size_t i = 0; i < ranges.size(); ++i) { +<<<<<<< HEAD if (!isExcluded(i)) result.push_back({field_ids[i], ranges[i].right}); +======= + if (!track_ranges[i]) + continue; + result.push_back({field_ids[i], ranges[i].right}); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return result; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h index cf78c9e1d6522..7ec665b3ea5df 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h @@ -47,7 +47,12 @@ class DataFileStatistics std::vector excluded; std::vector column_sizes; std::vector null_counts; +<<<<<<< HEAD Ranges ranges; +======= + std::vector ranges; + std::vector track_ranges; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) }; using DataFileStatisticsPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 1e4ca36843d3f..8050f04f063fa 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include "config.h" @@ -156,6 +157,7 @@ extern const SettingsString iceberg_metadata_compression_method; extern const SettingsBool allow_insert_into_iceberg; extern const SettingsBool allow_experimental_iceberg_compaction; extern const SettingsBool allow_experimental_geo_types_in_iceberg; +extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; extern const SettingsBool allow_iceberg_remove_orphan_files; extern const SettingsBool allow_experimental_expire_snapshots; extern const SettingsSeconds lock_acquire_timeout; @@ -288,7 +290,11 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom } std::pair IcebergMetadata::getRelevantState( +<<<<<<< HEAD const ContextPtr & context, bool force_fetch_latest_metadata) const +======= + const ContextPtr & context, bool force_fetch_latest_metadata, SchemaParsing schema_parsing) const +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { const auto [metadata_version, metadata_file_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( object_storage, @@ -300,7 +306,7 @@ std::pair IcebergMetadata::getReleva persistent_components.table_uuid, persistent_components.metadata_compression_method, force_fetch_latest_metadata); - return getState(context, metadata_file_path, metadata_version); + return getState(context, metadata_file_path, metadata_version, schema_parsing); } IcebergMetadata::IcebergMetadata( @@ -361,8 +367,10 @@ void IcebergMetadata::backgroundMetadataPrefetcherThread() /// first, we fetch the latest metadata version and cache it; /// as a part of the same method, we download metadata.json of the latest metadata version /// and after parsing it, we fetch manifest lists, parse and cache them + /// + /// Schema parsing depends on query settings, which a background prefetch has no authority to apply. auto ctx = Context::createCopy(Context::getGlobalContextInstance()); - auto [actual_data_snapshot, actual_table_state_snapshot] = getRelevantState(ctx, true); + auto [actual_data_snapshot, actual_table_state_snapshot] = getRelevantState(ctx, true, SchemaParsing::Skip); if (actual_data_snapshot) { for (const auto & entry : actual_data_snapshot->manifest_list_entries) @@ -394,19 +402,30 @@ Int32 IcebergMetadata::parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, ContextPtr context_, - LoggerPtr metadata_logger) + LoggerPtr metadata_logger, + SchemaParsing schema_parsing) { const auto format_version = metadata_object->getValue(f_format_version); +<<<<<<< HEAD if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) schema_processor.updateLastColumnId(metadata_object->getValue(f_last_column_id)); /// v2 and v3 (and later) store schemas in the `schemas` array. Only v1 has a top-level `schema`. /// Using the v1 parser first for v3 looks like a spec violation and logs a false warning. if (format_version >= 2) +======= + auto add_schema = [&](const Poco::JSON::Object::Ptr & schema) + { + if (schema_parsing == SchemaParsing::Parse) + schema_processor.addIcebergTableSchema(schema, context_); + }; + + if (format_version == 2) +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); - schema_processor.addIcebergTableSchema(schema, context_); + add_schema(schema); return current_schema_id; } else @@ -414,7 +433,7 @@ Int32 IcebergMetadata::parseTableSchema( try { auto [schema, current_schema_id] = parseTableSchemaV1Method(metadata_object); - schema_processor.addIcebergTableSchema(schema, context_); + add_schema(schema); return current_schema_id; } catch (const Exception & first_error) @@ -424,7 +443,7 @@ Int32 IcebergMetadata::parseTableSchema( try { auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); - schema_processor.addIcebergTableSchema(schema, context_); + add_schema(schema); LOG_WARNING( metadata_logger, "Iceberg table schema was parsed using v2 specification, but it was impossible to parse it using v1 " @@ -451,17 +470,26 @@ static Poco::JSON::Object::Ptr traverseMetadataAndFindNecessarySnapshotObject( Poco::JSON::Object::Ptr metadata_object, Int64 snapshot_id, IcebergSchemaProcessorPtr schema_processor, - ContextPtr local_context) + ContextPtr local_context, + IcebergMetadata::SchemaParsing schema_parsing) { if (!metadata_object->has(f_snapshots)) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "No snapshot set found in metadata for iceberg file"); +<<<<<<< HEAD if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) schema_processor->updateLastColumnId(metadata_object->getValue(f_last_column_id)); auto schemas = metadata_object->get(f_schemas).extract(); for (UInt32 j = 0; j < schemas->size(); ++j) +======= + if (schema_parsing == IcebergMetadata::SchemaParsing::Parse) +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { - auto schema = schemas->getObject(j); - schema_processor->addIcebergTableSchema(schema, local_context); + auto schemas = metadata_object->get(f_schemas).extract(); + for (UInt32 j = 0; j < schemas->size(); ++j) + { + auto schema = schemas->getObject(j); + schema_processor->addIcebergTableSchema(schema, local_context); + } } Poco::JSON::Object::Ptr current_snapshot = nullptr; auto snapshots = metadata_object->get(f_snapshots).extract(); @@ -525,14 +553,15 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO total_equality_deletes); } -IcebergDataSnapshotPtr -IcebergMetadata::getIcebergDataSnapshot(Poco::JSON::Object::Ptr metadata_object, Int64 snapshot_id, ContextPtr local_context) const +IcebergDataSnapshotPtr IcebergMetadata::getIcebergDataSnapshot( + Poco::JSON::Object::Ptr metadata_object, Int64 snapshot_id, ContextPtr local_context, SchemaParsing schema_parsing) const { auto object = traverseMetadataAndFindNecessarySnapshotObject( metadata_object, snapshot_id, persistent_components.schema_processor, - local_context); + local_context, + schema_parsing); if (!object) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "No snapshot found for id `{}`", snapshot_id); @@ -623,7 +652,7 @@ bool IcebergMetadata::optimizeManifestFiles( } std::pair -IcebergMetadata::getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object) const +IcebergMetadata::getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object, SchemaParsing schema_parsing) const { std::optional manifest_list_file; @@ -664,18 +693,18 @@ IcebergMetadata::getStateImpl(const ContextPtr & local_context, Poco::JSON::Obje ErrorCodes::BAD_ARGUMENTS, "No snapshot found in snapshot log before requested timestamp for iceberg table {}", persistent_components.table_path); - auto data_snapshot = getIcebergDataSnapshot(metadata_object, *current_snapshot_id, local_context); + auto data_snapshot = getIcebergDataSnapshot(metadata_object, *current_snapshot_id, local_context, schema_parsing); return {data_snapshot, static_cast(data_snapshot->schema_id_on_snapshot_commit)}; } else if (snapshot_id_changed) { Int64 current_snapshot_id = local_context->getSettingsRef()[Setting::iceberg_snapshot_id]; - auto data_snapshot = getIcebergDataSnapshot(metadata_object, current_snapshot_id, local_context); + auto data_snapshot = getIcebergDataSnapshot(metadata_object, current_snapshot_id, local_context, schema_parsing); return {data_snapshot, static_cast(data_snapshot->schema_id_on_snapshot_commit)}; } else { - auto schema_id = parseTableSchema(metadata_object, *persistent_components.schema_processor, local_context, log); + auto schema_id = parseTableSchema(metadata_object, *persistent_components.schema_processor, local_context, log, schema_parsing); if (!metadata_object->has(f_current_snapshot_id)) { return {nullptr, schema_id}; @@ -687,13 +716,14 @@ IcebergMetadata::getStateImpl(const ContextPtr & local_context, Poco::JSON::Obje { return {nullptr, schema_id}; } - auto data_snapshot = getIcebergDataSnapshot(metadata_object, current_snapshot_id, local_context); + auto data_snapshot = getIcebergDataSnapshot(metadata_object, current_snapshot_id, local_context, schema_parsing); return {data_snapshot, schema_id}; } } std::pair -IcebergMetadata::getState(const ContextPtr & local_context, const String & metadata_path, Int32 metadata_version) const +IcebergMetadata::getState( + const ContextPtr & local_context, const String & metadata_path, Int32 metadata_version, SchemaParsing schema_parsing) const { IcebergDataSnapshotPtr data_snapshot; TableStateSnapshot table_state_snapshot; @@ -710,7 +740,16 @@ IcebergMetadata::getState(const ContextPtr & local_context, const String & metad std::nullopt, std::nullopt); +<<<<<<< HEAD std::tie(data_snapshot, table_state_snapshot.schema_id) = getStateImpl(local_context, metadata_object); +======= + /// The Iceberg `format-version` in the metadata file may differ from the value cached in + /// `persistent_components` when an external tool (e.g. Spark) upgrades the table between + /// queries. Downstream parsers determine the version they need from the Avro metadata of + /// each manifest list / manifest file, so we do not update the shared cached value here. + + std::tie(data_snapshot, table_state_snapshot.schema_id) = getStateImpl(local_context, metadata_object, schema_parsing); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) table_state_snapshot.snapshot_id = data_snapshot ? std::optional{data_snapshot->snapshot_id} : std::nullopt; table_state_snapshot.metadata_version = metadata_version; table_state_snapshot.metadata_file_path = metadata_path; @@ -932,6 +971,7 @@ void IcebergMetadata::checkAlterIsPossible(const AlterCommands & commands) } } +<<<<<<< HEAD void IcebergMetadata::checkAlterPartitionIsPossible(const PartitionCommands & commands) const { checkTableRootIsQueriedPath("ALTER PARTITION"); @@ -990,6 +1030,22 @@ void IcebergMetadata::alterPartitionDropImpl( std::move(catalog), storage_id); executor.run(); +======= +static void checkAggregateFunctionStatesAllowed(const String & column_name, const DataTypePtr & type, const ContextPtr & context) +{ + if (!type || !needsClickHouseTypeAnnotation(type)) + return; + if (context->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats]) + return; + + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Column '{}' has type {}, which Iceberg cannot express natively: the aggregate state is stored as " + "binary and its ClickHouse type is recorded in the schema. To allow this, enable setting " + "allow_experimental_aggregate_function_states_in_open_formats", + column_name, + type->getName()); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } void IcebergMetadata::alter( @@ -1006,6 +1062,9 @@ void IcebergMetadata::alter( "To allow its usage, enable setting allow_insert_into_iceberg"); } + for (const auto & command : params) + checkAggregateFunctionStatesAllowed(command.column_name, command.data_type, context); + Iceberg::alter(params, context, storage_id, object_storage, data_lake_settings, persistent_components, write_format, catalog); } @@ -1131,6 +1190,9 @@ void IcebergMetadata::createInitial( } } + for (const auto & column : columns->getAll()) + checkAggregateFunctionStatesAllowed(column.name, column.type, local_context); + String location_path = configuration_ptr->getRawPath().path; if (!location_path.contains("://") && !location_path.starts_with('/')) location_path = "/" + location_path; @@ -1251,7 +1313,7 @@ Iceberg::IcebergDataSnapshotPtr IcebergMetadata::getRelevantDataSnapshotFromTabl if (!table_state_snapshot.snapshot_id.has_value()) return nullptr; Poco::JSON::Object::Ptr snapshot_object = traverseMetadataAndFindNecessarySnapshotObject( - metadata_object, *table_state_snapshot.snapshot_id, persistent_components.schema_processor, local_context); + metadata_object, *table_state_snapshot.snapshot_id, persistent_components.schema_processor, local_context, SchemaParsing::Parse); return createIcebergDataSnapshotFromSnapshotJSON(snapshot_object, *table_state_snapshot.snapshot_id, local_context); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index ab1d0762cafec..ac7e8b7eece07 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -106,11 +106,18 @@ class IcebergMetadata : public IDataLakeMetadata std::shared_ptr getInitialSchemaByPath(ContextPtr local_context, ObjectInfoPtr object_info) const override; std::shared_ptr getSchemaTransformer(ContextPtr local_context, ObjectInfoPtr object_info) const override; + enum class SchemaParsing + { + Parse, + Skip, + }; + static Int32 parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, Iceberg::IcebergSchemaProcessor & schema_processor, ContextPtr context_, - LoggerPtr metadata_logger); + LoggerPtr metadata_logger, + SchemaParsing schema_parsing = SchemaParsing::Parse); bool supportsUpdate() const override { return true; } bool supportsWrites() const override { return true; } @@ -120,6 +127,14 @@ class IcebergMetadata : public IDataLakeMetadata IcebergHistory getHistory(ContextPtr local_context) const; +<<<<<<< HEAD +======= + std::pair getRelevantState( + const ContextPtr & context, + bool force_fetch_latest_metadata = false, + SchemaParsing schema_parsing = SchemaParsing::Parse) const; + +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) /// Returns file records contributed by a single manifest list entry of `data_snapshot`. IcebergFiles getFilesForManifest( const Iceberg::IcebergDataSnapshotPtr & data_snapshot, @@ -268,12 +283,18 @@ class IcebergMetadata : public IDataLakeMetadata ContextPtr context_, LoggerPtr log); - Iceberg::IcebergDataSnapshotPtr - getIcebergDataSnapshot(Poco::JSON::Object::Ptr metadata_object, Int64 snapshot_id, ContextPtr local_context) const; + Iceberg::IcebergDataSnapshotPtr getIcebergDataSnapshot( + Poco::JSON::Object::Ptr metadata_object, Int64 snapshot_id, ContextPtr local_context, SchemaParsing schema_parsing) const; Iceberg::IcebergDataSnapshotPtr createIcebergDataSnapshotFromSnapshotJSON(Poco::JSON::Object::Ptr snapshot_object, Int64 snapshot_id, ContextPtr local_context) const; std::pair +<<<<<<< HEAD getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object) const; +======= + getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object, SchemaParsing schema_parsing) const; + std::pair + getState(const ContextPtr & local_context, const String & metadata_path, Int32 metadata_version, SchemaParsing schema_parsing) const; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) Iceberg::IcebergDataSnapshotPtr getRelevantDataSnapshotFromTableStateSnapshot(Iceberg::TableStateSnapshot table_state_snapshot, ContextPtr local_context) const; StorageObjectStorageConfigurationPtr getConfiguration() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 81390e2a2cdc6..f1b9265c8ac22 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -1,4 +1,10 @@ #include +<<<<<<< HEAD +======= +#include +#include +#include +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include #include @@ -345,7 +351,19 @@ Poco::JSON::Object::Ptr MetadataGenerator::getCurrentSchema() const return current_schema; } +<<<<<<< HEAD bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type, bool first, const String & after_column) const +======= +static bool clickHouseTypeAnnotationMatches(const Poco::JSON::Object::Ptr & field, const DataTypePtr & type) +{ + const bool expected = needsClickHouseTypeAnnotation(type); + if (expected != field->has(Iceberg::f_clickhouse_type)) + return false; + return !expected || field->getValue(Iceberg::f_clickhouse_type) == getClickHouseTypeAnnotationName(type); +} + +bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type) const +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { auto current_schema = findCurrentSchema(); if (!current_schema) @@ -365,7 +383,11 @@ bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypeP /// just used, so the ids of nested elements differ even for the very same type. return field->getValue(Iceberg::f_required) == expected_type.second && icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) +<<<<<<< HEAD && isFieldAtRequestedPosition(fields, i, column_name, first, after_column); +======= + && clickHouseTypeAnnotationMatches(field, type); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return false; } @@ -416,6 +438,7 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy auto field = fields->getObject(i); if (field->getValue(Iceberg::f_name) != column_name) continue; +<<<<<<< HEAD /// A position-only `MODIFY COLUMN c FIRST` carries no type. if (type) @@ -429,6 +452,11 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy } return isFieldAtRequestedPosition(fields, i, column_name, first, after_column); +======= + return field->getValue(Iceberg::f_required) == expected_type.second + && icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) + && clickHouseTypeAnnotationMatches(field, type); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return false; } @@ -856,6 +884,8 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da new_field->set(Iceberg::f_name, column_name); new_field->set(Iceberg::f_required, new_type.second); new_field->set(Iceberg::f_type, new_type.first); + if (needsClickHouseTypeAnnotation(type)) + new_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); if (first || !after_column.empty()) { @@ -920,11 +950,39 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (current_field->getValue(Iceberg::f_required) == new_type.second && icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) { +<<<<<<< HEAD Iceberg::IcebergSchemaProcessor schema_processor( context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); auto current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); if (!current_ch_type->equals(*type)) +======= + auto existing_iceberg_type = current_field->get(Iceberg::f_type); + if (!existing_iceberg_type.isString()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot MODIFY COLUMN '{}': the requested and existing types both map to the same " + "Iceberg complex type, and the change cannot be recorded in the Iceberg schema", + column_name); + + DataTypePtr reconstructed_ch_type; + if (current_field->has(Iceberg::f_clickhouse_type)) + { + reconstructed_ch_type + = DataTypeFactory::instance().get(current_field->getValue(Iceberg::f_clickhouse_type)); + } + else + { + reconstructed_ch_type = Iceberg::IcebergSchemaProcessor::getSimpleType( + existing_iceberg_type.extract(), + context, + context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); + if (!current_field->getValue(Iceberg::f_required) && reconstructed_ch_type->canBeInsideNullable()) + reconstructed_ch_type = makeNullable(reconstructed_ch_type); + } + + if (!reconstructed_ch_type->equals(*type)) +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot MODIFY COLUMN '{}' from {} to {}: both map to the same Iceberg type '{}' " @@ -932,9 +990,16 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, column_name, current_ch_type->getName(), type->getName(), +<<<<<<< HEAD current_field->get(Iceberg::f_type).toString()); if (!needs_reposition) +======= + existing_iceberg_type.extract()); + + /// `IDataType::equals` ignores the custom name of `SimpleAggregateFunction`. + if (clickHouseTypeAnnotationMatches(current_field, type)) +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) return false; } else @@ -945,9 +1010,32 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); +<<<<<<< HEAD type_changed = true; } break; +======= + if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); + + const auto next_schema_id = getNextSchemaId(metadata_object); + + current_schema = deepCopy(current_schema); + schema_fields = current_schema->getArray(Iceberg::f_fields); + current_field = schema_fields->getObject(i); + + current_field->set(Iceberg::f_type, new_type.first); + current_field->set(Iceberg::f_required, new_type.second); + if (needsClickHouseTypeAnnotation(type)) + current_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); + else + current_field->remove(Iceberg::f_clickhouse_type); + + metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); + current_schema->set(Iceberg::f_schema_id, next_schema_id); + metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); + return true; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index f4193e96f0c48..0a8cc1edbf6e9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include +#include #include #include #include @@ -53,10 +55,12 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int ICEBERG_SPECIFICATION_VIOLATION; +extern const int SUPPORT_IS_DISABLED; } namespace Setting { +extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; extern const SettingsTimezone iceberg_timezone_for_timestamptz; } @@ -445,6 +449,10 @@ void IcebergSchemaProcessor::addIcebergTableSchema( std::lock_guard lock(mutex); +<<<<<<< HEAD +======= + Int32 schema_id = schema_ptr->getValue(f_schema_id); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); @@ -501,6 +509,7 @@ void IcebergSchemaProcessor::addIcebergTableSchema( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", schema_id); +<<<<<<< HEAD } } @@ -536,6 +545,44 @@ void IcebergSchemaProcessor::addIcebergTableSchema( if (source == SchemaSource::ManifestFile) manifest_sourced_schema_ids.insert(schema_id); current_schema_id = std::nullopt; +======= + return; + } + + current_schema_id = schema_id; + bool parsed = false; + const auto rollback_unless_parsed = make_scope_guard( + [&]() TSA_NO_THREAD_SAFETY_ANALYSIS + { + /// TSA cannot see the lock held by the caller through this lambda. + current_schema_id = std::nullopt; + if (parsed) + return; + iceberg_table_schemas_by_ids.erase(schema_id); + clickhouse_table_schemas_by_ids.erase(schema_id); + std::erase_if(clickhouse_types_by_source_ids, [schema_id](const auto & entry) { return entry.first.first == schema_id; }); + std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & entry) { return entry.first.first == schema_id; }); + }); + + auto fields = schema_ptr->get(f_fields).extract(); + auto clickhouse_schema = std::make_shared(); + String current_full_name{}; + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue(f_name); + bool required = field->getValue(f_required); + current_full_name = name; + auto type = getFieldType(field, f_type, context_, required, current_full_name, true); + clickhouse_schema->push_back(NameAndTypePair{name, type}); + clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; + clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + } + + iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; + parsed = true; +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } NameAndTypePair IcebergSchemaProcessor::getFieldCharacteristics(Int32 schema_version, Int32 source_id) const @@ -727,18 +774,48 @@ DataTypePtr IcebergSchemaProcessor::getFieldType( String & current_full_name, bool is_subfield_of_root) { + DataTypePtr derived_type; if (field->isObject(type_key)) - return getComplexTypeFromObject(field->getObject(type_key), current_full_name, context_, is_subfield_of_root); - - auto type = field->get(type_key); - if (type.isString()) + derived_type = getComplexTypeFromObject(field->getObject(type_key), current_full_name, context_, is_subfield_of_root); + else { - const String & type_name = type.extract(); - auto data_type = getSimpleType(type_name, context_, allow_geo_parser); - return required || !data_type->canBeInsideNullable() ? data_type : makeNullable(data_type); + auto type = field->get(type_key); + if (!type.isString()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected 'type' field: {}", type.toString()); + + derived_type = getSimpleType(type.extract(), context_, allow_geo_parser); + if (!required && derived_type->canBeInsideNullable()) + derived_type = makeNullable(derived_type); } - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected 'type' field: {}", type.toString()); + if (!field->has(f_clickhouse_type)) + return derived_type; + + const String annotated_name = field->getValue(f_clickhouse_type); + auto annotated_type = DataTypeFactory::instance().get(annotated_name); + + /// Read the gate from the query context rather than the context captured at `ATTACH`. + if (hasAggregateFunctionType(annotated_type) + && !context_->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats]) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Iceberg field '{}' records ClickHouse type {} in its `{}` key. Reading aggregate function " + "states from an Iceberg table is disabled: enable setting " + "allow_experimental_aggregate_function_states_in_open_formats to honour the recorded type", + field->has(f_name) ? field->getValue(f_name) : type_key, + annotated_name, + f_clickhouse_type); + + if (!annotatedTypeMatchesDerived(annotated_type, derived_type)) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg field '{}' records ClickHouse type {} in its `{}` key, but the field itself reads as {}", + field->has(f_name) ? field->getValue(f_name) : type_key, + annotated_name, + f_clickhouse_type, + derived_type->getName()); + + return annotated_type; } DataTypePtr IcebergSchemaProcessor::getClickHouseFieldType(const Poco::JSON::Object::Ptr & field, ContextPtr context_) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 3155e86a5c2f9..328db41edf64d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -694,6 +695,8 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); case TypeIndex::String: return {"string", true}; + case TypeIndex::AggregateFunction: + return {Iceberg::f_binary, true}; case TypeIndex::UUID: return {"uuid", true}; case TypeIndex::Nothing: @@ -1266,6 +1269,8 @@ std::pair createEmptyMetadataFile( auto type = getIcebergType(column.type, iter, format_version); field->set(Iceberg::f_required, type.second); field->set(Iceberg::f_type, type.first); + if (needsClickHouseTypeAnnotation(column.type)) + field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(column.type)); column_name_to_source_id[column.name] = iter_for_initial_columns; schema_fields->add(field); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp index e262df663178b..b962ddb65deff 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp @@ -6,8 +6,13 @@ #include #include +<<<<<<< HEAD #include #include +======= +#include +#include +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include #include @@ -537,6 +542,7 @@ Poco::JSON::Object::Ptr makeMetadataWithField( return metadata; } +<<<<<<< HEAD /// Ordered field names from the schema `current-schema-id` points at. std::vector getCurrentFieldNames(const Poco::JSON::Object::Ptr & metadata) { @@ -559,6 +565,9 @@ std::vector getCurrentFieldNames(const Poco::JSON::Object::Ptr & metadat /// The Iceberg type recorded for `name` in the schema `current-schema-id` points at. Poco::Dynamic::Var findCurrentFieldType(const Poco::JSON::Object::Ptr & metadata, const String & name) +======= +Poco::Dynamic::Var findCurrentFieldValue(const Poco::JSON::Object::Ptr & metadata, const String & name, const String & key) +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { auto current_schema_id = metadata->getValue(f_current_schema_id); auto schemas = metadata->getArray(f_schemas); @@ -572,12 +581,26 @@ Poco::Dynamic::Var findCurrentFieldType(const Poco::JSON::Object::Ptr & metadata { auto field = fields->getObject(j); if (field->getValue(f_name) == name) - return field->get(f_type); + return field->get(key); } } return {}; } +Poco::JSON::Object::Ptr makeMetadataWithAnnotatedField( + const String & name, const Poco::Dynamic::Var & iceberg_type, bool required, const String & annotation) +{ + auto metadata = makeMetadataWithField(name, iceberg_type, required); + metadata->getArray(f_schemas)->getObject(0)->getArray(f_fields)->getObject(0)->set(f_clickhouse_type, annotation); + return metadata; +} + +std::optional findCurrentFieldAnnotation(const Poco::JSON::Object::Ptr & metadata, const String & name) +{ + auto value = findCurrentFieldValue(metadata, name, f_clickhouse_type); + return value.isEmpty() ? std::nullopt : std::optional{value.extract()}; +} + void expectModifyRejected( const Poco::JSON::Object::Ptr & metadata, const String & column, @@ -787,11 +810,12 @@ TEST(IcebergMetadataGenerator, ModifyColumnWideningRecordsTheNewTypeInANewSchema EXPECT_EQ(after.schema_count, before.schema_count + 1); EXPECT_NE(after.current_schema_id, before.current_schema_id); - auto stored_type = findCurrentFieldType(metadata, "x"); + auto stored_type = findCurrentFieldValue(metadata, "x", f_type); ASSERT_TRUE(stored_type.isString()); EXPECT_EQ(stored_type.extract(), "long"); } +<<<<<<< HEAD /// --- Multi-argument transform (Iceberg V3 source-ids) tests --- @@ -1231,6 +1255,84 @@ TEST(IcebergMetadataGenerator, ModifyColumnSameComplexTypeWithDocOnlyRepositions auto stored_fields = stored_type.extract()->getArray(f_fields); ASSERT_EQ(stored_fields->size(), 1u); EXPECT_EQ(stored_fields->getObject(0)->getValue(f_type), "int"); +======= +TEST(IcebergMetadataGenerator, ModifyColumnToTheSameSimpleAggregateFunctionAddsNoSchema) +{ + tryRegisterAggregateFunctions(); + auto metadata + = makeMetadataWithAnnotatedField("s", "long", /* required */ true, "SimpleAggregateFunction(sum, UInt64)"); + const auto before = readSchemaState(metadata); + + EXPECT_FALSE(MetadataGenerator(metadata).generateModifyColumnMetadata( + "s", DataTypeFactory::instance().get("SimpleAggregateFunction(sum, UInt64)"), getContext().context)); + expectSchemaUnchanged(metadata, before); +} + +TEST(IcebergMetadataGenerator, ModifyColumnChangingTheSimpleAggregateFunctionRewritesTheAnnotation) +{ + tryRegisterAggregateFunctions(); + auto metadata + = makeMetadataWithAnnotatedField("s", "long", /* required */ true, "SimpleAggregateFunction(sum, UInt64)"); + const auto before = readSchemaState(metadata); + + EXPECT_TRUE(MetadataGenerator(metadata).generateModifyColumnMetadata( + "s", DataTypeFactory::instance().get("SimpleAggregateFunction(max, UInt64)"), getContext().context)); + + const auto after = readSchemaState(metadata); + EXPECT_EQ(after.schema_count, before.schema_count + 1); + EXPECT_NE(after.current_schema_id, before.current_schema_id); + EXPECT_EQ(findCurrentFieldAnnotation(metadata, "s"), "SimpleAggregateFunction(max, UInt64)"); +} + +TEST(IcebergMetadataGenerator, ModifyColumnAddingASimpleAggregateFunctionRecordsTheAnnotation) +{ + tryRegisterAggregateFunctions(); + auto metadata = makeMetadataWithField("s", "long", /* required */ true); + ASSERT_EQ(findCurrentFieldAnnotation(metadata, "s"), std::nullopt); + + EXPECT_TRUE(MetadataGenerator(metadata).generateModifyColumnMetadata( + "s", DataTypeFactory::instance().get("SimpleAggregateFunction(sum, Int64)"), getContext().context)); + + EXPECT_EQ(findCurrentFieldAnnotation(metadata, "s"), "SimpleAggregateFunction(sum, Int64)"); +} + +TEST(IcebergMetadataGenerator, ModifyColumnDroppingASimpleAggregateFunctionRemovesTheAnnotation) +{ + tryRegisterAggregateFunctions(); + auto metadata + = makeMetadataWithAnnotatedField("s", "long", /* required */ true, "SimpleAggregateFunction(sum, UInt64)"); + + EXPECT_TRUE(MetadataGenerator(metadata).generateModifyColumnMetadata( + "s", std::make_shared(), getContext().context)); + + EXPECT_EQ(findCurrentFieldAnnotation(metadata, "s"), std::nullopt); +} + +TEST(IcebergMetadataGenerator, ModifyColumnRejectsChangingTheAggregateFunctionOfAState) +{ + tryRegisterAggregateFunctions(); + auto metadata + = makeMetadataWithAnnotatedField("u", f_binary, /* required */ true, "AggregateFunction(uniq, UInt64)"); + expectModifyRejected(metadata, "u", DataTypeFactory::instance().get("AggregateFunction(sum, UInt64)")); + EXPECT_EQ(findCurrentFieldAnnotation(metadata, "u"), "AggregateFunction(uniq, UInt64)"); +} + +TEST(IcebergMetadataGenerator, AddColumnRecordsAndDetectsTheClickHouseTypeAnnotation) +{ + tryRegisterAggregateFunctions(); + auto metadata = makeMetadataWithGap(); + MetadataGenerator gen(metadata); + + auto type = DataTypeFactory::instance().get("SimpleAggregateFunction(anyLast, Nullable(String))"); + EXPECT_FALSE(gen.isAddColumnApplied("extra", type)); + + gen.generateAddColumnMetadata("extra", type); + EXPECT_EQ(findCurrentFieldAnnotation(metadata, "extra"), "SimpleAggregateFunction(anyLast, Nullable(String))"); + EXPECT_TRUE(gen.isAddColumnApplied("extra", type)); + EXPECT_FALSE(gen.isAddColumnApplied( + "extra", DataTypeFactory::instance().get("SimpleAggregateFunction(any, Nullable(String))"))); + EXPECT_FALSE(gen.isAddColumnApplied("extra", makeNullable(std::make_shared()))); +>>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index 77908fb3e0c08..f7894c22db282 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -1,9 +1,16 @@ #include #include +#include #include #include #include +#include + +#include "config.h" +#if USE_AVRO +#include +#endif #include #include @@ -15,6 +22,12 @@ namespace DB::ErrorCodes using namespace DB::Iceberg; +namespace DB::ErrorCodes +{ +extern const int ICEBERG_SPECIFICATION_VIOLATION; +extern const int SUPPORT_IS_DISABLED; +} + namespace { Poco::JSON::Object::Ptr parseSchema(const std::string & json) @@ -22,6 +35,28 @@ Poco::JSON::Object::Ptr parseSchema(const std::string & json) Poco::JSON::Parser parser; return parser.parse(json).extract(); } + +DB::ContextMutablePtr contextWithAggregateFunctionStates(bool allow) +{ + auto context = DB::Context::createCopy(getContext().context); + context->setSetting("allow_experimental_aggregate_function_states_in_open_formats", DB::Field(allow)); + return context; +} + +void expectAnnotatedSchemaRejected(const Poco::JSON::Object::Ptr & schema) +{ + auto context = contextWithAggregateFunctionStates(true); + IcebergSchemaProcessor processor(context); + try + { + processor.addIcebergTableSchema(schema, context); + FAIL() << "The annotation does not describe the field type and must be rejected"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION) << e.message(); + } +} } TEST(IcebergSchemaProcessor, GetSimpleTypeBoolean) @@ -531,3 +566,97 @@ TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalSignOnlyScaleThrows) { EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("decimal(20,+)", getContext().context), DB::Exception); } + +TEST(IcebergSchemaProcessor, AnnotatedSimpleAggregateFunctionFieldNeedsNoSetting) +{ + tryRegisterAggregateFunctions(); + auto schema = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long","clickhouse.type":"SimpleAggregateFunction(sum, Int64)"}]})json"); + auto context = contextWithAggregateFunctionStates(false); + IcebergSchemaProcessor processor(context); + processor.addIcebergTableSchema(schema, context); + + auto columns = processor.getClickhouseTableSchemaById(0); + ASSERT_EQ(columns->size(), 1u); + EXPECT_EQ(columns->front().name, "s"); + EXPECT_EQ(columns->front().type->getName(), "SimpleAggregateFunction(sum, Int64)"); +} + +TEST(IcebergSchemaProcessor, AnnotationNotMatchingTheFieldTypeIsRejected) +{ + tryRegisterAggregateFunctions(); + expectAnnotatedSchemaRejected(parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"u","required":true,"type":"long","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json")); +} + +TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSucceeds) +{ + tryRegisterAggregateFunctions(); + const std::string refused_json + = R"json({"schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"long"},{"id":2,"name":"u","required":false,"type":"binary","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json"; + IcebergSchemaProcessor processor(getContext().context); + + auto refused_context = contextWithAggregateFunctionStates(false); + try + { + processor.addIcebergTableSchema(parseSchema(refused_json), refused_context); + FAIL() << "The annotation names an aggregate function and must be refused without the setting"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::SUPPORT_IS_DISABLED) << e.message(); + } + + EXPECT_FALSE(processor.hasClickhouseTableSchemaById(0)); + EXPECT_FALSE(processor.tryGetFieldCharacteristics(0, 1).has_value()); + EXPECT_FALSE(processor.tryGetColumnIDByName(0, "a").has_value()); + + auto allowed_context = contextWithAggregateFunctionStates(true); + const std::string allowed_json + = R"json({"schema-id":0,"fields":[{"id":1,"name":"a2","required":false,"type":"int"},{"id":2,"name":"u","required":false,"type":"binary","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json"; + ASSERT_NO_THROW(processor.addIcebergTableSchema(parseSchema(allowed_json), allowed_context)); + auto columns = processor.getClickhouseTableSchemaById(0); + ASSERT_EQ(columns->size(), 2u); + EXPECT_EQ(columns->front().name, "a2"); + auto field = processor.getFieldCharacteristics(0, 1); + EXPECT_EQ(field.name, "a2"); + EXPECT_EQ(field.type->getName(), "Nullable(Int32)"); + EXPECT_EQ(columns->back().type->getName(), "AggregateFunction(uniq, UInt64)"); + EXPECT_FALSE(processor.tryGetColumnIDByName(0, "a").has_value()); +} + +#if USE_AVRO +TEST(IcebergSchemaProcessor, SkippingTheSchemaNeitherAppliesTheGateNorPublishesTheSchema) +{ + tryRegisterAggregateFunctions(); + Poco::JSON::Parser parser; + auto metadata = parser + .parse( + R"json({"format-version":2,"current-schema-id":0,"schemas":[{"schema-id":0,)json" + R"json("fields":[{"id":1,"name":"u","required":false,"type":"binary",)json" + R"json("clickhouse.type":"AggregateFunction(uniq, UInt64)"}]}]})json") + .extract(); + + auto log = getLogger("IcebergSchemaProcessorTest"); + auto context = contextWithAggregateFunctionStates(false); + DB::Iceberg::IcebergSchemaProcessor processor(context); + + Int32 schema_id = -1; + ASSERT_NO_THROW( + schema_id = DB::IcebergMetadata::parseTableSchema( + metadata, processor, context, log, DB::IcebergMetadata::SchemaParsing::Skip)); + EXPECT_EQ(schema_id, 0); + + EXPECT_FALSE(processor.hasClickhouseTableSchemaById(0)); + EXPECT_FALSE(processor.tryGetFieldCharacteristics(0, 1).has_value()); + EXPECT_FALSE(processor.tryGetColumnIDByName(0, "u").has_value()); + + auto allowed_context = contextWithAggregateFunctionStates(true); + ASSERT_NO_THROW(DB::IcebergMetadata::parseTableSchema( + metadata, processor, allowed_context, log, DB::IcebergMetadata::SchemaParsing::Parse)); + auto columns = processor.getClickhouseTableSchemaById(0); + ASSERT_EQ(columns->size(), 1u); + EXPECT_EQ(columns->front().name, "u"); + EXPECT_EQ(columns->front().type->getName(), "AggregateFunction(uniq, UInt64)"); +} +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp index 3168cdbf90922..58e747005e3c9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp @@ -5,6 +5,9 @@ #include #include +#include +#include +#include #include #include #include @@ -176,4 +179,64 @@ TEST(IcebergTypeMapping, NullableDecimalPrecisionAboveSpecLimitIsRejected) expectIcebergTypeRejected(makeNullable(createDecimal(76, 1))); } +TEST(IcebergTypeMapping, AggregateFunctionMapsToBinary) +{ + tryRegisterAggregateFunctions(); + auto type = DataTypeFactory::instance().get("AggregateFunction(uniq, UInt64)"); + Int32 iter = 0; + auto [iceberg_type, required] = getIcebergType(type, iter); + ASSERT_TRUE(iceberg_type.isString()); + EXPECT_EQ(iceberg_type.extract(), "binary"); + EXPECT_TRUE(required); +} + +TEST(IcebergTypeMapping, AggregateStatesNeedATypeAnnotation) +{ + tryRegisterAggregateFunctions(); + EXPECT_TRUE(needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("AggregateFunction(uniq, UInt64)"))); + EXPECT_TRUE(needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("SimpleAggregateFunction(sum, UInt64)"))); + EXPECT_TRUE(needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("Array(AggregateFunction(uniq, UInt64))"))); + EXPECT_TRUE( + needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("SimpleAggregateFunction(anyLast, Nullable(String))"))); + + EXPECT_FALSE(needsClickHouseTypeAnnotation(std::make_shared())); + EXPECT_FALSE(needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("Array(Nullable(String))"))); + EXPECT_FALSE(needsClickHouseTypeAnnotation(DataTypeFactory::instance().get("Tuple(a UInt32, b String)"))); +} + +TEST(IcebergTypeMapping, AnnotationNamePinsAggregateStateVersion) +{ + tryRegisterAggregateFunctions(); + + auto v0 = DataTypeFactory::instance().get("AggregateFunction(0, sumMap, Array(UInt8), Array(UInt8))"); + EXPECT_EQ(v0->getName(), "AggregateFunction(sumMap, Array(UInt8), Array(UInt8))"); + EXPECT_EQ(getClickHouseTypeAnnotationName(v0), "AggregateFunction(0, sumMap, Array(UInt8), Array(UInt8))"); + EXPECT_EQ( + assert_cast( + *DataTypeFactory::instance().get(getClickHouseTypeAnnotationName(v0))) + .getVersion(), + 0u); + + EXPECT_EQ( + getClickHouseTypeAnnotationName( + DataTypeFactory::instance().get("Array(AggregateFunction(0, sumMap, Array(UInt8), Array(UInt8)))")), + "Array(AggregateFunction(0, sumMap, Array(UInt8), Array(UInt8)))"); + + for (const auto * name : + {"AggregateFunction(sumMap, Array(UInt8), Array(UInt8))", + "AggregateFunction(1, sumMap, Array(UInt8), Array(UInt8))", + "AggregateFunction(1, groupBitmap, UInt64)", + "AggregateFunction(groupBitmap, UInt64)", + "AggregateFunction(uniq, UInt64)", + "AggregateFunction(sum, UInt64)", + "Array(AggregateFunction(uniq, UInt64))", + "SimpleAggregateFunction(sum, UInt64)", + "SimpleAggregateFunction(anyLast, Nullable(String))", + "Tuple(a UInt32, b String)"}) + { + auto type = DataTypeFactory::instance().get(name); + EXPECT_EQ(getClickHouseTypeAnnotationName(type), type->getName()) << name; + } +} + #endif diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 801fc08fbb418..650f49bc1919d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -145,6 +145,7 @@ namespace Setting extern const SettingsTimezone iceberg_partition_timezone; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; + extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; } namespace ServerSetting @@ -4065,6 +4066,8 @@ void StorageMergeTree::exportPartitionToTable(const PartitionCommand & command, descriptor.filename_pattern = query_context->getSettingsRef()[Setting::export_merge_tree_part_filename_pattern].value; descriptor.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; descriptor.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; + descriptor.allow_aggregate_function_states_in_open_formats + = query_context->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats]; descriptor.iceberg_partition_timezone = query_context->getSettingsRef()[Setting::iceberg_partition_timezone].toString(); descriptor.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; descriptor.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index dfcce7ddeb26b..879d60183d645 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -254,6 +254,7 @@ namespace Setting extern const SettingsUInt64 iceberg_insert_max_bytes_in_data_file; extern const SettingsUInt64 iceberg_insert_max_rows_in_data_file; extern const SettingsTimezone iceberg_partition_timezone; + extern const SettingsBool allow_experimental_aggregate_function_states_in_open_formats; } @@ -8845,6 +8846,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.filename_pattern = query_context->getSettingsRef()[Setting::export_merge_tree_part_filename_pattern].value; manifest.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; manifest.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; + manifest.allow_aggregate_function_states_in_open_formats + = query_context->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats]; manifest.iceberg_partition_timezone = query_context->getSettingsRef()[Setting::iceberg_partition_timezone].toString(); manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; diff --git a/tests/integration/helpers/export_partition_helpers.py b/tests/integration/helpers/export_partition_helpers.py index d0cb8eb3d379b..0043647a8bccd 100644 --- a/tests/integration/helpers/export_partition_helpers.py +++ b/tests/integration/helpers/export_partition_helpers.py @@ -231,14 +231,15 @@ def make_mt( columns, partition_by, order_by="tuple()", + engine="MergeTree()", extra_settings="", ): - """Create a MergeTree table with block-number settings.""" + """Create a MergeTree-family table with block-number settings.""" settings = f"{_BLOCK_SETTINGS}, {extra_settings}" if extra_settings else _BLOCK_SETTINGS node.query( f""" CREATE TABLE {name} ({columns}) - ENGINE = MergeTree() + ENGINE = {engine} PARTITION BY {partition_by} ORDER BY {order_by} SETTINGS {settings} @@ -335,21 +336,22 @@ def make_iceberg_s3( url=None, s3_retry_attempts=3, if_not_exists=False, + extra_settings="", ): - """Create an IcebergS3 table at a MinIO prefix. - - *url* defaults to ``http://minio1:9001/root/data/{name}/``. - """ + """Create an IcebergS3 table at a MinIO prefix.""" if url is None: url = f"http://minio1:9001/root/data/{name}/" ine = "IF NOT EXISTS " if if_not_exists else "" pclause = f"PARTITION BY {partition_by}" if partition_by else "" + settings = f"s3_retry_attempts = {s3_retry_attempts}" + if extra_settings: + settings += ", " + extra_settings node.query( f""" CREATE TABLE {ine}{name} ({columns}) ENGINE = IcebergS3('{url}', '{MINIO_USER}', '{MINIO_PASS}') {pclause} - SETTINGS s3_retry_attempts = {s3_retry_attempts} + SETTINGS {settings} """ ) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index c1197373be39a..2be9058941843 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -16,6 +16,7 @@ test_export_part_multi_column_partition_key_success – composite (a, b, c) partition key round-trips test_export_part_partition_key_mismatch_variants_are_rejected (parametrized) – partition key column reordering, cardinality mismatches, and transform-expression reordering between src/dst are all rejected synchronously + test_export_part_aggregate_function_states – AggregateFunction / SimpleAggregateFunction states survive the export """ import logging @@ -1508,3 +1509,51 @@ def test_export_part_tuple_subcolumn_partition_key_iceberg_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_aggregate_function_states(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_agg_states_{sfx}" + iceberg = f"iceberg_agg_states_{sfx}" + + columns = ( + "k Int32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64)" + ) + + make_mt(node, mt, columns, "k", order_by="k", engine="AggregatingMergeTree()") + make_iceberg_s3( + node, + iceberg, + columns, + "k", + extra_settings="allow_experimental_aggregate_function_states_in_open_formats = 1", + ) + + node.query( + f"INSERT INTO {mt} " + f"SELECT toInt32(number % 2), uniqState(toUInt64(number % 23)), sumSimpleState(number) " + f"FROM numbers(200) GROUP BY number % 2" + ) + + part = get_part(node, mt, "0") + export_part( + node, + mt, + part, + iceberg, + extra_settings="allow_experimental_aggregate_function_states_in_open_formats = 1", + ) + wait_for_export_part(node, mt, part) + assert_part_log(node, mt, part) + + exported = node.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {iceberg} GROUP BY k ORDER BY k " + f"SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1" + ) + assert exported == node.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {mt} WHERE k = 0 GROUP BY k ORDER BY k" + ), f"Exported states do not match the source table:\n{exported}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_partition_to_iceberg/test_schema_match.py b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py index 32de3888ec755..6448792227417 100644 --- a/tests/integration/test_export_partition_to_iceberg/test_schema_match.py +++ b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py @@ -983,3 +983,47 @@ def test_export_partition_all_iceberg_types_lossy(cluster, source_engine): f"SELECT abs(f - 2.718281828459045) < 1e-6, abs(f - 2.718281828459045) > 1e-9 FROM {iceberg_table}" ).strip() assert f_checks == "1\t1", f"Expected Float32 precision loss within tolerance, got: {f_checks!r}" + + +def test_export_partition_aggregate_function_states(cluster, source_engine): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_agg_states_{uid}" + iceberg_table = f"iceberg_agg_states_{uid}" + + columns = ( + "k Int32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64)" + ) + + make_source(node, mt_table, columns, "k", engine=source_engine, order_by="k", replica_name="replica1") + make_iceberg_s3( + node, + iceberg_table, + columns, + partition_by="k", + extra_settings="allow_experimental_aggregate_function_states_in_open_formats = 1", + ) + + node.query( + f"INSERT INTO {mt_table} " + f"SELECT toInt32(number % 2), uniqState(toUInt64(number % 23)), sumSimpleState(number) " + f"FROM numbers(200) GROUP BY number % 2" + ) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '0' TO TABLE {iceberg_table}", + settings={ + "allow_insert_into_iceberg": 1, + "allow_experimental_aggregate_function_states_in_open_formats": 1, + }, + ) + wait_for_export_status(node, mt_table, iceberg_table, "0", "COMPLETED") + + exported = node.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {iceberg_table} GROUP BY k ORDER BY k " + f"SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1" + ) + assert exported == node.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {mt_table} WHERE k = 0 GROUP BY k ORDER BY k" + ), f"Exported states do not match the source partition:\n{exported}" diff --git a/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py new file mode 100644 index 0000000000000..3167b89340b8b --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py @@ -0,0 +1,210 @@ +import glob +import json +import os +import re + +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_download_directory, + get_creation_expression, + get_uuid_str, +) + +from helpers.test_tools import TSV + + +SETTING = "allow_experimental_aggregate_function_states_in_open_formats" + +STATE_SETTINGS = {SETTING: 1} +WRITE_SETTINGS = {"allow_insert_into_iceberg": 1} +PARQUET_STATE_SETTINGS = [f"{SETTING} = 1"] + +SCHEMA = "(k Int32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64))" + +# Spark's Arrow reader does not accept the UINT_64 converted type written for UInt64. +SPARK_SCHEMA = "(k Int32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, Int64))" + +WAREHOUSE = "/var/lib/clickhouse/user_files/iceberg_data/default" + + +def insert_states(instance, table_name, sum_expression="number"): + instance.query( + f""" + INSERT INTO {table_name} (k, u, s) + SELECT toInt32(number % 2), uniqState(toUInt64(number % 23)), sumSimpleState({sum_expression}) + FROM numbers(200) + GROUP BY number % 2 + """, + settings={**WRITE_SETTINGS, **STATE_SETTINGS}, + ) + + +def latest_metadata(table_name): + files = glob.glob(os.path.join(WAREHOUSE, table_name, "metadata", "v*.metadata.json")) + assert files, f"No metadata JSON downloaded for {table_name}" + newest = max(files, key=lambda path: int(re.fullmatch(r"v(\d+)\.metadata\.json", os.path.basename(path)).group(1))) + with open(newest) as f: + return json.load(f) + + +def fields_by_name(metadata): + schema_id = metadata["current-schema-id"] + for schema in metadata["schemas"]: + if schema["schema-id"] == schema_id: + return {field["name"]: field for field in schema["fields"]} + raise AssertionError(f"Schema {schema_id} not found in {metadata['schemas']}") + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_aggregate_states_require_setting(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_agg_states_setting_" + storage_type + "_" + get_uuid_str() + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, SCHEMA, format_version=2 + ) + error = instance.query_and_get_error(creation_expression) + assert SETTING in error, f"CREATE TABLE error does not name the setting: {error}" + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + SCHEMA, + format_version=2, + settings=STATE_SETTINGS, + additional_settings=PARQUET_STATE_SETTINGS, + ) + + error = instance.query_and_get_error( + f"ALTER TABLE {TABLE_NAME} ADD COLUMN extra SimpleAggregateFunction(anyLast, Nullable(String))", + settings=WRITE_SETTINGS, + ) + assert SETTING in error, f"ADD COLUMN error does not name the setting: {error}" + + instance.query( + f"ALTER TABLE {TABLE_NAME} ADD COLUMN extra SimpleAggregateFunction(anyLast, Nullable(String))", + settings={**WRITE_SETTINGS, **STATE_SETTINGS}, + ) + + insert_states(instance, TABLE_NAME) + assert instance.query(f"SELECT count() FROM {TABLE_NAME}", settings=STATE_SETTINGS) == "2\n" + + table_function_expr = get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster_iceberg_with_spark, + table_function=True, + ) + error = instance.query_and_get_error(f"SELECT count() FROM {table_function_expr}") + assert SETTING in error, f"Read error does not name the setting: {error}" + + assert ( + instance.query(f"SELECT count() FROM {table_function_expr}", settings=STATE_SETTINGS) + == "2\n" + ) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_aggregate_states_round_trip(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_agg_states_round_trip_" + storage_type + "_" + get_uuid_str() + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + SCHEMA, + format_version=2, + partition_by="k", + settings=STATE_SETTINGS, + additional_settings=PARQUET_STATE_SETTINGS, + ) + insert_states(instance, TABLE_NAME) + + table_function_expr = get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster_iceberg_with_spark, + table_function=True, + ) + assert instance.query( + f"DESCRIBE {table_function_expr} FORMAT TSV", + settings={"print_pretty_type_names": 0, **STATE_SETTINGS}, + ) == TSV( + [ + ["k", "Int32"], + ["u", "AggregateFunction(uniq, UInt64)"], + ["s", "SimpleAggregateFunction(sum, UInt64)"], + ] + ) + + assert instance.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {TABLE_NAME} GROUP BY k ORDER BY k", + settings=STATE_SETTINGS, + ) == instance.query( + "SELECT toInt32(number % 2) AS k, uniq(toUInt64(number % 23)), sum(number)" + " FROM numbers(200) GROUP BY k ORDER BY k" + ) + + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"{WAREHOUSE}/{TABLE_NAME}/", + f"{WAREHOUSE}/{TABLE_NAME}/", + ) + + fields = fields_by_name(latest_metadata(TABLE_NAME)) + assert fields["k"]["type"] == "int" + assert "clickhouse.type" not in fields["k"] + assert fields["u"]["type"] == "binary" + assert fields["u"]["clickhouse.type"] == "AggregateFunction(uniq, UInt64)" + assert fields["s"]["type"] == "long" + assert fields["s"]["clickhouse.type"] == "SimpleAggregateFunction(sum, UInt64)" + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_aggregate_states_read_by_spark(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_agg_states_spark_" + storage_type + "_" + get_uuid_str() + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + SPARK_SCHEMA, + format_version=2, + settings=STATE_SETTINGS, + additional_settings=PARQUET_STATE_SETTINGS, + ) + insert_states(instance, TABLE_NAME, "toInt64(number)") + + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"{WAREHOUSE}/{TABLE_NAME}/", + f"{WAREHOUSE}/{TABLE_NAME}/", + ) + + with open(f"{WAREHOUSE}/{TABLE_NAME}/metadata/version-hint.text", "wb") as f: + f.write(b"1") + + df = spark.read.format("iceberg").load(f"{WAREHOUSE}/{TABLE_NAME}") + assert dict(df.dtypes) == {"k": "int", "u": "binary", "s": "bigint"} + + spark_rows = {row["k"]: (bytes(row["u"]).hex().upper(), row["s"]) for row in df.collect()} + assert len(spark_rows) == 2 + + expected = instance.query( + f"SELECT k, hex(u), s FROM {TABLE_NAME} ORDER BY k FORMAT TSV", settings=STATE_SETTINGS + ).strip() + assert expected != "" + for line in expected.split("\n"): + k, state_hex, sum_value = line.split("\t") + assert spark_rows[int(k)] == (state_hex, int(sum_value)) diff --git a/tests/queries/0_stateless/04673_parquet_aggregate_function_state.reference b/tests/queries/0_stateless/04673_parquet_aggregate_function_state.reference new file mode 100644 index 0000000000000..d88845416f99c --- /dev/null +++ b/tests/queries/0_stateless/04673_parquet_aggregate_function_state.reference @@ -0,0 +1,87 @@ +-- without the setting writing a state is refused, as it was before Parquet supported states +1 +-- a SimpleAggregateFunction is an ordinary value of its storage type, so it needs no setting +s SimpleAggregateFunction(sum, UInt64) +-- without the setting the recorded state type is refused, not silently read as String +1 +-- schema inference recovers the aggregate types from the file +k UInt8 +u AggregateFunction(uniq, UInt8) +s SimpleAggregateFunction(sum, UInt64) +-- states round-trip: merged per group +0 17 1683 +1 17 1617 +2 17 1650 +-- an explicit structure overrides the recorded type +17 +-- a column chunk holding both a dictionary page and a plain page +['PLAIN','RLE_DICTIONARY'] +500 1 +-- a state nested in an Array +a Array(AggregateFunction(uniq, UInt64)) +[10,10] +-- a state pinned to a non-default version keeps that version +AggregateFunction(0, sumMap, Array(UInt8), Array(UInt32)) +1 +-- a refused state column can be skipped instead of refusing the whole file +k UInt8 +s SimpleAggregateFunction(sum, UInt64) +3 +-- an annotation naming an aggregate function this server does not have breaks only its column +1 +k UInt8 +s SimpleAggregateFunction(sum, UInt64) +3 +-- every annotated type this writer can produce still reads back, values intact +num SimpleAggregateFunction(sum, UInt64) +str SimpleAggregateFunction(anyLast, String) +nullable SimpleAggregateFunction(anyLast, Nullable(String)) +low_cardinality SimpleAggregateFunction(anyLast, LowCardinality(String)) +arr SimpleAggregateFunction(anyLast, Array(UInt64)) +m SimpleAggregateFunction(anyLast, Map(String, UInt64)) +d SimpleAggregateFunction(anyLast, Date) +dt SimpleAggregateFunction(anyLast, DateTime(\'UTC\')) +dt64 SimpleAggregateFunction(anyLast, DateTime64(4, \'UTC\')) +e SimpleAggregateFunction(anyLast, Enum8(\'a\' = 1, \'b\' = 2)) +ip4 SimpleAggregateFunction(anyLast, IPv4) +ip6 SimpleAggregateFunction(anyLast, IPv6) +i128 SimpleAggregateFunction(anyLast, Int128) +fs SimpleAggregateFunction(anyLast, FixedString(16)) +uu SimpleAggregateFunction(anyLast, UUID) +dec SimpleAggregateFunction(anyLast, Decimal(18, 4)) +state AggregateFunction(uniq, UInt64) +tup Tuple(AggregateFunction(uniq, UInt64), UInt64) +Row 1: +────── +num: 3 +str: abc +nullable: xyz +low_cardinality: lc +arr: [1,2] +m: {'k':7} +d: 2020-01-02 +dt: 2020-01-02 03:04:05 +dt64: 2020-01-02 03:04:05.1234 +e: b +ip4: 1.2.3.4 +ip6: ::1 +i128: 170141183460469231731687303715884105727 +hex(fs): 66697865640000000000000000000000 +uu: 00000000-0000-0000-0000-000000000001 +dec: 1.25 +uniq_state: 3 +uniq_in_tuple: 3 +plain_in_tuple: 7 +-- the same file reads back with the nullability the reader is told to infer +1 +1 +-- a recorded type that re-reads the stored bytes as something else is refused +1 +3 +-- a schema inferred with the setting on is not served to a query that has it off +k UInt8 +u AggregateFunction(uniq, UInt8) +1 +-- nor is a schema inferred with the refused column skipped served to a query that is not skipping +k UInt8 +1 diff --git a/tests/queries/0_stateless/04673_parquet_aggregate_function_state.sh b/tests/queries/0_stateless/04673_parquet_aggregate_function_state.sh new file mode 100755 index 0000000000000..3898fbb54416f --- /dev/null +++ b/tests/queries/0_stateless/04673_parquet_aggregate_function_state.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +SUFFIX="${CLICKHOUSE_DATABASE}_${RANDOM}" +FILE="agg_state_${SUFFIX}.parquet" +NESTED="agg_state_nested_${SUFFIX}.parquet" +MIXED="agg_state_mixed_${SUFFIX}.parquet" +VERSIONED="agg_state_versioned_${SUFFIX}.parquet" +REFUSED="agg_state_refused_${SUFFIX}.parquet" +SIMPLE="agg_state_simple_${SUFFIX}.parquet" +UNKNOWN="agg_state_unknown_${SUFFIX}.parquet" +MATRIX="agg_state_matrix_${SUFFIX}.parquet" +RETYPED_SOURCE="agg_state_retyped_source_${SUFFIX}.parquet" +RETYPED="agg_state_retyped_${SUFFIX}.parquet" +CACHED="agg_state_cached_${SUFFIX}.parquet" +CACHED_SKIP="agg_state_cached_skip_${SUFFIX}.parquet" + +cleanup() +{ + rm -f "${USER_FILES_PATH}/${FILE}" "${USER_FILES_PATH}/${NESTED}" "${USER_FILES_PATH}/${MIXED}" \ + "${USER_FILES_PATH}/${VERSIONED}" "${USER_FILES_PATH}/${REFUSED}" \ + "${USER_FILES_PATH}/${SIMPLE}" "${USER_FILES_PATH}/${UNKNOWN}" "${USER_FILES_PATH}/${MATRIX}" \ + "${USER_FILES_PATH}/${RETYPED_SOURCE}" "${USER_FILES_PATH}/${RETYPED}" \ + "${USER_FILES_PATH}/${CACHED}" "${USER_FILES_PATH}/${CACHED_SKIP}" +} +trap cleanup EXIT + +STATES="--allow_experimental_aggregate_function_states_in_open_formats=1" +SKIP="--input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference=1" +NO_CACHE="--schema_inference_use_cache_for_file=0" + +echo '-- without the setting writing a state is refused, as it was before Parquet supported states' +if ${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION file('${REFUSED}', Parquet) SELECT uniqState(number) AS u FROM numbers(10) +" 2>&1 | grep -q "allow_experimental_aggregate_function_states_in_open_formats" +then + echo 1 +fi + +echo '-- a SimpleAggregateFunction is an ordinary value of its storage type, so it needs no setting' +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION file('${SIMPLE}', Parquet) SELECT sumSimpleState(number) AS s FROM numbers(10) +" +${CLICKHOUSE_CLIENT} --query "DESC file('${SIMPLE}', Parquet)" + +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${FILE}', Parquet) + SELECT + number % 3 AS k, + uniqState(toUInt8(number % 17)) AS u, + sumSimpleState(number) AS s + FROM numbers(100) + GROUP BY k +" + +echo '-- without the setting the recorded state type is refused, not silently read as String' +if ${CLICKHOUSE_CLIENT} --query "DESC file('${FILE}', Parquet)" 2>&1 \ + | grep -q "allow_experimental_aggregate_function_states_in_open_formats" +then + echo 1 +fi + +echo '-- schema inference recovers the aggregate types from the file' +${CLICKHOUSE_CLIENT} ${STATES} --query "DESC file('${FILE}', Parquet)" + +echo '-- states round-trip: merged per group' +${CLICKHOUSE_CLIENT} ${STATES} --query " + SELECT k, uniqMerge(u), sum(s) FROM file('${FILE}', Parquet) GROUP BY k ORDER BY k +" + +echo '-- an explicit structure overrides the recorded type' +${CLICKHOUSE_CLIENT} --query " + SELECT uniqMerge(u) FROM file('${FILE}', Parquet, 'u AggregateFunction(uniq, UInt8)') +" + +echo '-- a column chunk holding both a dictionary page and a plain page' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${MIXED}', Parquet) + SELECT uniqState(number) AS u FROM numbers(2000) + GROUP BY number % 500 + SETTINGS output_format_parquet_max_dictionary_size = 20000, output_format_parquet_row_group_size = 100000 +" +${CLICKHOUSE_CLIENT} --query " + SELECT arraySort(tupleElement(arrayJoin(columns), 'encodings')) FROM file('${MIXED}', ParquetMetadata) +" +${CLICKHOUSE_CLIENT} ${STATES} --query " + SELECT count(), uniqMerge(u) = (SELECT uniq(number) FROM numbers(2000)) AS matches + FROM file('${MIXED}', Parquet) +" + +echo '-- a state nested in an Array' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${NESTED}', Parquet) + SELECT [uniqState(number), uniqState(number + 100)] AS a FROM numbers(10) +" +${CLICKHOUSE_CLIENT} ${STATES} --query "DESC file('${NESTED}', Parquet)" +${CLICKHOUSE_CLIENT} ${STATES} --query "SELECT arrayMap(x -> finalizeAggregation(x), a) FROM file('${NESTED}', Parquet)" + +echo '-- a state pinned to a non-default version keeps that version' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${VERSIONED}', Parquet) + SELECT CAST(sumMapState([number % 3], [toUInt32(number)]) AS AggregateFunction(0, sumMap, Array(UInt8), Array(UInt32))) AS m + FROM numbers(100) +" +grep -ao 'AggregateFunction(0, sumMap[^"]*' "${USER_FILES_PATH}/${VERSIONED}" +${CLICKHOUSE_CLIENT} ${STATES} --query " + SELECT sumMapMerge(m) = (SELECT sumMap([number % 3], [toUInt32(number)]) FROM numbers(100)) AS matches + FROM file('${VERSIONED}', Parquet) +" + +echo '-- a refused state column can be skipped instead of refusing the whole file' +${CLICKHOUSE_CLIENT} ${SKIP} ${NO_CACHE} --query "DESC file('${FILE}', Parquet)" +${CLICKHOUSE_CLIENT} ${SKIP} ${NO_CACHE} --query "SELECT sum(k) FROM file('${FILE}', Parquet)" + +echo '-- an annotation naming an aggregate function this server does not have breaks only its column' +python3 -c " +import sys +data = open(sys.argv[1], 'rb').read() +open(sys.argv[2], 'wb').write(data.replace(b'AggregateFunction(uniq,', b'AggregateFunction(zzzz,')) +" "${USER_FILES_PATH}/${FILE}" "${USER_FILES_PATH}/${UNKNOWN}" +if ${CLICKHOUSE_CLIENT} ${STATES} ${NO_CACHE} --query "DESC file('${UNKNOWN}', Parquet)" 2>&1 \ + | grep -q "for column u" +then + echo 1 +fi +${CLICKHOUSE_CLIENT} ${STATES} ${SKIP} ${NO_CACHE} --query "DESC file('${UNKNOWN}', Parquet)" +${CLICKHOUSE_CLIENT} ${STATES} ${SKIP} ${NO_CACHE} --query "SELECT sum(k) FROM file('${UNKNOWN}', Parquet)" + +echo '-- every annotated type this writer can produce still reads back, values intact' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${MATRIX}', Parquet) + SELECT + sumSimpleState(number) AS num, + anyLastSimpleState('abc') AS str, + anyLastSimpleState(CAST('xyz', 'Nullable(String)')) AS nullable, + CAST(anyLastSimpleState(toLowCardinality('lc')), + 'SimpleAggregateFunction(anyLast, LowCardinality(String))') AS low_cardinality, + anyLastSimpleState([toUInt64(1), toUInt64(2)]) AS arr, + anyLastSimpleState(map('k', toUInt64(7))) AS m, + anyLastSimpleState(toDate('2020-01-02')) AS d, + anyLastSimpleState(toDateTime('2020-01-02 03:04:05', 'UTC')) AS dt, + anyLastSimpleState(toDateTime64('2020-01-02 03:04:05.1234', 4, 'UTC')) AS dt64, + anyLastSimpleState(CAST('b', 'Enum8(''a'' = 1, ''b'' = 2)')) AS e, + anyLastSimpleState(toIPv4('1.2.3.4')) AS ip4, + anyLastSimpleState(toIPv6('::1')) AS ip6, + anyLastSimpleState(toInt128('170141183460469231731687303715884105727')) AS i128, + anyLastSimpleState(CAST('fixed', 'FixedString(16)')) AS fs, + anyLastSimpleState(toUUID('00000000-0000-0000-0000-000000000001')) AS uu, + anyLastSimpleState(toDecimal64('1.25', 4)) AS dec, + uniqState(number) AS state, + tuple(uniqState(number), toUInt64(7)) AS tup + FROM numbers(3) +" +${CLICKHOUSE_CLIENT} ${STATES} --query "DESC file('${MATRIX}', Parquet)" +${CLICKHOUSE_CLIENT} ${STATES} --query " + SELECT num, str, nullable, low_cardinality, arr, m, d, dt, dt64, e, ip4, ip6, toString(i128) AS i128, hex(fs), uu, dec, + finalizeAggregation(state) AS uniq_state, finalizeAggregation(tup.1) AS uniq_in_tuple, tup.2 AS plain_in_tuple + FROM file('${MATRIX}', Parquet) + FORMAT Vertical +" + +echo '-- the same file reads back with the nullability the reader is told to infer' +${CLICKHOUSE_CLIENT} ${STATES} ${NO_CACHE} --schema_inference_make_columns_nullable=1 --query " + SELECT count() FROM file('${MATRIX}', Parquet) +" +${CLICKHOUSE_CLIENT} ${STATES} ${NO_CACHE} --schema_inference_make_columns_nullable=0 --query " + SELECT count() FROM file('${MATRIX}', Parquet) +" + +echo '-- a recorded type that re-reads the stored bytes as something else is refused' +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION file('${RETYPED_SOURCE}', Parquet) + SELECT toUInt8(number) AS k, sumWithOverflowSimpleState(toInt64(number)) AS v FROM numbers(3) GROUP BY k +" +python3 -c " +import sys +old = b'SimpleAggregateFunction(sumWithOverflow, Int64)' +new = b'SimpleAggregateFunction(anyLast, DateTime64(9))' +assert len(old) == len(new) +data = open(sys.argv[1], 'rb').read() +assert old in data +open(sys.argv[2], 'wb').write(data.replace(old, new)) +" "${USER_FILES_PATH}/${RETYPED_SOURCE}" "${USER_FILES_PATH}/${RETYPED}" +if ${CLICKHOUSE_CLIENT} ${NO_CACHE} --query "DESC file('${RETYPED}', Parquet)" 2>&1 \ + | grep -q "reads as Int64" +then + echo 1 +fi +${CLICKHOUSE_CLIENT} ${SKIP} ${NO_CACHE} --query "SELECT sum(k) FROM file('${RETYPED}', Parquet)" + +echo '-- a schema inferred with the setting on is not served to a query that has it off' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${CACHED}', Parquet) + SELECT toUInt8(number % 3) AS k, uniqState(toUInt8(number)) AS u FROM numbers(30) GROUP BY k +" +# Ensure the file predates the cache entry despite one-second timestamp granularity. +touch -t 200001010000 "${USER_FILES_PATH}/${CACHED}" +${CLICKHOUSE_CLIENT} ${STATES} --query "DESC file('${CACHED}', Parquet)" +if ${CLICKHOUSE_CLIENT} --query "DESC file('${CACHED}', Parquet)" 2>&1 \ + | grep -q "allow_experimental_aggregate_function_states_in_open_formats" +then + echo 1 +fi + +echo '-- nor is a schema inferred with the refused column skipped served to a query that is not skipping' +${CLICKHOUSE_CLIENT} ${STATES} --query " + INSERT INTO FUNCTION file('${CACHED_SKIP}', Parquet) + SELECT toUInt8(number % 3) AS k, uniqState(toUInt8(number)) AS u FROM numbers(30) GROUP BY k +" +touch -t 200001010000 "${USER_FILES_PATH}/${CACHED_SKIP}" +${CLICKHOUSE_CLIENT} ${SKIP} --query "DESC file('${CACHED_SKIP}', Parquet)" +if ${CLICKHOUSE_CLIENT} --query "DESC file('${CACHED_SKIP}', Parquet)" 2>&1 \ + | grep -q "allow_experimental_aggregate_function_states_in_open_formats" +then + echo 1 +fi From ba724edc409d5abed879ccf81bf6255cc3e44ccf Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:30:26 +0100 Subject: [PATCH 2/6] Resolve conflicts in backport of #2301 Merged the PR's aggregate-state support with antalya-26.8's newer code (column reordering, manifest-schema tracking, excluded statistics columns, public `getState`); dropped `updateVersionFromRevision`, which doesn't exist on this branch. --- src/DataTypes/DataTypeAggregateFunction.cpp | 12 --- src/Formats/FormatSettings.h | 3 - .../Formats/Impl/Parquet/Decoding.cpp | 3 - .../Formats/Impl/Parquet/SchemaConverter.h | 10 +- src/Processors/Formats/Impl/Parquet/Write.h | 3 - .../Formats/Impl/ParquetBlockOutputFormat.cpp | 3 - .../Impl/ParquetV3BlockInputFormat.cpp | 10 +- .../DataLakes/Iceberg/DataFileStatistics.cpp | 14 +-- .../DataLakes/Iceberg/DataFileStatistics.h | 4 - .../DataLakes/Iceberg/IcebergMetadata.cpp | 33 +------ .../DataLakes/Iceberg/IcebergMetadata.h | 24 ++--- .../DataLakes/Iceberg/MetadataGenerator.cpp | 92 ++++--------------- .../DataLakes/Iceberg/SchemaProcessor.cpp | 35 +------ .../gtest_iceberg_metadata_generator.cpp | 23 ++--- 14 files changed, 49 insertions(+), 220 deletions(-) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 35e11af5154fd..8dadbc4f8f406 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -12,8 +12,6 @@ #include #include #include -<<<<<<< HEAD -======= #include #include #include @@ -22,7 +20,6 @@ #include #include #include ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include #include @@ -108,16 +105,7 @@ bool DataTypeAggregateFunction::isVersioned() const return function->isVersioned(); } -<<<<<<< HEAD -String DataTypeAggregateFunction::getNameImpl(bool with_version) const -======= -void DataTypeAggregateFunction::updateVersionFromRevision(size_t revision, bool if_empty) const -{ - setVersion(function->getVersionFromRevision(revision), if_empty); -} - String DataTypeAggregateFunction::getNameImpl(bool with_version, bool always_emit_version) const ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { WriteBufferFromOwnString stream; stream << "AggregateFunction("; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 91d0b7556dd30..91fd7add4b25c 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -420,11 +420,8 @@ struct FormatSettings double bloom_filter_bits_per_value = 10.5; size_t bloom_filter_flush_threshold_bytes = 1024 * 1024 * 128; bool allow_geoparquet_parser = true; -<<<<<<< HEAD bool spatial_filter_push_down = true; -======= bool allow_aggregate_function_states = false; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) bool write_geometadata = true; size_t max_dictionary_size = 1024 * 1024; } parquet{}; diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.cpp b/src/Processors/Formats/Impl/Parquet/Decoding.cpp index 1d43a86dd8858..6d0180225638f 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.cpp +++ b/src/Processors/Formats/Impl/Parquet/Decoding.cpp @@ -6,12 +6,9 @@ #include #include #include -<<<<<<< HEAD #include #include -======= #include ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h index b8d92dfd30bf7..d9bfbd4c41863 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h @@ -44,7 +44,9 @@ struct SchemaConverter /// The key is the parquet column name, without ColumnMapper. std::unordered_map geo_columns; -<<<<<<< HEAD + /// Type names from the `clickhouse.column_types` footer metadata. + std::unordered_map annotated_column_type_names; + /// If precomputed_geo_columns has a value it is used directly (including the empty-map case) /// and the constructor skips parsing the "geo" key-value metadata. This ensures that a /// failed parse caught by the caller (which leaves an empty map) does not cause @@ -52,12 +54,6 @@ struct SchemaConverter /// parse according to its own settings. SchemaConverter(const parq::FileMetaData &, const ReadOptions &, const Block *, std::optional> precomputed_geo_columns = std::nullopt); -======= - /// Type names from the `clickhouse.column_types` footer metadata. - std::unordered_map annotated_column_type_names; - - SchemaConverter(const parq::FileMetaData &, const ReadOptions &, const Block *); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) void prepareForReading(); NamesAndTypesList inferSchema(); diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index 2a7d2f60c4fc3..49d331da25977 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -27,11 +27,8 @@ struct WriteOptions bool output_datetime_as_uint32 = false; bool output_date_as_uint16 = false; bool output_enum_as_byte_array = false; -<<<<<<< HEAD bool output_timestamp_adjusted_to_utc_from_timezone = false; -======= bool allow_aggregate_function_states = false; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) /// Note: the meaning of some compression methods here is different from /// wrapReadBufferWithCompressionMethod: diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index fa2ddd8db999b..1cb11cb39fc9b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -55,11 +55,8 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea options.output_datetime_as_uint32 = format_settings.parquet.output_datetime_as_uint32; options.output_date_as_uint16 = format_settings.parquet.output_date_as_uint16; options.output_enum_as_byte_array = format_settings.parquet.output_enum_as_byte_array; -<<<<<<< HEAD options.output_timestamp_adjusted_to_utc_from_timezone = format_settings.parquet.output_timestamp_adjusted_to_utc_from_timezone; -======= options.allow_aggregate_function_states = format_settings.parquet.allow_aggregate_function_states; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) options.data_page_size = format_settings.parquet.data_page_size; options.write_batch_size = format_settings.parquet.write_batch_size; options.write_page_index = format_settings.parquet.write_page_index; diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index c5d2679afbaf8..9be9f8b6c5b6f 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -663,23 +663,17 @@ void registerParquetSchemaReader(FormatFactory & factory) [](const FormatSettings & settings) { return fmt::format( -<<<<<<< HEAD "schema_inference_make_columns_nullable={};schema_inference_make_json_columns_nullable={};" "enable_json_parsing={};max_parser_depth={};" - "local_time_as_utc={};allow_geoparquet_parser={};skip_columns_with_unsupported_types={}", + "local_time_as_utc={};allow_geoparquet_parser={};allow_aggregate_function_states={};" + "skip_columns_with_unsupported_types={}", settings.schema_inference_make_columns_nullable, settings.schema_inference_make_json_columns_nullable, settings.parquet.enable_json_parsing, settings.max_parser_depth, settings.parquet.local_time_as_utc, settings.parquet.allow_geoparquet_parser, -======= - "schema_inference_make_columns_nullable={};enable_json_parsing={};" - "allow_aggregate_function_states={};skip_columns_with_unsupported_types_in_schema_inference={}", - settings.schema_inference_make_columns_nullable, - settings.parquet.enable_json_parsing, settings.parquet.allow_aggregate_function_states, ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) settings.parquet.skip_columns_with_unsupported_types_in_schema_inference); }); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp index cbe3b84c9949b..8b0bb4e25198c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp @@ -195,14 +195,9 @@ std::vector> DataFileStatistics::getLowerBounds() const std::vector> result; for (size_t i = 0; i < ranges.size(); ++i) { -<<<<<<< HEAD - if (!isExcluded(i)) - result.push_back({field_ids[i], ranges[i].left}); -======= - if (!track_ranges[i]) + if (isExcluded(i) || !track_ranges[i]) continue; result.push_back({field_ids[i], ranges[i].left}); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return result; } @@ -212,14 +207,9 @@ std::vector> DataFileStatistics::getUpperBounds() const std::vector> result; for (size_t i = 0; i < ranges.size(); ++i) { -<<<<<<< HEAD - if (!isExcluded(i)) - result.push_back({field_ids[i], ranges[i].right}); -======= - if (!track_ranges[i]) + if (isExcluded(i) || !track_ranges[i]) continue; result.push_back({field_ids[i], ranges[i].right}); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return result; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h index 7ec665b3ea5df..6172cb8237a04 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h @@ -47,12 +47,8 @@ class DataFileStatistics std::vector excluded; std::vector column_sizes; std::vector null_counts; -<<<<<<< HEAD Ranges ranges; -======= - std::vector ranges; std::vector track_ranges; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) }; using DataFileStatisticsPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 8050f04f063fa..bbe1c80755ca5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -290,11 +290,7 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom } std::pair IcebergMetadata::getRelevantState( -<<<<<<< HEAD - const ContextPtr & context, bool force_fetch_latest_metadata) const -======= const ContextPtr & context, bool force_fetch_latest_metadata, SchemaParsing schema_parsing) const ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { const auto [metadata_version, metadata_file_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( object_storage, @@ -407,22 +403,18 @@ Int32 IcebergMetadata::parseTableSchema( { const auto format_version = metadata_object->getValue(f_format_version); -<<<<<<< HEAD if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) schema_processor.updateLastColumnId(metadata_object->getValue(f_last_column_id)); - /// v2 and v3 (and later) store schemas in the `schemas` array. Only v1 has a top-level `schema`. - /// Using the v1 parser first for v3 looks like a spec violation and logs a false warning. - if (format_version >= 2) -======= auto add_schema = [&](const Poco::JSON::Object::Ptr & schema) { if (schema_parsing == SchemaParsing::Parse) schema_processor.addIcebergTableSchema(schema, context_); }; - if (format_version == 2) ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) + /// v2 and v3 (and later) store schemas in the `schemas` array. Only v1 has a top-level `schema`. + /// Using the v1 parser first for v3 looks like a spec violation and logs a false warning. + if (format_version >= 2) { auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); add_schema(schema); @@ -475,14 +467,9 @@ static Poco::JSON::Object::Ptr traverseMetadataAndFindNecessarySnapshotObject( { if (!metadata_object->has(f_snapshots)) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "No snapshot set found in metadata for iceberg file"); -<<<<<<< HEAD if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) schema_processor->updateLastColumnId(metadata_object->getValue(f_last_column_id)); - auto schemas = metadata_object->get(f_schemas).extract(); - for (UInt32 j = 0; j < schemas->size(); ++j) -======= if (schema_parsing == IcebergMetadata::SchemaParsing::Parse) ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { auto schemas = metadata_object->get(f_schemas).extract(); for (UInt32 j = 0; j < schemas->size(); ++j) @@ -740,16 +727,7 @@ IcebergMetadata::getState( std::nullopt, std::nullopt); -<<<<<<< HEAD - std::tie(data_snapshot, table_state_snapshot.schema_id) = getStateImpl(local_context, metadata_object); -======= - /// The Iceberg `format-version` in the metadata file may differ from the value cached in - /// `persistent_components` when an external tool (e.g. Spark) upgrades the table between - /// queries. Downstream parsers determine the version they need from the Avro metadata of - /// each manifest list / manifest file, so we do not update the shared cached value here. - std::tie(data_snapshot, table_state_snapshot.schema_id) = getStateImpl(local_context, metadata_object, schema_parsing); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) table_state_snapshot.snapshot_id = data_snapshot ? std::optional{data_snapshot->snapshot_id} : std::nullopt; table_state_snapshot.metadata_version = metadata_version; table_state_snapshot.metadata_file_path = metadata_path; @@ -971,7 +949,6 @@ void IcebergMetadata::checkAlterIsPossible(const AlterCommands & commands) } } -<<<<<<< HEAD void IcebergMetadata::checkAlterPartitionIsPossible(const PartitionCommands & commands) const { checkTableRootIsQueriedPath("ALTER PARTITION"); @@ -1030,7 +1007,8 @@ void IcebergMetadata::alterPartitionDropImpl( std::move(catalog), storage_id); executor.run(); -======= +} + static void checkAggregateFunctionStatesAllowed(const String & column_name, const DataTypePtr & type, const ContextPtr & context) { if (!type || !needsClickHouseTypeAnnotation(type)) @@ -1045,7 +1023,6 @@ static void checkAggregateFunctionStatesAllowed(const String & column_name, cons "allow_experimental_aggregate_function_states_in_open_formats", column_name, type->getName()); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } void IcebergMetadata::alter( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index ac7e8b7eece07..02a168a3b3cd5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -127,14 +127,6 @@ class IcebergMetadata : public IDataLakeMetadata IcebergHistory getHistory(ContextPtr local_context) const; -<<<<<<< HEAD -======= - std::pair getRelevantState( - const ContextPtr & context, - bool force_fetch_latest_metadata = false, - SchemaParsing schema_parsing = SchemaParsing::Parse) const; - ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) /// Returns file records contributed by a single manifest list entry of `data_snapshot`. IcebergFiles getFilesForManifest( const Iceberg::IcebergDataSnapshotPtr & data_snapshot, @@ -260,10 +252,14 @@ class IcebergMetadata : public IDataLakeMetadata std::pair getRelevantState( const ContextPtr & context, - bool force_fetch_latest_metadata = false) const; + bool force_fetch_latest_metadata = false, + SchemaParsing schema_parsing = SchemaParsing::Parse) const; - std::pair - getState(const ContextPtr & local_context, const String & metadata_path, Int32 metadata_version) const; + std::pair getState( + const ContextPtr & local_context, + const String & metadata_path, + Int32 metadata_version, + SchemaParsing schema_parsing = SchemaParsing::Parse) const; const DB::Iceberg::PersistentTableComponents & getPersistentComponents() const { @@ -288,13 +284,7 @@ class IcebergMetadata : public IDataLakeMetadata Iceberg::IcebergDataSnapshotPtr createIcebergDataSnapshotFromSnapshotJSON(Poco::JSON::Object::Ptr snapshot_object, Int64 snapshot_id, ContextPtr local_context) const; std::pair -<<<<<<< HEAD - getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object) const; -======= getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object, SchemaParsing schema_parsing) const; - std::pair - getState(const ContextPtr & local_context, const String & metadata_path, Int32 metadata_version, SchemaParsing schema_parsing) const; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) Iceberg::IcebergDataSnapshotPtr getRelevantDataSnapshotFromTableStateSnapshot(Iceberg::TableStateSnapshot table_state_snapshot, ContextPtr local_context) const; StorageObjectStorageConfigurationPtr getConfiguration() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index f1b9265c8ac22..ff137bfd4240a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -1,10 +1,6 @@ #include -<<<<<<< HEAD -======= #include #include -#include ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) #include #include #include @@ -351,9 +347,6 @@ Poco::JSON::Object::Ptr MetadataGenerator::getCurrentSchema() const return current_schema; } -<<<<<<< HEAD -bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type, bool first, const String & after_column) const -======= static bool clickHouseTypeAnnotationMatches(const Poco::JSON::Object::Ptr & field, const DataTypePtr & type) { const bool expected = needsClickHouseTypeAnnotation(type); @@ -362,8 +355,7 @@ static bool clickHouseTypeAnnotationMatches(const Poco::JSON::Object::Ptr & fiel return !expected || field->getValue(Iceberg::f_clickhouse_type) == getClickHouseTypeAnnotationName(type); } -bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type) const ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) +bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type, bool first, const String & after_column) const { auto current_schema = findCurrentSchema(); if (!current_schema) @@ -383,11 +375,8 @@ bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypeP /// just used, so the ids of nested elements differ even for the very same type. return field->getValue(Iceberg::f_required) == expected_type.second && icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) -<<<<<<< HEAD + && clickHouseTypeAnnotationMatches(field, type) && isFieldAtRequestedPosition(fields, i, column_name, first, after_column); -======= - && clickHouseTypeAnnotationMatches(field, type); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return false; } @@ -438,7 +427,6 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy auto field = fields->getObject(i); if (field->getValue(Iceberg::f_name) != column_name) continue; -<<<<<<< HEAD /// A position-only `MODIFY COLUMN c FIRST` carries no type. if (type) @@ -447,16 +435,12 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy auto format_version = metadata_object->getValue(Iceberg::f_format_version); auto expected_type = Iceberg::getIcebergType(type, unused_field_id, format_version); if (field->getValue(Iceberg::f_required) != expected_type.second - || !icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first)) + || !icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) + || !clickHouseTypeAnnotationMatches(field, type)) return false; } return isFieldAtRequestedPosition(fields, i, column_name, first, after_column); -======= - return field->getValue(Iceberg::f_required) == expected_type.second - && icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) - && clickHouseTypeAnnotationMatches(field, type); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } return false; } @@ -950,39 +934,20 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (current_field->getValue(Iceberg::f_required) == new_type.second && icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) { -<<<<<<< HEAD - Iceberg::IcebergSchemaProcessor schema_processor( - context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); - auto current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); - - if (!current_ch_type->equals(*type)) -======= - auto existing_iceberg_type = current_field->get(Iceberg::f_type); - if (!existing_iceberg_type.isString()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot MODIFY COLUMN '{}': the requested and existing types both map to the same " - "Iceberg complex type, and the change cannot be recorded in the Iceberg schema", - column_name); - - DataTypePtr reconstructed_ch_type; + DataTypePtr current_ch_type; if (current_field->has(Iceberg::f_clickhouse_type)) { - reconstructed_ch_type + current_ch_type = DataTypeFactory::instance().get(current_field->getValue(Iceberg::f_clickhouse_type)); } else { - reconstructed_ch_type = Iceberg::IcebergSchemaProcessor::getSimpleType( - existing_iceberg_type.extract(), - context, - context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); - if (!current_field->getValue(Iceberg::f_required) && reconstructed_ch_type->canBeInsideNullable()) - reconstructed_ch_type = makeNullable(reconstructed_ch_type); + Iceberg::IcebergSchemaProcessor schema_processor( + context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); + current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); } - if (!reconstructed_ch_type->equals(*type)) ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) + if (!current_ch_type->equals(*type)) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot MODIFY COLUMN '{}' from {} to {}: both map to the same Iceberg type '{}' " @@ -990,16 +955,12 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, column_name, current_ch_type->getName(), type->getName(), -<<<<<<< HEAD current_field->get(Iceberg::f_type).toString()); - if (!needs_reposition) -======= - existing_iceberg_type.extract()); - /// `IDataType::equals` ignores the custom name of `SimpleAggregateFunction`. - if (clickHouseTypeAnnotationMatches(current_field, type)) ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) + if (!clickHouseTypeAnnotationMatches(current_field, type)) + type_changed = true; + else if (!needs_reposition) return false; } else @@ -1010,32 +971,9 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); -<<<<<<< HEAD type_changed = true; } break; -======= - if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); - - const auto next_schema_id = getNextSchemaId(metadata_object); - - current_schema = deepCopy(current_schema); - schema_fields = current_schema->getArray(Iceberg::f_fields); - current_field = schema_fields->getObject(i); - - current_field->set(Iceberg::f_type, new_type.first); - current_field->set(Iceberg::f_required, new_type.second); - if (needsClickHouseTypeAnnotation(type)) - current_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); - else - current_field->remove(Iceberg::f_clickhouse_type); - - metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); - current_schema->set(Iceberg::f_schema_id, next_schema_id); - metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); - return true; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } } @@ -1064,6 +1002,10 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, auto new_type = Iceberg::getIcebergType(type, last_column_id, format_version); target_field->set(Iceberg::f_type, new_type.first); target_field->set(Iceberg::f_required, new_type.second); + if (needsClickHouseTypeAnnotation(type)) + target_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); + else + target_field->remove(Iceberg::f_clickhouse_type); } if (needs_reposition) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 0a8cc1edbf6e9..ab7e2e1b5bd01 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -449,10 +449,6 @@ void IcebergSchemaProcessor::addIcebergTableSchema( std::lock_guard lock(mutex); -<<<<<<< HEAD -======= - Int32 schema_id = schema_ptr->getValue(f_schema_id); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); @@ -509,7 +505,6 @@ void IcebergSchemaProcessor::addIcebergTableSchema( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", schema_id); -<<<<<<< HEAD } } @@ -526,29 +521,6 @@ void IcebergSchemaProcessor::addIcebergTableSchema( field->getValue(f_id)); } - current_schema_id = schema_id; - auto clickhouse_schema = std::make_shared(); - String current_full_name{}; - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue(f_name); - bool required = field->getValue(f_required); - current_full_name = name; - auto type = getFieldType(field, f_type, context_, required, current_full_name, true); - clickhouse_schema->push_back(NameAndTypePair{name, type}); - clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; - clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); - } - clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; - iceberg_table_schemas_by_ids[schema_id] = schema_ptr; - if (source == SchemaSource::ManifestFile) - manifest_sourced_schema_ids.insert(schema_id); - current_schema_id = std::nullopt; -======= - return; - } - current_schema_id = schema_id; bool parsed = false; const auto rollback_unless_parsed = make_scope_guard( @@ -564,7 +536,6 @@ void IcebergSchemaProcessor::addIcebergTableSchema( std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & entry) { return entry.first.first == schema_id; }); }); - auto fields = schema_ptr->get(f_fields).extract(); auto clickhouse_schema = std::make_shared(); String current_full_name{}; for (size_t i = 0; i != fields->size(); ++i) @@ -578,11 +549,11 @@ void IcebergSchemaProcessor::addIcebergTableSchema( clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); } - - iceberg_table_schemas_by_ids[schema_id] = schema_ptr; clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; + iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + if (source == SchemaSource::ManifestFile) + manifest_sourced_schema_ids.insert(schema_id); parsed = true; ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } NameAndTypePair IcebergSchemaProcessor::getFieldCharacteristics(Int32 schema_version, Int32 source_id) const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp index b962ddb65deff..bb18c96c12173 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp @@ -6,13 +6,10 @@ #include #include -<<<<<<< HEAD -#include -#include -======= #include +#include #include ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) +#include #include #include #include @@ -542,7 +539,6 @@ Poco::JSON::Object::Ptr makeMetadataWithField( return metadata; } -<<<<<<< HEAD /// Ordered field names from the schema `current-schema-id` points at. std::vector getCurrentFieldNames(const Poco::JSON::Object::Ptr & metadata) { @@ -563,11 +559,7 @@ std::vector getCurrentFieldNames(const Poco::JSON::Object::Ptr & metadat return {}; } -/// The Iceberg type recorded for `name` in the schema `current-schema-id` points at. -Poco::Dynamic::Var findCurrentFieldType(const Poco::JSON::Object::Ptr & metadata, const String & name) -======= Poco::Dynamic::Var findCurrentFieldValue(const Poco::JSON::Object::Ptr & metadata, const String & name, const String & key) ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) { auto current_schema_id = metadata->getValue(f_current_schema_id); auto schemas = metadata->getArray(f_schemas); @@ -587,6 +579,12 @@ Poco::Dynamic::Var findCurrentFieldValue(const Poco::JSON::Object::Ptr & metadat return {}; } +/// The Iceberg type recorded for `name` in the schema `current-schema-id` points at. +Poco::Dynamic::Var findCurrentFieldType(const Poco::JSON::Object::Ptr & metadata, const String & name) +{ + return findCurrentFieldValue(metadata, name, f_type); +} + Poco::JSON::Object::Ptr makeMetadataWithAnnotatedField( const String & name, const Poco::Dynamic::Var & iceberg_type, bool required, const String & annotation) { @@ -815,7 +813,6 @@ TEST(IcebergMetadataGenerator, ModifyColumnWideningRecordsTheNewTypeInANewSchema EXPECT_EQ(stored_type.extract(), "long"); } -<<<<<<< HEAD /// --- Multi-argument transform (Iceberg V3 source-ids) tests --- @@ -1255,7 +1252,8 @@ TEST(IcebergMetadataGenerator, ModifyColumnSameComplexTypeWithDocOnlyRepositions auto stored_fields = stored_type.extract()->getArray(f_fields); ASSERT_EQ(stored_fields->size(), 1u); EXPECT_EQ(stored_fields->getObject(0)->getValue(f_type), "int"); -======= +} + TEST(IcebergMetadataGenerator, ModifyColumnToTheSameSimpleAggregateFunctionAddsNoSchema) { tryRegisterAggregateFunctions(); @@ -1332,7 +1330,6 @@ TEST(IcebergMetadataGenerator, AddColumnRecordsAndDetectsTheClickHouseTypeAnnota EXPECT_FALSE(gen.isAddColumnApplied( "extra", DataTypeFactory::instance().get("SimpleAggregateFunction(any, Nullable(String))"))); EXPECT_FALSE(gen.isAddColumnApplied("extra", makeNullable(std::make_shared()))); ->>>>>>> 234f8251bd2 (Support aggregate function states in Parquet and Iceberg (Altinity/ClickHouse#2301)) } #endif From 90ca479663c0baca2109e2482b0398dfda11191a Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:55:57 +0100 Subject: [PATCH 3/6] Fix CI failures (Build (amd_debug), Build (amd_asan_ubsan)) Changed the PR's new Iceberg unit tests to use antalya-26.8's API: renamed get/hasClickhouseTableSchemaById to the ClickHouse spelling and added the format_version argument to getIcebergType. --- .../Iceberg/tests/gtest_iceberg_schema_processor.cpp | 10 +++++----- .../Iceberg/tests/gtest_iceberg_type_mapping.cpp | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index f7894c22db282..6097db3a91404 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -576,7 +576,7 @@ TEST(IcebergSchemaProcessor, AnnotatedSimpleAggregateFunctionFieldNeedsNoSetting IcebergSchemaProcessor processor(context); processor.addIcebergTableSchema(schema, context); - auto columns = processor.getClickhouseTableSchemaById(0); + auto columns = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(columns->size(), 1u); EXPECT_EQ(columns->front().name, "s"); EXPECT_EQ(columns->front().type->getName(), "SimpleAggregateFunction(sum, Int64)"); @@ -607,7 +607,7 @@ TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSuccee EXPECT_EQ(e.code(), DB::ErrorCodes::SUPPORT_IS_DISABLED) << e.message(); } - EXPECT_FALSE(processor.hasClickhouseTableSchemaById(0)); + EXPECT_FALSE(processor.hasClickHouseTableSchemaById(0)); EXPECT_FALSE(processor.tryGetFieldCharacteristics(0, 1).has_value()); EXPECT_FALSE(processor.tryGetColumnIDByName(0, "a").has_value()); @@ -615,7 +615,7 @@ TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSuccee const std::string allowed_json = R"json({"schema-id":0,"fields":[{"id":1,"name":"a2","required":false,"type":"int"},{"id":2,"name":"u","required":false,"type":"binary","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json"; ASSERT_NO_THROW(processor.addIcebergTableSchema(parseSchema(allowed_json), allowed_context)); - auto columns = processor.getClickhouseTableSchemaById(0); + auto columns = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(columns->size(), 2u); EXPECT_EQ(columns->front().name, "a2"); auto field = processor.getFieldCharacteristics(0, 1); @@ -647,14 +647,14 @@ TEST(IcebergSchemaProcessor, SkippingTheSchemaNeitherAppliesTheGateNorPublishesT metadata, processor, context, log, DB::IcebergMetadata::SchemaParsing::Skip)); EXPECT_EQ(schema_id, 0); - EXPECT_FALSE(processor.hasClickhouseTableSchemaById(0)); + EXPECT_FALSE(processor.hasClickHouseTableSchemaById(0)); EXPECT_FALSE(processor.tryGetFieldCharacteristics(0, 1).has_value()); EXPECT_FALSE(processor.tryGetColumnIDByName(0, "u").has_value()); auto allowed_context = contextWithAggregateFunctionStates(true); ASSERT_NO_THROW(DB::IcebergMetadata::parseTableSchema( metadata, processor, allowed_context, log, DB::IcebergMetadata::SchemaParsing::Parse)); - auto columns = processor.getClickhouseTableSchemaById(0); + auto columns = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(columns->size(), 1u); EXPECT_EQ(columns->front().name, "u"); EXPECT_EQ(columns->front().type->getName(), "AggregateFunction(uniq, UInt64)"); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp index 58e747005e3c9..193b1f2e74945 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp @@ -184,7 +184,8 @@ TEST(IcebergTypeMapping, AggregateFunctionMapsToBinary) tryRegisterAggregateFunctions(); auto type = DataTypeFactory::instance().get("AggregateFunction(uniq, UInt64)"); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "binary"); EXPECT_TRUE(required); From 69d8a697d15e1585f875bc44b55a43b0d39c2429 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:14:45 +0100 Subject: [PATCH 4/6] Store Iceberg ClickHouse type annotations in table properties The field-level `clickhouse.type` key was dropped by REST catalogs and by commits from other engines, so aggregate-state columns silently read back as `String` or their storage type. Annotations are now table properties `clickhouse.type..`; a field's type in a schema is the entry with the greatest schema id not above it, so older schemas keep theirs. - Pass annotations explicitly to `IcebergSchemaProcessor::addIcebergTableSchema` and store them per schema; re-parse a schema transactionally when they change. - Validate an annotation exactly against the field's Iceberg type. - Ignore `clickhouse.type.*` properties that are not annotation keys. - Register metadata schemas before maintenance operations walk manifests. - Send property changes with REST `updateSchema`, forward properties on create. - Move the docs sections into the engine and format `Documentation` sources. Related: https://github.com/Altinity/ClickHouse/pull/2680#issuecomment-6425997300 --- .../table-engines/integrations/iceberg.mdx | 62 ----- docs/reference/formats/Parquet/Parquet.mdx | 35 --- src/Core/Settings.cpp | 6 +- src/DataTypes/DataTypeAggregateFunction.cpp | 12 +- src/DataTypes/DataTypeAggregateFunction.h | 4 +- .../gtest_annotated_type_matches_derived.cpp | 112 ++++---- src/Databases/DataLake/GlueCatalog.cpp | 1 + src/Databases/DataLake/GlueCatalog.h | 1 + src/Databases/DataLake/ICatalog.cpp | 1 + src/Databases/DataLake/ICatalog.h | 2 + src/Databases/DataLake/RestCatalog.cpp | 90 ++++++- src/Databases/DataLake/RestCatalog.h | 7 +- src/Databases/DataLake/UnityV2Catalog.cpp | 4 +- src/Databases/DataLake/UnityV2Catalog.h | 1 + .../gtest_rest_catalog_update_metadata.cpp | 22 ++ .../Formats/Impl/Parquet/SchemaConverter.cpp | 2 +- .../Impl/ParquetV3BlockInputFormat.cpp | 35 +++ .../DataLakes/Iceberg/Compaction.cpp | 12 +- .../Iceberg/ExpireSnapshotsExecute.cpp | 3 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 8 +- .../Iceberg/ManifestFileIterator.cpp | 3 +- .../DataLakes/Iceberg/MetadataGenerator.cpp | 194 +++++--------- .../DataLakes/Iceberg/Mutations.cpp | 10 +- .../Iceberg/RemoveOrphanFilesExecute.cpp | 3 + .../DataLakes/Iceberg/SchemaProcessor.cpp | 140 +++++++--- .../DataLakes/Iceberg/SchemaProcessor.h | 24 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 175 ++++++++++++- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 18 ++ .../gtest_iceberg_metadata_generator.cpp | 36 ++- .../tests/gtest_iceberg_schema_processor.cpp | 241 +++++++++++++----- .../registerStorageObjectStorage.cpp | 66 +++++ .../integration/test_database_iceberg/test.py | 64 +++++ .../test_aggregate_function_states.py | 11 +- ...tion_state_annotation_per_schema.reference | 18 ++ ...te_function_state_annotation_per_schema.sh | 47 ++++ 35 files changed, 1043 insertions(+), 427 deletions(-) create mode 100644 tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.reference create mode 100755 tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.sh diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index c39f0ba1bdf0c..7a05dab165b5d 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -106,68 +106,6 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `map` | `Map` | | `struct` | `Tuple` | -### Aggregate function states {#aggregate-function-states} - -Iceberg has no aggregate-state type, so ClickHouse stores the two aggregate-state types as ordinary -Iceberg values and records the ClickHouse type name in a `clickhouse.type` key on the schema field: - -| ClickHouse type | Iceberg type | Stored as | -|---|---|---| -| `AggregateFunction(f, T...)` | `binary` | The serialized state, the same bytes `f` writes with the `-State` combinator | -| `SimpleAggregateFunction(f, T)` | whatever `T` maps to | An ordinary value of type `T` | - -Other query engines ignore the key and see a plain `binary` (or `T`-typed) column. - -Creating such a table requires -[`allow_experimental_aggregate_function_states_in_open_formats`](/operations/settings/settings#allow_experimental_aggregate_function_states_in_open_formats), -and so does every query that reads a column whose `clickhouse.type` names an `AggregateFunction` - an -`INSERT` or an `ALTER TABLE ... EXPORT PART` as much as a `SELECT`. A state is an opaque blob handed -to the deserializer of the aggregate function the table's metadata names, so with the setting enabled -it is that metadata, not the query, which chooses the deserializer; keep it disabled for tables from -untrusted sources, where such a field is then rejected rather than read as `String`. A -`SimpleAggregateFunction` column is not gated on read, holding ordinary values of its storage type. - -The setting is read from the query that parses the schema, so a session `SET` or a `SETTINGS` clause -on that query supplies it. The `iceberg*` table functions build a fresh storage object per query, so -for them it takes effect per query; the table engine parses the schema once and keeps it, so the -query that first touches the table after `ATTACH` decides, and that outcome holds - including for -queries that do not set it - until `DETACH TABLE` or a server restart. - -Writing goes through the ordinary Parquet writer, which takes the same setting from its format -settings. An object storage table engine freezes its format settings at `CREATE TABLE` - the server -settings plus that query's `SETTINGS` clause, session settings ignored - so for `INSERT` the setting -must also be in that clause; given only on the `INSERT`, it does not reach the writer: - -```sql -CREATE TABLE agg (k UInt32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64)) -ENGINE = IcebergLocal('/path/to/table/') -PARTITION BY k -SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; - --- The states merge exactly as they do in an AggregatingMergeTree table. -SELECT k, uniqMerge(u), sum(s) FROM agg GROUP BY k -SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; -``` - -`ALTER TABLE ... EXPORT PART` and `ALTER TABLE ... EXPORT PARTITION` from an `AggregatingMergeTree` -table into such an Iceberg table work as well, so a partition of pre-aggregated states can be moved -into the lake without finalizing it. Both take the setting off the `ALTER` query itself: - -```sql -ALTER TABLE mt EXPORT PART 'all_1_1_0' TO TABLE agg -SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; - --- EXPORT PARTITION, which only ReplicatedMergeTree implements, records the value in its manifest, --- so every replica executing the task applies it in place of its own profile. -ALTER TABLE rmt EXPORT PARTITION ID '1' TO TABLE agg -SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; -``` - -The Parquet footer of each data file carries the same information, so `DESCRIBE file('data.parquet')` -reports the aggregate types too; that path goes through Parquet schema inference, gated by the same -setting. See -[aggregate function states in Parquet](/interfaces/formats/Parquet#aggregate-function-states). - ## Schema evolution {#schema-evolution} ClickHouse supports reading Iceberg tables whose schema has evolved over time. This includes tables where columns have been added, removed, or reordered, as well as columns changed from required to nullable. Additionally, the following type casts are supported: diff --git a/docs/reference/formats/Parquet/Parquet.mdx b/docs/reference/formats/Parquet/Parquet.mdx index 45812fb206506..ef6cd249d41dc 100644 --- a/docs/reference/formats/Parquet/Parquet.mdx +++ b/docs/reference/formats/Parquet/Parquet.mdx @@ -115,41 +115,6 @@ On write, top-level columns of type `Point`, `MultiPoint`, `LineString`, `Polygo Geometry columns must appear at the root of the schema or nested inside `Tuple` (`struct`); nesting them inside `Array` or `Map` is not supported. `Nullable` is not supported for geo columns either. -## Aggregate function states {#aggregate-function-states} - -Parquet has no aggregate-state type, so ClickHouse writes an [`AggregateFunction`](/sql-reference/data-types/aggregatefunction.md) column as a plain `BYTE_ARRAY` holding the serialized state - the same bytes the `-State` combinator produces - with no `STRING`/`UTF8` logical type, since a state is arbitrary binary data rather than text. A [`SimpleAggregateFunction(f, T)`](/sql-reference/data-types/simpleaggregatefunction.md) column is written as an ordinary value of type `T`. - -Neither type can be recovered from the Parquet schema alone: every `AggregateFunction` state is just a binary column, and a `SimpleAggregateFunction` is indistinguishable from its storage type. ClickHouse therefore records the type names in a `clickhouse.column_types` key in the file-level Parquet metadata, as a JSON object mapping column name to ClickHouse type name. The recorded name includes the state version, which is what pins the serialized layout across server versions. - -Both writing an `AggregateFunction` column and reconstructing one from that metadata on read are gated by [`allow_experimental_aggregate_function_states_in_open_formats`](/operations/settings/settings#allow_experimental_aggregate_function_states_in_open_formats), which is disabled by default. A state is an opaque blob passed to the deserializer of whichever aggregate function the file names, so with the setting enabled it is the file, not the query, choosing that deserializer; keep it disabled for files from untrusted sources, where such a file is then rejected rather than read as `String`. With it disabled, writing is refused with `UNKNOWN_TYPE`, exactly as in versions that did not support states in Parquet at all. `SimpleAggregateFunction` is gated in neither direction: it is stored as an ordinary value of its storage type and has always been written that way. - -With the setting enabled the states round-trip without any hint: - -```sql -SET allow_experimental_aggregate_function_states_in_open_formats = 1; - -INSERT INTO FUNCTION file('states.parquet') -SELECT k, uniqState(v) AS u, sumSimpleState(v) AS s FROM source GROUP BY k; - -DESCRIBE file('states.parquet'); --- k UInt64 --- u AggregateFunction(uniq, UInt64) --- s SimpleAggregateFunction(sum, UInt64) - -SELECT k, uniqMerge(u), sum(s) FROM file('states.parquet') GROUP BY k; -``` - -An explicit structure overrides the recorded metadata and is unaffected by the setting, so a state column can also be read as `String` to get the raw serialized bytes: - -```sql -SELECT uniqMerge(CAST(u AS AggregateFunction(uniq, UInt64))) -FROM file('states.parquet', Parquet, 'u String'); -``` - -Reading fails rather than guessing if the recorded type does not describe the data actually in the file. Min/max statistics are never used for pruning a state column, because bounds over serialized states are meaningless. - -Other query engines see a plain binary (or `T`-typed) column and ignore the metadata key. The same mechanism backs [aggregate-state support in Iceberg tables](/engines/table-engines/integrations/iceberg#aggregate-function-states), whose data files are Parquet. - ## Example usage {#example-usage} ### Inserting data {#inserting-data} diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index d7dea69f88d6f..189b0ab137512 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8137,12 +8137,12 @@ Allow parsing Iceberg `geometry` and `geography` field types as ClickHouse `Geom DECLARE(Bool, allow_experimental_aggregate_function_states_in_open_formats, false, R"( Allow `AggregateFunction` states in Parquet files and Iceberg tables, and `SimpleAggregateFunction` columns in Iceberg tables. A state is stored as opaque binary, with its ClickHouse type recorded in -`clickhouse.column_types` (Parquet file metadata) or `clickhouse.type` (Iceberg schema field). +`clickhouse.column_types` (Parquet file metadata) or `clickhouse.type..` (Iceberg table properties). Gates both writing such columns and reconstructing their types on read. When enabled, the data, not the query, chooses the deserializer for the stored bytes, so keep it disabled for untrusted sources. -More [in Parquet](/interfaces/formats/Parquet#aggregate-function-states) and -[in Iceberg](/engines/table-engines/integrations/iceberg#aggregate-function-states). +More [in Parquet](/reference/formats/Parquet/Parquet#aggregate-function-states) and +[in Iceberg](/reference/engines/table-engines/integrations/iceberg#aggregate-function-states). )", 0) \ DECLARE(Bool, show_data_lake_catalogs_in_system_tables, false, R"( Enables showing data lake catalogs in system tables. diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 8dadbc4f8f406..3da53d0239336 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -694,7 +694,7 @@ bool parquetWriterCouldProduce(const DataTypePtr & annotated, const DataTypePtr } -bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataTypePtr & derived_type, bool strict) +bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataTypePtr & derived_type) { auto annotated = removeLowCardinalityAndNullable(annotated_type); auto derived = removeLowCardinalityAndNullable(derived_type); @@ -710,7 +710,7 @@ bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataT if (!derived_array) return false; return annotatedTypeMatchesDerived( - assert_cast(*annotated).getNestedType(), derived_array->getNestedType(), strict); + assert_cast(*annotated).getNestedType(), derived_array->getNestedType()); } case TypeIndex::Tuple: { @@ -719,7 +719,7 @@ bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataT if (!derived_tuple || derived_tuple->getElements().size() != annotated_tuple.getElements().size()) return false; for (size_t i = 0; i < annotated_tuple.getElements().size(); ++i) - if (!annotatedTypeMatchesDerived(annotated_tuple.getElement(i), derived_tuple->getElement(i), strict)) + if (!annotatedTypeMatchesDerived(annotated_tuple.getElement(i), derived_tuple->getElement(i))) return false; return true; } @@ -729,11 +729,11 @@ bool annotatedTypeMatchesDerived(const DataTypePtr & annotated_type, const DataT const auto * derived_map = typeid_cast(derived.get()); if (!derived_map) return false; - return annotatedTypeMatchesDerived(annotated_map.getKeyType(), derived_map->getKeyType(), strict) - && annotatedTypeMatchesDerived(annotated_map.getValueType(), derived_map->getValueType(), strict); + return annotatedTypeMatchesDerived(annotated_map.getKeyType(), derived_map->getKeyType()) + && annotatedTypeMatchesDerived(annotated_map.getValueType(), derived_map->getValueType()); } default: - return !strict || parquetWriterCouldProduce(annotated, derived); + return parquetWriterCouldProduce(annotated, derived); } } diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h index f208be37dd824..0ce507f3d640e 100644 --- a/src/DataTypes/DataTypeAggregateFunction.h +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -113,7 +113,7 @@ bool needsClickHouseTypeAnnotation(const DataTypePtr & type); /// Returns an annotation name that preserves explicit aggregate-state versions. String getClickHouseTypeAnnotationName(const DataTypePtr & type); -/// Checks an annotation against the schema-derived type. Strict mode also verifies the Parquet mapping. -bool annotatedTypeMatchesDerived(const DataTypePtr & annotated, const DataTypePtr & derived, bool strict = false); +/// Checks an annotation against the type derived from a Parquet schema, including the Parquet writer's mapping. +bool annotatedTypeMatchesDerived(const DataTypePtr & annotated, const DataTypePtr & derived); } diff --git a/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp b/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp index 3c1cd1a782424..5654db7e6b42c 100644 --- a/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp +++ b/src/DataTypes/tests/gtest_annotated_type_matches_derived.cpp @@ -20,11 +20,6 @@ bool matches(const String & annotated, const String & derived) return annotatedTypeMatchesDerived(typeFromString(annotated), typeFromString(derived)); } -bool matchesStrictly(const String & annotated, const String & derived) -{ - return annotatedTypeMatchesDerived(typeFromString(annotated), typeFromString(derived), /*strict=*/ true); -} - } TEST(AnnotatedTypeMatchesDerived, AggregateFunctionNeedsAStringPosition) @@ -35,9 +30,6 @@ TEST(AnnotatedTypeMatchesDerived, AggregateFunctionNeedsAStringPosition) EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "Int64")); EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "FixedString(16)")); EXPECT_FALSE(matches("AggregateFunction(uniq, UInt64)", "Array(String)")); - - EXPECT_TRUE(matchesStrictly("AggregateFunction(uniq, UInt64)", "String")); - EXPECT_FALSE(matchesStrictly("AggregateFunction(uniq, UInt64)", "Int64")); } TEST(AnnotatedTypeMatchesDerived, ContainersAreMatchedElementwise) @@ -49,80 +41,76 @@ TEST(AnnotatedTypeMatchesDerived, ContainersAreMatchedElementwise) EXPECT_TRUE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Map(String, String)")); EXPECT_FALSE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Map(String, Int64)")); - EXPECT_TRUE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a String, b Int64)")); EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a Int64, b String)")); EXPECT_FALSE(matches("Array(AggregateFunction(uniq, UInt64))", "String")); EXPECT_FALSE(matches("Map(String, AggregateFunction(uniq, UInt64))", "Array(Tuple(String, String))")); EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "String")); EXPECT_FALSE(matches("Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a String)")); - EXPECT_TRUE(matches("SimpleAggregateFunction(sum, UInt64)", "Int64")); EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Nullable(String))", "Nullable(String)")); - EXPECT_TRUE(matches("Array(SimpleAggregateFunction(sum, UInt64))", "Array(Int64)")); } TEST(AnnotatedTypeMatchesDerived, StrictModeRefusesAnAnnotationThatRetypesTheColumn) { - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(9))", "Int64")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "Int64")); - EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(9))", "Int64")); - - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "Int32")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv4)", "Int32")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Float64)", "Int64")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, String)", "FixedString(16)")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 2)")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(3)")); - EXPECT_FALSE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(9)")); - - EXPECT_FALSE(matchesStrictly( + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, DateTime64(9))", "Int64")); + EXPECT_FALSE(matches("SimpleAggregateFunction(sum, UInt64)", "Int64")); + + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, Date)", "Int32")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, IPv4)", "Int32")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, Float64)", "Int64")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, String)", "FixedString(16)")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 2)")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(3)")); + EXPECT_FALSE(matches("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(9)")); + + EXPECT_FALSE(matches( "Tuple(a AggregateFunction(uniq, UInt64), b DateTime64(9))", "Tuple(a String, b Int64)")); - EXPECT_FALSE(matchesStrictly("Array(SimpleAggregateFunction(sum, UInt64))", "Array(Int64)")); - EXPECT_FALSE(matchesStrictly( + EXPECT_FALSE(matches("Array(SimpleAggregateFunction(sum, UInt64))", "Array(Int64)")); + EXPECT_FALSE(matches( "Map(String, SimpleAggregateFunction(anyLast, DateTime64(9)))", "Map(String, Int64)")); } TEST(AnnotatedTypeMatchesDerived, StrictModeAcceptsWhatTheParquetWriterProduces) { - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "UInt64")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, String)", "String")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, UUID)", "UUID")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 4)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, FixedString(16))", "FixedString(16)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date32)", "Date32")); - - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(sum, UInt64)", "Nullable(UInt64)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Nullable(String))", "String")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Nullable(String))", "Nullable(String)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Array(UInt64))", "Array(Nullable(UInt64))")); - EXPECT_TRUE(matchesStrictly( + EXPECT_TRUE(matches("SimpleAggregateFunction(sum, UInt64)", "UInt64")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, String)", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, UUID)", "UUID")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Decimal(18, 4))", "Decimal(18, 4)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, FixedString(16))", "FixedString(16)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Date32)", "Date32")); + + EXPECT_TRUE(matches("SimpleAggregateFunction(sum, UInt64)", "Nullable(UInt64)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Nullable(String))", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Nullable(String))", "Nullable(String)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Array(UInt64))", "Array(Nullable(UInt64))")); + EXPECT_TRUE(matches( "SimpleAggregateFunction(anyLast, Map(String, UInt64))", "Map(String, Nullable(UInt64))")); - EXPECT_TRUE(matchesStrictly( + EXPECT_TRUE(matches( "Tuple(a AggregateFunction(uniq, UInt64), b UInt64)", "Tuple(a Nullable(String), b Nullable(UInt64))")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, LowCardinality(String))", "String")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, LowCardinality(String))", "Nullable(String)")); - EXPECT_TRUE(matchesStrictly( + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, LowCardinality(String))", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, LowCardinality(String))", "Nullable(String)")); + EXPECT_TRUE(matches( "SimpleAggregateFunction(anyLast, LowCardinality(Nullable(String)))", "Nullable(String)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "Date32")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Date)", "UInt16")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime)", "DateTime64(3, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime)", "UInt32")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(9))", "DateTime64(9, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(0))", "DateTime64(3, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(6, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, DateTime64(7))", "DateTime64(9, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time)", "DateTime64(6, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time64(3))", "DateTime64(6, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Time64(9))", "DateTime64(9, 'UTC')")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "String")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "Int8")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "String")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "Int16")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv4)", "UInt32")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, IPv6)", "FixedString(16)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, Int128)", "FixedString(16)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, UInt256)", "FixedString(32)")); - EXPECT_TRUE(matchesStrictly("SimpleAggregateFunction(anyLast, FixedString(16))", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Date)", "Date32")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Date)", "UInt16")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime)", "DateTime64(3, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime)", "UInt32")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(9))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(0))", "DateTime64(3, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(4))", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, DateTime64(7))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Time)", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Time64(3))", "DateTime64(6, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Time64(9))", "DateTime64(9, 'UTC')")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Enum8('a' = 1))", "Int8")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "String")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Enum16('a' = 1))", "Int16")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, IPv4)", "UInt32")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, IPv6)", "FixedString(16)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, Int128)", "FixedString(16)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, UInt256)", "FixedString(32)")); + EXPECT_TRUE(matches("SimpleAggregateFunction(anyLast, FixedString(16))", "String")); } diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index e97d585c1db31..df61cf012deb4 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -1261,6 +1261,7 @@ bool GlueCatalog::updateSchema( Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, Poco::JSON::Object::Ptr /*metadata*/, + Poco::JSON::Object::Ptr /*previous_properties*/, const DB::ForwardedAuthTokenPtr & auth_token) const { std::vector columns; diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 0dd11e7b199a7..682bdca3bad1b 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -95,6 +95,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const override; void dropTable( diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 9cc28afc48917..579d9a39be4fc 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -531,6 +531,7 @@ bool ICatalog::updateSchema( Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, Poco::JSON::Object::Ptr /*metadata*/, + Poco::JSON::Object::Ptr /*previous_properties*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateSchema is not implemented"); diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 5e092e92b75f2..2f442d8308301 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -339,6 +339,7 @@ class ICatalog /// `new_schema` is the full new schema object and `previous_schema_id` is the schema id that the /// catalog is expected to currently have - used to build the optimistic-concurrency requirement /// (`assert-current-schema-id`) on transactional catalogs (e.g. Iceberg REST). + /// `previous_properties` are the table properties before the change. /// Returns `false` on a recoverable conflict so the caller can retry, throws otherwise. virtual bool updateSchema( const String & namespace_name, @@ -348,6 +349,7 @@ class ICatalog Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Drop table from catalog. diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 6160a8bb57ed2..0f1a8241294b2 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -353,7 +353,8 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( Poco::JSON::Object::Ptr metadata, Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, - Int32 new_last_column_id) + Int32 new_last_column_id, + Poco::JSON::Object::Ptr previous_properties) { Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; { @@ -381,11 +382,14 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( schema_for_rest->set(IDENTIFIER_FIELD_IDS, empty_identifier_field_ids); } + const auto new_schema_id = new_schema->getValue(DB::Iceberg::f_schema_id); + /// The change keeps the current schema and touches only the table properties. + const bool schema_unchanged = new_schema_id == previous_schema_id; + std::optional existing_equivalent_schema_id; - if (metadata && metadata->has(DB::Iceberg::f_schemas)) + if (!schema_unchanged && metadata && metadata->has(DB::Iceberg::f_schemas)) { auto schemas = metadata->getArray(DB::Iceberg::f_schemas); - auto new_schema_id = new_schema->getValue(DB::Iceberg::f_schema_id); for (UInt32 i = 0; i < schemas->size(); ++i) { auto existing_schema = schemas->getObject(i); @@ -399,6 +403,36 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( } } + /// The catalog keeps the id of the equivalent schema, so the ClickHouse type annotations recorded for the new id + /// are not sent and the ones in effect for the equivalent schema must already be the same. + std::unordered_set skipped_properties; + if (existing_equivalent_schema_id.has_value()) + { + const auto fields = DB::Iceberg::IcebergSchemaProcessor::traverseSchema(new_schema->getArray(DB::Iceberg::f_fields)); + const auto existing_annotations = DB::Iceberg::getClickHouseTypeAnnotations(metadata, *existing_equivalent_schema_id); + const auto new_annotations = DB::Iceberg::getClickHouseTypeAnnotations(metadata, new_schema_id); + for (const auto & [name, field_id] : fields) + { + const auto existing_it = existing_annotations.find(static_cast(field_id)); + const auto new_it = new_annotations.find(static_cast(field_id)); + const String existing_annotation = existing_it == existing_annotations.end() ? String{} : existing_it->second; + const String new_annotation = new_it == new_annotations.end() ? String{} : new_it->second; + if (existing_annotation != new_annotation) + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "Cannot commit schema {} of table {}.{}: the catalog reuses the equivalent schema {} for it, " + "where column '{}' has ClickHouse type annotation '{}' instead of '{}'", + new_schema_id, + namespace_name, + table_name, + *existing_equivalent_schema_id, + name, + existing_annotation, + new_annotation); + skipped_properties.insert(DB::Iceberg::getClickHouseTypePropertyKey(static_cast(field_id), new_schema_id)); + } + } + Poco::JSON::Array::Ptr updates = new Poco::JSON::Array; if (existing_equivalent_schema_id.has_value()) { @@ -407,7 +441,7 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( set_current_schema->set("schema-id", *existing_equivalent_schema_id); updates->add(set_current_schema); } - else + else if (!schema_unchanged) { { Poco::JSON::Object::Ptr add_schema = new Poco::JSON::Object; @@ -424,6 +458,45 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( } } + if (previous_properties && metadata) + { + Poco::JSON::Object::Ptr properties + = metadata->has(DB::Iceberg::f_properties) ? metadata->getObject(DB::Iceberg::f_properties) : nullptr; + + Poco::JSON::Object::Ptr changed = new Poco::JSON::Object; + if (properties) + { + for (const auto & [key, value] : *properties) + { + if (skipped_properties.contains(key)) + continue; + if (!previous_properties->has(key) || previous_properties->get(key).toString() != value.toString()) + changed->set(key, value); + } + } + if (changed->size() > 0) + { + Poco::JSON::Object::Ptr set_properties = new Poco::JSON::Object; + set_properties->set("action", "set-properties"); + set_properties->set("updates", changed); + updates->add(set_properties); + } + + Poco::JSON::Array::Ptr removed = new Poco::JSON::Array; + for (const auto & [key, value] : *previous_properties) + { + if (!properties || !properties->has(key)) + removed->add(key); + } + if (removed->size() > 0) + { + Poco::JSON::Object::Ptr remove_properties = new Poco::JSON::Object; + remove_properties->set("action", "remove-properties"); + remove_properties->set("removals", removed); + updates->add(remove_properties); + } + } + request_body->set("updates", updates); return request_body; } @@ -2730,6 +2803,12 @@ bool RestCatalog::createTable( if (metadata_content->has("format-version")) properties->set("format-version", std::to_string(metadata_content->getValue("format-version"))); + if (metadata_content->has(DB::Iceberg::f_properties)) + { + for (const auto & [key, value] : *metadata_content->getObject(DB::Iceberg::f_properties)) + properties->set(key, value); + } + request_body->set("properties", properties); try @@ -2802,6 +2881,7 @@ bool RestCatalog::updateSchema( Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); @@ -2812,7 +2892,7 @@ bool RestCatalog::updateSchema( const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( - namespace_name, table_name, metadata, new_schema, previous_schema_id, new_last_column_id); + namespace_name, table_name, metadata, new_schema, previous_schema_id, new_last_column_id, previous_properties); try { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index f088e0c6f658f..5f1ea6f38c637 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -129,6 +129,7 @@ class RestCatalog : public ICatalog, public DB::WithContext Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool isTransactional() const override { return true; } @@ -600,14 +601,16 @@ class HorizonCatalog : public RestCatalog /// Builds the JSON body for a schema-update commit via the Iceberg REST catalog. /// Includes an assert-current-schema-id requirement (when previous_schema_id >= 0), /// schema deduplication against existing schemas in metadata, and last-column-id -/// propagation when adding a new schema. +/// propagation when adding a new schema, and property changes since `previous_properties`. +/// No schema update is sent when `new_schema` is the schema `previous_schema_id`. Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( const String & namespace_name, const String & table_name, Poco::JSON::Object::Ptr metadata, Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, - Int32 new_last_column_id); + Int32 new_last_column_id, + Poco::JSON::Object::Ptr previous_properties = nullptr); Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( const String & namespace_name, diff --git a/src/Databases/DataLake/UnityV2Catalog.cpp b/src/Databases/DataLake/UnityV2Catalog.cpp index 14a497d1d3735..16a630fece521 100644 --- a/src/Databases/DataLake/UnityV2Catalog.cpp +++ b/src/Databases/DataLake/UnityV2Catalog.cpp @@ -447,12 +447,14 @@ bool UnityV2Catalog::updateSchema( Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const { return requestWithRetry([&](bool force_refresh) { return getIcebergRestCatalog(force_refresh)->updateSchema( - namespace_name, table_name, new_metadata_path, new_schema, previous_schema_id, new_last_column_id, metadata, auth_token); + namespace_name, table_name, new_metadata_path, new_schema, previous_schema_id, new_last_column_id, metadata, + previous_properties, auth_token); }); } diff --git a/src/Databases/DataLake/UnityV2Catalog.h b/src/Databases/DataLake/UnityV2Catalog.h index d40a90f125c22..27ae8c1d500e0 100644 --- a/src/Databases/DataLake/UnityV2Catalog.h +++ b/src/Databases/DataLake/UnityV2Catalog.h @@ -99,6 +99,7 @@ class UnityV2Catalog final : public ICatalog, private DB::WithContext Int32 previous_schema_id, Int32 new_last_column_id, Poco::JSON::Object::Ptr metadata, + Poco::JSON::Object::Ptr previous_properties, const DB::ForwardedAuthTokenPtr & auth_token) const override; private: diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp index 8b209262cf628..57372b140eba4 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp @@ -228,6 +228,28 @@ TEST(RestCatalogUpdateSchemaBody, NonEmptyIdentifierFieldIdsPreventDeduplication EXPECT_EQ(set_schema->getValue("schema-id"), -1); } +TEST(RestCatalogUpdateSchemaBody, UnchangedSchemaSendsOnlyThePropertyChanges) +{ + /// A `MODIFY COLUMN` that changes only the ClickHouse type annotation keeps the current schema. + auto schema = makeSingleColumnSchema(0, std::nullopt); + auto metadata = makeMetadataWithSchemas({schema}); + Poco::JSON::Object::Ptr properties = new Poco::JSON::Object; + properties->set("clickhouse.type.1.0", "SimpleAggregateFunction(max, Int32)"); + metadata->set(Iceberg::f_properties, properties); + Poco::JSON::Object::Ptr previous_properties = new Poco::JSON::Object; + previous_properties->set("clickhouse.type.1.0", "SimpleAggregateFunction(sum, Int32)"); + + auto body = DataLake::buildUpdateSchemaRequestBody("ns", "t", metadata, schema, 0, 1, previous_properties); + ASSERT_TRUE(body); + + auto updates = body->getArray("updates"); + ASSERT_EQ(updates->size(), 1u); + auto set_properties = findUpdateByAction(updates, "set-properties"); + ASSERT_TRUE(set_properties); + EXPECT_EQ( + set_properties->getObject("updates")->getValue("clickhouse.type.1.0"), "SimpleAggregateFunction(max, Int32)"); +} + TEST(RestCatalogUpdateSchemaBody, NormalPathEmitsAddSchema) { Poco::JSON::Object::Ptr metadata = new Poco::JSON::Object; diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index 192e8b3c793fc..be77b96c5c33e 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -207,7 +207,7 @@ DataTypePtr SchemaConverter::resolveAnnotatedType(const String & column_name, co type_name, column_name, clickhouse_column_types_key, e.message()); } - if (!annotatedTypeMatchesDerived(annotated_type, derived_type, /*strict=*/ true)) + if (!annotatedTypeMatchesDerived(annotated_type, derived_type)) throw Exception( ErrorCodes::INCORRECT_DATA, "Parquet file records ClickHouse type {} for column {} in its `{}` metadata, but the " diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 9be9f8b6c5b6f..23e7a4517d5c5 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -560,6 +560,41 @@ On write, top-level columns of type `Point`, `MultiPoint`, `LineString`, `Polygo Geometry columns must appear at the root of the schema or nested inside `Tuple` (`struct`); nesting them inside `Array` or `Map` is not supported. `Nullable` is not supported for geo columns either. +## Aggregate function states {#aggregate-function-states} + +Parquet has no aggregate-state type, so ClickHouse writes an [`AggregateFunction`](/reference/data-types/aggregatefunction) column as a plain `BYTE_ARRAY` holding the serialized state - the same bytes the `-State` combinator produces - with no `STRING`/`UTF8` logical type, since a state is arbitrary binary data rather than text. A [`SimpleAggregateFunction(f, T)`](/reference/data-types/simpleaggregatefunction) column is written as an ordinary value of type `T`. + +Neither type can be recovered from the Parquet schema alone: every `AggregateFunction` state is just a binary column, and a `SimpleAggregateFunction` is indistinguishable from its storage type. ClickHouse therefore records the type names in a `clickhouse.column_types` key in the file-level Parquet metadata, as a JSON object mapping column name to ClickHouse type name. The recorded name includes the state version, which is what pins the serialized layout across server versions. + +Both writing an `AggregateFunction` column and reconstructing one from that metadata on read are gated by [`allow_experimental_aggregate_function_states_in_open_formats`](/reference/settings/session-settings/allow-experimental#allow_experimental_aggregate_function_states_in_open_formats), which is disabled by default. A state is an opaque blob passed to the deserializer of whichever aggregate function the file names, so with the setting enabled it is the file, not the query, choosing that deserializer; keep it disabled for files from untrusted sources, where such a file is then rejected rather than read as `String`. With it disabled, writing is refused with `UNKNOWN_TYPE`, exactly as in versions that did not support states in Parquet at all. `SimpleAggregateFunction` is gated in neither direction: it is stored as an ordinary value of its storage type and has always been written that way. + +With the setting enabled the states round-trip without any hint: + +```sql +SET allow_experimental_aggregate_function_states_in_open_formats = 1; + +INSERT INTO FUNCTION file('states.parquet') +SELECT k, uniqState(v) AS u, sumSimpleState(v) AS s FROM source GROUP BY k; + +DESCRIBE file('states.parquet'); +-- k UInt64 +-- u AggregateFunction(uniq, UInt64) +-- s SimpleAggregateFunction(sum, UInt64) + +SELECT k, uniqMerge(u), sum(s) FROM file('states.parquet') GROUP BY k; +``` + +An explicit structure overrides the recorded metadata and is unaffected by the setting, so a state column can also be read as `String` to get the raw serialized bytes: + +```sql +SELECT uniqMerge(CAST(u AS AggregateFunction(uniq, UInt64))) +FROM file('states.parquet', Parquet, 'u String'); +``` + +Reading fails rather than guessing if the recorded type does not describe the data actually in the file. Min/max statistics are never used for pruning a state column, because bounds over serialized states are meaningless. + +Other query engines see a plain binary (or `T`-typed) column and ignore the metadata key. The same mechanism backs [aggregate-state support in Iceberg tables](/reference/engines/table-engines/integrations/iceberg#aggregate-function-states), whose data files are Parquet. + ## Example usage {#example-usage} ### Inserting data {#inserting-data} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index c28414b47fc0f..1c99fc0ed8033 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -264,6 +264,9 @@ static Plan getPlan( plan.initial_metadata_object = initial_metadata_object; plan.initial_metadata_path = metadata_file_path; + /// Register the metadata.json schemas before the manifest headers carry copies of them without the type annotations. + addIcebergTableSchemasFromMetadata(initial_metadata_object, *persistent_table_components.schema_processor, context); + std::vector all_positional_delete_files; std::unordered_map> manifest_files; std::unordered_set referenced_file_paths; @@ -557,6 +560,10 @@ static bool writeConsolidatedManifestFile( "Iceberg metadata does not contain a schema entry matching current-schema-id {}", current_schema_id); + /// Register the metadata.json schemas before the manifest headers carry copies of them without the type annotations, + /// and so that the partition spec resolution below can query any of them by id. + addIcebergTableSchemasFromMetadata(metadata_object, *persistent_table_components.schema_processor, context); + auto partitions_specs = metadata_object->getArray(f_partition_specs); /// After partition evolution each manifest must be rewritten under the spec its source files used; resolve and cache spec info per spec-id. @@ -602,10 +609,7 @@ static bool writeConsolidatedManifestFile( source_ids.push_back(spec_field->getValue(Iceberg::f_source_id)); } - /// Derive partition value types from a schema that defines every source column the spec references, preferring the current schema then any historical one; register all schemas first so they can be queried by id. - for (UInt32 i = 0; i < schemas->size(); ++i) - persistent_table_components.schema_processor->addIcebergTableSchema(schemas->getObject(i), context); - + /// Derive partition value types from a schema that defines every source column the spec references, preferring the current schema then any historical one. auto build_sample_block = [&](Int32 schema_id) -> std::optional { auto fields_characteristics diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 4d1418fd30ad0..fcb45de106318 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -908,6 +908,9 @@ ExpireSnapshotsResult expireSnapshots( Int32 current_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + /// Register the metadata.json schemas before the manifest headers carry copies of them without the type annotations. + addIcebergTableSchemasFromMetadata(metadata, *persistent_table_components.schema_processor, context); + std::set retained_manifest_paths; std::set retained_data_file_paths; std::set retained_manifest_list_paths; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index bbe1c80755ca5..dc1fabe608104 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -409,7 +409,8 @@ Int32 IcebergMetadata::parseTableSchema( auto add_schema = [&](const Poco::JSON::Object::Ptr & schema) { if (schema_parsing == SchemaParsing::Parse) - schema_processor.addIcebergTableSchema(schema, context_); + schema_processor.addIcebergTableSchema( + schema, context_, getClickHouseTypeAnnotations(metadata_object, schema->getValue(f_schema_id))); }; /// v2 and v3 (and later) store schemas in the `schemas` array. Only v1 has a top-level `schema`. @@ -475,7 +476,8 @@ static Poco::JSON::Object::Ptr traverseMetadataAndFindNecessarySnapshotObject( for (UInt32 j = 0; j < schemas->size(); ++j) { auto schema = schemas->getObject(j); - schema_processor->addIcebergTableSchema(schema, local_context); + schema_processor->addIcebergTableSchema( + schema, local_context, getClickHouseTypeAnnotations(metadata_object, schema->getValue(f_schema_id))); } } Poco::JSON::Object::Ptr current_snapshot = nullptr; @@ -1019,7 +1021,7 @@ static void checkAggregateFunctionStatesAllowed(const String & column_name, cons throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, "Column '{}' has type {}, which Iceberg cannot express natively: the aggregate state is stored as " - "binary and its ClickHouse type is recorded in the schema. To allow this, enable setting " + "binary and its ClickHouse type is recorded in the table properties. To allow this, enable setting " "allow_experimental_aggregate_function_states_in_open_formats", column_name, type->getName()); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 7cc34022e32d0..9f86882782565 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -308,8 +308,9 @@ std::shared_ptr ManifestFileIterator::create( f_schema); manifest_schema_id = embedded_schema_id; + /// The table metadata properties are not available here, so the header schema gets no ClickHouse type annotations. schema_processor.addIcebergTableSchema( - schema_object, context_, IcebergSchemaProcessor::SchemaSource::ManifestFile, tolerate_conflicting_manifest_schemas); + schema_object, context_, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, tolerate_conflicting_manifest_schemas); } PartitionSpecification partition_spec_vec; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index ff137bfd4240a..76d3eba2e071d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -13,7 +13,6 @@ #include #include -#include #include #include @@ -162,99 +161,6 @@ bool checkValidSchemaEvolution(Poco::Dynamic::Var old_type, Poco::Dynamic::Var n return false; } - -String removeASCIIWhitespace(const String & s) -{ - String result; - result.reserve(s.size()); - for (char c : s) - { - if (!isWhitespaceASCII(c)) - result.push_back(c); - } - return result; -} - -/// A missing key matches only a missing key. -bool optionalBoolsEqual(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second, const char * key) -{ - const bool first_has = first->has(key); - if (first_has != second->has(key)) - return false; - return !first_has || first->getValue(key) == second->getValue(key); -} - -/// Whether two Iceberg type descriptors denote the same type. -/// Only the properties that define the type are compared. Field ids are ignored because -/// `getIcebergType` allocates them from a running counter, so a previous attempt may have written -/// the very same type with lower ids. Other keys (`doc`, defaults, the `required` that -/// `getIcebergType` puts on a list) are ignored because other writers emit or omit them freely. -bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type) -{ - if (old_type.isString() || new_type.isString()) - { - if (!old_type.isString() || !new_type.isString()) - return false; - /// Writers differ in spacing, e.g. `decimal(10,2)` and `decimal(10, 2)`. - return removeASCIIWhitespace(old_type.extract()) == removeASCIIWhitespace(new_type.extract()); - } - - if (old_type.type() != typeid(Poco::JSON::Object::Ptr) || new_type.type() != typeid(Poco::JSON::Object::Ptr)) - return false; - - auto old_object = old_type.extract(); - auto new_object = new_type.extract(); - if (!old_object || !new_object) - return false; - - if (old_object->has("precision") || new_object->has("precision")) - { - return old_object->has("precision") && new_object->has("precision") - && old_object->getValue("precision") == new_object->getValue("precision") - && old_object->getValue("scale") == new_object->getValue("scale"); - } - - if (!old_object->has(Iceberg::f_type) || !new_object->has(Iceberg::f_type)) - return false; - const auto kind = old_object->getValue(Iceberg::f_type); - if (kind != new_object->getValue(Iceberg::f_type)) - return false; - - if (kind == Iceberg::f_list) - { - return optionalBoolsEqual(old_object, new_object, Iceberg::f_element_required) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_element), new_object->get(Iceberg::f_element)); - } - - if (kind == Iceberg::f_map) - { - return optionalBoolsEqual(old_object, new_object, Iceberg::f_value_required) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_key), new_object->get(Iceberg::f_key)) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_value), new_object->get(Iceberg::f_value)); - } - - if (kind == Iceberg::f_struct) - { - auto old_fields = old_object->getArray(Iceberg::f_fields); - auto new_fields = new_object->getArray(Iceberg::f_fields); - if (!old_fields || !new_fields || old_fields->size() != new_fields->size()) - return false; - for (UInt32 i = 0; i < old_fields->size(); ++i) - { - auto old_field = old_fields->getObject(i); - auto new_field = new_fields->getObject(i); - if (!old_field || !new_field - || old_field->getValue(Iceberg::f_name) != new_field->getValue(Iceberg::f_name) - || !optionalBoolsEqual(old_field, new_field, Iceberg::f_required) - || !icebergTypesEqualIgnoringIds(old_field->get(Iceberg::f_type), new_field->get(Iceberg::f_type))) - return false; - } - return true; - } - - return false; -} - /// The Iceberg spec marks `snapshots`, `metadata-log` and `snapshot-log` as optional, so table /// metadata written by another engine may omit any of them - typically for a table that has never /// been written to. Return the array, creating an empty one when the field is absent, so that @@ -281,6 +187,29 @@ Int32 getNextSchemaId(Poco::JSON::Object::Ptr metadata_object) return max_id + 1; } +/// Appends `schema` as the current schema `schema_id`. Its fields keep the ClickHouse type annotations they had +/// in the previous current schema, which need not be the one with the highest id. +void addCurrentSchema(Poco::JSON::Object::Ptr metadata_object, Poco::JSON::Object::Ptr schema, Int32 schema_id) +{ + const auto previous_annotations + = Iceberg::getClickHouseTypeAnnotations(metadata_object, metadata_object->getValue(Iceberg::f_current_schema_id)); + schema->set(Iceberg::f_schema_id, schema_id); + metadata_object->set(Iceberg::f_current_schema_id, schema_id); + metadata_object->getArray(Iceberg::f_schemas)->add(schema); + for (const auto & [name, field_id] : Iceberg::IcebergSchemaProcessor::traverseSchema(schema->getArray(Iceberg::f_fields))) + { + const auto it = previous_annotations.find(static_cast(field_id)); + Iceberg::setClickHouseTypeAnnotation( + metadata_object, static_cast(field_id), schema_id, it == previous_annotations.end() ? String{} : it->second); + } +} + +/// The ClickHouse type annotation recorded for `type`, empty if it needs none. +String getClickHouseTypeAnnotation(const DataTypePtr & type) +{ + return needsClickHouseTypeAnnotation(type) ? getClickHouseTypeAnnotationName(type) : String{}; +} + /// Whether the field at `index` of `fields` satisfies the `FIRST` / `AFTER after_column` clause of an ALTER. /// `AFTER` the column itself does not move it, so any position satisfies it. bool isFieldAtRequestedPosition( @@ -347,12 +276,14 @@ Poco::JSON::Object::Ptr MetadataGenerator::getCurrentSchema() const return current_schema; } -static bool clickHouseTypeAnnotationMatches(const Poco::JSON::Object::Ptr & field, const DataTypePtr & type) +/// Whether the annotation of `field` in the current schema is the one recorded for `type`. +static bool clickHouseTypeAnnotationMatches( + const Poco::JSON::Object::Ptr & metadata_object, const Poco::JSON::Object::Ptr & field, const DataTypePtr & type) { - const bool expected = needsClickHouseTypeAnnotation(type); - if (expected != field->has(Iceberg::f_clickhouse_type)) - return false; - return !expected || field->getValue(Iceberg::f_clickhouse_type) == getClickHouseTypeAnnotationName(type); + const auto annotations + = Iceberg::getClickHouseTypeAnnotations(metadata_object, metadata_object->getValue(Iceberg::f_current_schema_id)); + const auto it = annotations.find(field->getValue(Iceberg::f_id)); + return (it == annotations.end() ? String{} : it->second) == getClickHouseTypeAnnotation(type); } bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypePtr type, bool first, const String & after_column) const @@ -374,8 +305,8 @@ bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypeP /// The stored descriptor was produced from a lower `last-column-id` than the one we /// just used, so the ids of nested elements differ even for the very same type. return field->getValue(Iceberg::f_required) == expected_type.second - && icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) - && clickHouseTypeAnnotationMatches(field, type) + && Iceberg::icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) + && clickHouseTypeAnnotationMatches(metadata_object, field, type) && isFieldAtRequestedPosition(fields, i, column_name, first, after_column); } return false; @@ -435,8 +366,8 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy auto format_version = metadata_object->getValue(Iceberg::f_format_version); auto expected_type = Iceberg::getIcebergType(type, unused_field_id, format_version); if (field->getValue(Iceberg::f_required) != expected_type.second - || !icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) - || !clickHouseTypeAnnotationMatches(field, type)) + || !Iceberg::icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first) + || !clickHouseTypeAnnotationMatches(metadata_object, field, type)) return false; } @@ -832,9 +763,7 @@ void MetadataGenerator::generateDropColumnMetadata(const String & column_name) } current_schema->getArray(Iceberg::f_fields)->remove(index_to_drop); - current_schema->set(Iceberg::f_schema_id, next_schema_id); - metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); - metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); + addCurrentSchema(metadata_object, current_schema, next_schema_id); } void MetadataGenerator::generateAddColumnMetadata(const String & column_name, DataTypePtr type, bool first, const String & after_column) @@ -868,8 +797,6 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da new_field->set(Iceberg::f_name, column_name); new_field->set(Iceberg::f_required, new_type.second); new_field->set(Iceberg::f_type, new_type.first); - if (needsClickHouseTypeAnnotation(type)) - new_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); if (first || !after_column.empty()) { @@ -902,9 +829,8 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da existing_fields->add(new_field); } - current_schema->set(Iceberg::f_schema_id, next_schema_id); - metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); - metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); + addCurrentSchema(metadata_object, current_schema, next_schema_id); + Iceberg::setClickHouseTypeAnnotation(metadata_object, new_field_id, next_schema_id, getClickHouseTypeAnnotation(type)); } bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, DataTypePtr type, ContextPtr context, bool first, const String & after_column) @@ -920,6 +846,8 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, /// `AFTER` the column itself keeps its position, as in `ColumnsDescription::modifyColumnOrder`. bool needs_reposition = first || (!after_column.empty() && after_column != column_name); bool type_changed = false; + /// The Iceberg type stays, only the ClickHouse type annotation changes. + bool annotation_changed = false; if (type) { @@ -932,19 +860,20 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, continue; if (current_field->getValue(Iceberg::f_required) == new_type.second - && icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) + && Iceberg::icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) { DataTypePtr current_ch_type; - if (current_field->has(Iceberg::f_clickhouse_type)) + const auto annotations = Iceberg::getClickHouseTypeAnnotations( + metadata_object, metadata_object->getValue(Iceberg::f_current_schema_id)); + if (auto annotation = annotations.find(current_field->getValue(Iceberg::f_id)); annotation != annotations.end()) { - current_ch_type - = DataTypeFactory::instance().get(current_field->getValue(Iceberg::f_clickhouse_type)); + current_ch_type = DataTypeFactory::instance().get(annotation->second); } else { Iceberg::IcebergSchemaProcessor schema_processor( context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); - current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); + current_ch_type = schema_processor.getClickHouseFieldType(current_field, annotations, context); } if (!current_ch_type->equals(*type)) @@ -958,8 +887,8 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, current_field->get(Iceberg::f_type).toString()); /// `IDataType::equals` ignores the custom name of `SimpleAggregateFunction`. - if (!clickHouseTypeAnnotationMatches(current_field, type)) - type_changed = true; + if (!clickHouseTypeAnnotationMatches(metadata_object, current_field, type)) + annotation_changed = true; else if (!needs_reposition) return false; } @@ -989,8 +918,20 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (target_index == schema_fields->size()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} not found in schema", column_name); + const auto target_field_id = schema_fields->getObject(target_index)->getValue(Iceberg::f_id); + if (!type_changed && !needs_reposition) - return false; + { + if (!annotation_changed) + return false; + /// The schema stays the same, so the annotation is recorded for its id, which a REST catalog keeps as well. + Iceberg::setClickHouseTypeAnnotation( + metadata_object, + target_field_id, + metadata_object->getValue(Iceberg::f_current_schema_id), + getClickHouseTypeAnnotation(type)); + return true; + } const auto next_schema_id = getNextSchemaId(metadata_object); current_schema = deepCopy(current_schema); @@ -1002,10 +943,6 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, auto new_type = Iceberg::getIcebergType(type, last_column_id, format_version); target_field->set(Iceberg::f_type, new_type.first); target_field->set(Iceberg::f_required, new_type.second); - if (needsClickHouseTypeAnnotation(type)) - target_field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(type)); - else - target_field->remove(Iceberg::f_clickhouse_type); } if (needs_reposition) @@ -1040,9 +977,9 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, current_schema->set(Iceberg::f_fields, new_fields); } - metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); - current_schema->set(Iceberg::f_schema_id, next_schema_id); - metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); + addCurrentSchema(metadata_object, current_schema, next_schema_id); + if (type_changed || annotation_changed) + Iceberg::setClickHouseTypeAnnotation(metadata_object, target_field_id, next_schema_id, getClickHouseTypeAnnotation(type)); return true; } @@ -1073,10 +1010,7 @@ void MetadataGenerator::generateRenameColumnMetadata(const String & column_name, if (!found) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found column {}", column_name); - const auto next_schema_id = getNextSchemaId(metadata_object); - metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); - current_schema->set(Iceberg::f_schema_id, next_schema_id); - metadata_object->getArray(Iceberg::f_schemas)->add(current_schema); + addCurrentSchema(metadata_object, current_schema, getNextSchemaId(metadata_object)); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index e6d8a388ac6e8..8ae2ac1c7c9e0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -831,6 +831,9 @@ void mutate( } } + /// Register the metadata.json schemas before the manifest headers carry copies of them without the type annotations. + addIcebergTableSchemasFromMetadata(metadata, *persistent_table_components.schema_processor, context); + /// Validate against the freshly-fetched snapshot. Bound to this exact /// metadata version, so a concurrent writer that commits non-Parquet /// data files between iterations is caught by the next retry. @@ -939,6 +942,9 @@ void alter( auto metadata = getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); const auto previous_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + Poco::JSON::Object::Ptr previous_properties = metadata->has(Iceberg::f_properties) + ? new Poco::JSON::Object(*metadata->getObject(Iceberg::f_properties)) + : new Poco::JSON::Object; auto metadata_json_generator = MetadataGenerator(metadata); @@ -1026,7 +1032,9 @@ void alter( metadata->getValue(Iceberg::f_last_column_id), getHighestFieldId(new_schema)); commit_attempted = true; - if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, context->getForwardedAuthToken())) + if (!catalog->updateSchema( + namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, + previous_properties, context->getForwardedAuthToken())) { ++i; continue; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/RemoveOrphanFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/RemoveOrphanFilesExecute.cpp index 715a7e98bcf36..4a9dbf053f29c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/RemoveOrphanFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/RemoveOrphanFilesExecute.cpp @@ -385,6 +385,9 @@ Pipe executeRemoveOrphanFiles( params.location = parsed.getAs("location"); params.dry_run = parsed.getAs("dry_run") != 0; + /// Register the metadata.json schemas before the manifest headers carry copies of them without the type annotations. + addIcebergTableSchemasFromMetadata(latest_metadata, *persistent_components.schema_processor, context); + auto result = removeOrphanFiles(params, context, object_storage, data_lake_settings, persistent_components, external_storages); return resultToPipe(result); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index ab7e2e1b5bd01..5680ad7d7092e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -46,6 +45,7 @@ #include #include +#include namespace DB { @@ -409,13 +409,18 @@ void IcebergSchemaProcessor::dropCachedSchema(Int32 schema_id) { iceberg_table_schemas_by_ids.erase(schema_id); clickhouse_table_schemas_by_ids.erase(schema_id); + clickhouse_type_annotations_by_schema_ids.erase(schema_id); std::erase_if(transform_dags_by_ids, [schema_id](const auto & item) { return item.first.first == schema_id || item.first.second == schema_id; }); std::erase_if(clickhouse_types_by_source_ids, [schema_id](const auto & item) { return item.first.first == schema_id; }); std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & item) { return item.first.first == schema_id; }); } void IcebergSchemaProcessor::addIcebergTableSchema( - Poco::JSON::Object::Ptr schema_ptr, ContextPtr context_, SchemaSource source, bool tolerate_conflicting_manifest_schemas) + Poco::JSON::Object::Ptr schema_ptr, + ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, + SchemaSource source, + bool tolerate_conflicting_manifest_schemas) { Int32 schema_id = schema_ptr->getValue(f_schema_id); @@ -424,6 +429,8 @@ void IcebergSchemaProcessor::addIcebergTableSchema( if (!schema_ptr->isArray(f_fields) || schema_ptr->getArray(f_fields)->size() == 0) return; + auto schema_annotations = getClickHouseTypeAnnotationsOfSchema(schema_ptr, clickhouse_type_annotations); + std::unordered_map type_mapping; if (allow_geo_parser) { @@ -440,7 +447,8 @@ void IcebergSchemaProcessor::addIcebergTableSchema( const bool registered_from_manifest = manifest_sourced_schema_ids.contains(schema_id); if (source == SchemaSource::ManifestFile && tolerate_conflicting_manifest_schemas && !registered_from_manifest) return; - if (source == SchemaSource::ManifestFile || !registered_from_manifest) + if (source == SchemaSource::ManifestFile + || (!registered_from_manifest && clickHouseTypeAnnotationsUnchanged(schema_id, schema_annotations))) registered_schema = it->second; } } @@ -449,7 +457,12 @@ void IcebergSchemaProcessor::addIcebergTableSchema( std::lock_guard lock(mutex); - if (iceberg_table_schemas_by_ids.contains(schema_id)) + /// Table properties changed the ClickHouse types of an already registered schema: it is replaced once the new copy parsed. + const bool replace_registered_schema = source == SchemaSource::Metadata && iceberg_table_schemas_by_ids.contains(schema_id) + && schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping) + && !clickHouseTypeAnnotationsUnchanged(schema_id, schema_annotations); + + if (!replace_registered_schema && iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); if (schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) @@ -522,18 +535,13 @@ void IcebergSchemaProcessor::addIcebergTableSchema( } current_schema_id = schema_id; - bool parsed = false; - const auto rollback_unless_parsed = make_scope_guard( + const auto reset_parse_state = make_scope_guard( [&]() TSA_NO_THREAD_SAFETY_ANALYSIS { /// TSA cannot see the lock held by the caller through this lambda. current_schema_id = std::nullopt; - if (parsed) - return; - iceberg_table_schemas_by_ids.erase(schema_id); - clickhouse_table_schemas_by_ids.erase(schema_id); - std::erase_if(clickhouse_types_by_source_ids, [schema_id](const auto & entry) { return entry.first.first == schema_id; }); - std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & entry) { return entry.first.first == schema_id; }); + parsed_clickhouse_types_by_source_ids.clear(); + parsed_clickhouse_ids_by_source_names.clear(); }); auto clickhouse_schema = std::make_shared(); @@ -544,16 +552,26 @@ void IcebergSchemaProcessor::addIcebergTableSchema( auto name = field->getValue(f_name); bool required = field->getValue(f_required); current_full_name = name; - auto type = getFieldType(field, f_type, context_, required, current_full_name, true); + auto type = getFieldType(field, f_type, context_, schema_annotations, required, current_full_name, true); clickhouse_schema->push_back(NameAndTypePair{name, type}); - clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; - clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + parsed_clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; + parsed_clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + } + + if (replace_registered_schema) + { + dropCachedSchema(schema_id); + manifest_sourced_schema_ids.erase(schema_id); } + for (auto & [key, value] : parsed_clickhouse_types_by_source_ids) + clickhouse_types_by_source_ids[key] = std::move(value); + for (auto & [key, value] : parsed_clickhouse_ids_by_source_names) + clickhouse_ids_by_source_names[key] = value; clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + clickhouse_type_annotations_by_schema_ids[schema_id] = std::move(schema_annotations); if (source == SchemaSource::ManifestFile) manifest_sourced_schema_ids.insert(schema_id); - parsed = true; } NameAndTypePair IcebergSchemaProcessor::getFieldCharacteristics(Int32 schema_version, Int32 source_id) const @@ -677,6 +695,7 @@ IcebergSchemaProcessor::getComplexTypeFromObject( const Poco::JSON::Object::Ptr & type, String & current_full_name, ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, bool is_subfield_of_root) { /// The schema comes from the table metadata and can be nested arbitrarily deeply. @@ -686,15 +705,15 @@ IcebergSchemaProcessor::getComplexTypeFromObject( if (type_name == f_list) { bool element_required = type->getValue("element-required"); - auto element_type = getFieldType(type, f_element, context_, element_required); + auto element_type = getFieldType(type, f_element, context_, clickhouse_type_annotations, element_required); return std::make_shared(element_type); } if (type_name == f_map) { - auto key_type = getFieldType(type, f_key, context_, true); + auto key_type = getFieldType(type, f_key, context_, clickhouse_type_annotations, true); auto value_required = type->getValue("value-required"); - auto value_type = getFieldType(type, f_value, context_, value_required); + auto value_type = getFieldType(type, f_value, context_, clickhouse_type_annotations, value_required); return std::make_shared(key_type, value_type); } @@ -718,16 +737,16 @@ IcebergSchemaProcessor::getComplexTypeFromObject( (current_full_name += ".").append(element_names.back()); scope_guard guard([&] { current_full_name.resize(current_full_name.size() - element_names.back().size() - 1); }); - element_types.push_back(getFieldType(field, f_type, context_, required, current_full_name, true)); - TSA_SUPPRESS_WARNING_FOR_WRITE(clickhouse_types_by_source_ids) + element_types.push_back(getFieldType(field, f_type, context_, clickhouse_type_annotations, required, current_full_name, true)); + TSA_SUPPRESS_WARNING_FOR_WRITE(parsed_clickhouse_types_by_source_ids) [{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, element_types.back()}; - TSA_SUPPRESS_WARNING_FOR_WRITE(clickhouse_ids_by_source_names) + TSA_SUPPRESS_WARNING_FOR_WRITE(parsed_clickhouse_ids_by_source_names) [{schema_id, current_full_name}] = field->getValue(f_id); } else { - element_types.push_back(getFieldType(field, f_type, context_, required)); + element_types.push_back(getFieldType(field, f_type, context_, clickhouse_type_annotations, required)); } } @@ -741,13 +760,15 @@ DataTypePtr IcebergSchemaProcessor::getFieldType( const Poco::JSON::Object::Ptr & field, const String & type_key, ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, bool required, String & current_full_name, bool is_subfield_of_root) { DataTypePtr derived_type; if (field->isObject(type_key)) - derived_type = getComplexTypeFromObject(field->getObject(type_key), current_full_name, context_, is_subfield_of_root); + derived_type = getComplexTypeFromObject( + field->getObject(type_key), current_full_name, context_, clickhouse_type_annotations, is_subfield_of_root); else { auto type = field->get(type_key); @@ -759,10 +780,15 @@ DataTypePtr IcebergSchemaProcessor::getFieldType( derived_type = makeNullable(derived_type); } - if (!field->has(f_clickhouse_type)) + if (!field->has(f_id)) return derived_type; - const String annotated_name = field->getValue(f_clickhouse_type); + const Int32 field_id = field->getValue(f_id); + auto annotation_it = clickhouse_type_annotations.find(field_id); + if (annotation_it == clickhouse_type_annotations.end()) + return derived_type; + + const String & annotated_name = annotation_it->second; auto annotated_type = DataTypeFactory::instance().get(annotated_name); /// Read the gate from the query context rather than the context captured at `ATTACH`. @@ -770,28 +796,36 @@ DataTypePtr IcebergSchemaProcessor::getFieldType( && !context_->getSettingsRef()[Setting::allow_experimental_aggregate_function_states_in_open_formats]) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, - "Iceberg field '{}' records ClickHouse type {} in its `{}` key. Reading aggregate function " + "Iceberg field '{}' records ClickHouse type {} in table properties `{}.{}.`. Reading aggregate function " "states from an Iceberg table is disabled: enable setting " "allow_experimental_aggregate_function_states_in_open_formats to honour the recorded type", field->has(f_name) ? field->getValue(f_name) : type_key, annotated_name, - f_clickhouse_type); - - if (!annotatedTypeMatchesDerived(annotated_type, derived_type)) + f_clickhouse_type, + field_id); + + /// The annotation is valid only for the Iceberg type ClickHouse writes for it. The format version affects only + /// whether `timestamp_ns` may be written, so the latest one is used and the field's own type decides. + Int32 unused_field_id = 0; + const auto [annotated_iceberg_type, annotated_required] + = getIcebergType(annotated_type, unused_field_id, /*format_version=*/ 3); + if (annotated_required != required || !icebergTypesEqualIgnoringIds(field->get(type_key), annotated_iceberg_type)) throw Exception( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Iceberg field '{}' records ClickHouse type {} in its `{}` key, but the field itself reads as {}", + "Iceberg field '{}' records ClickHouse type {} in table properties `{}.{}.`, but the field itself reads as {}", field->has(f_name) ? field->getValue(f_name) : type_key, annotated_name, f_clickhouse_type, + field_id, derived_type->getName()); return annotated_type; } -DataTypePtr IcebergSchemaProcessor::getClickHouseFieldType(const Poco::JSON::Object::Ptr & field, ContextPtr context_) +DataTypePtr IcebergSchemaProcessor::getClickHouseFieldType( + const Poco::JSON::Object::Ptr & field, const ClickHouseTypeAnnotations & clickhouse_type_annotations, ContextPtr context_) { - return getFieldType(field, f_type, context_, field->getValue(f_required)); + return getFieldType(field, f_type, context_, clickhouse_type_annotations, field->getValue(f_required)); } /** @@ -824,6 +858,8 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema, + const ClickHouseTypeAnnotations & old_annotations, + const ClickHouseTypeAnnotations & new_annotations, ContextPtr context_, Int32 old_id, Int32 new_id) @@ -838,7 +874,7 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( size_t id = field->getValue(f_id); auto name = field->getValue(f_name); bool required = field->getValue(f_required); - old_schema_entries[id] = {field, &dag->addInput(name, getFieldType(field, f_type, context_, required))}; + old_schema_entries[id] = {field, &dag->addInput(name, getFieldType(field, f_type, context_, old_annotations, required))}; } auto new_schema_fields = new_schema->get(f_fields).extract(); for (size_t i = 0; i != new_schema_fields->size(); ++i) @@ -847,7 +883,7 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( size_t id = field->getValue(f_id); auto name = field->getValue(f_name); bool required = field->getValue(f_required); - auto type = getFieldType(field, f_type, context_, required); + auto type = getFieldType(field, f_type, context_, new_annotations, required); auto old_node_it = old_schema_entries.find(id); if (old_node_it != old_schema_entries.end()) { @@ -866,7 +902,7 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( old_id, new_id); } - auto old_type = getFieldType(old_json, "type", context_, required); + auto old_type = getFieldType(old_json, "type", context_, old_annotations, required); auto transform = std::make_shared(DataTypes{type}, DataTypes{old_type}, old_json, field); old_node = &dag->addFunction(transform, std::vector{old_node}, name); @@ -907,7 +943,7 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( } else if (allowPrimitiveTypeConversion(old_type, new_type)) { - node = &dag->addCast(*old_node, getFieldType(field, f_type, context_, required), name, nullptr); + node = &dag->addCast(*old_node, getFieldType(field, f_type, context_, new_annotations, required), name, nullptr); } outputs.push_back(node); } @@ -956,8 +992,14 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio if (new_schema_it == iceberg_table_schemas_by_ids.end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); - return transform_dags_by_ids[{old_id, new_id}] - = getSchemaTransformationDag(old_schema_it->second, new_schema_it->second, context_, old_id, new_id); + return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag( + old_schema_it->second, + new_schema_it->second, + clickhouse_type_annotations_by_schema_ids.at(old_id), + clickhouse_type_annotations_by_schema_ids.at(new_id), + context_, + old_id, + new_id); } Poco::JSON::Object::Ptr IcebergSchemaProcessor::getIcebergTableSchemaById(Int32 id) const @@ -1081,6 +1123,26 @@ void IcebergSchemaProcessor::updateLastColumnId(Int32 last_column_id_) ; } +ClickHouseTypeAnnotations IcebergSchemaProcessor::getClickHouseTypeAnnotationsOfSchema( + const Poco::JSON::Object::Ptr & schema, const ClickHouseTypeAnnotations & clickhouse_type_annotations) +{ + ClickHouseTypeAnnotations result; + if (clickhouse_type_annotations.empty()) + return result; + for (const auto & name_and_id : traverseSchema(schema->getArray(f_fields))) + { + if (auto it = clickhouse_type_annotations.find(static_cast(name_and_id.second)); it != clickhouse_type_annotations.end()) + result.emplace(it->first, it->second); + } + return result; +} + +bool IcebergSchemaProcessor::clickHouseTypeAnnotationsUnchanged(Int32 schema_id, const ClickHouseTypeAnnotations & schema_annotations) const +{ + auto it = clickhouse_type_annotations_by_schema_ids.find(schema_id); + return it != clickhouse_type_annotations_by_schema_ids.end() && it->second == schema_annotations; +} + ColumnMapperPtr IcebergSchemaProcessor::getColumnMapperById(Int32 id) const { auto schema = getIcebergTableSchemaById(id); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index 0745f67fe8d2c..55cf70273ea02 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -1,6 +1,7 @@ #pragma once +#include #include #include @@ -28,6 +29,9 @@ ColumnMapperPtr createColumnMapperFromFields(Poco::JSON::Array::Ptr fields); ColumnMapperPtr createColumnMapper(Poco::JSON::Object::Ptr schema_object); +/// ClickHouse type names in effect for the fields of a schema (`clickhouse.type..` table properties), by field id. +using ClickHouseTypeAnnotations = std::map; + /** * Iceberg supports the following data types (see https://iceberg.apache.org/spec/#schemas-and-data-types): * - Primitive types: @@ -102,9 +106,11 @@ class IcebergSchemaProcessor : private WithContext explicit IcebergSchemaProcessor(ContextPtr context_, bool allow_geo_parser_ = false) : WithContext(context_), allow_geo_parser(allow_geo_parser_) {} + /// `clickhouse_type_annotations` are the ones in effect for the schema in the table metadata it belongs to. void addIcebergTableSchema( Poco::JSON::Object::Ptr schema_ptr, ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, SchemaSource source = SchemaSource::Metadata, bool tolerate_conflicting_manifest_schemas = false); std::shared_ptr getClickHouseTableSchemaById(Int32 id); @@ -120,7 +126,8 @@ class IcebergSchemaProcessor : private WithContext static DataTypePtr getSimpleType(const String & type_name, ContextPtr context_, bool allow_geo_parser = true); /// The ClickHouse type the reader derives for a top-level schema field. - DataTypePtr getClickHouseFieldType(const Poco::JSON::Object::Ptr & field, ContextPtr context_); + DataTypePtr getClickHouseFieldType( + const Poco::JSON::Object::Ptr & field, const ClickHouseTypeAnnotations & clickhouse_type_annotations, ContextPtr context_); static std::unordered_map traverseSchema(Poco::JSON::Array::Ptr schema); @@ -148,28 +155,41 @@ class IcebergSchemaProcessor : private WithContext mutable std::map, NameAndTypePair> clickhouse_types_by_source_ids TSA_GUARDED_BY(mutex); mutable std::map, Int32> clickhouse_ids_by_source_names TSA_GUARDED_BY(mutex); std::optional current_schema_id TSA_GUARDED_BY(mutex) = 0; + /// Per-field lookups of the schema `addIcebergTableSchema` is parsing, installed only once the whole schema parsed. + std::map, NameAndTypePair> parsed_clickhouse_types_by_source_ids TSA_GUARDED_BY(mutex); + std::map, Int32> parsed_clickhouse_ids_by_source_names TSA_GUARDED_BY(mutex); std::unordered_map schema_id_by_snapshot TSA_GUARDED_BY(mutex); /// Schema-ids whose registered copy came from a manifest file header and has not been confirmed /// by an identical metadata.json copy yet. Such a copy is replaced when metadata.json binds the /// schema-id to a different schema, and two manifest headers disagreeing on such a schema-id is /// an error, because no authoritative copy is left to decide between them. std::unordered_set manifest_sourced_schema_ids TSA_GUARDED_BY(mutex); + /// The annotations each registered schema was parsed with. + std::unordered_map clickhouse_type_annotations_by_schema_ids TSA_GUARDED_BY(mutex); /// Forget the schema registered for `schema_id` together with everything derived from it: the /// per-field lookups and the cached schema transformation DAGs in either direction. They are /// keyed by schema-id and never rebuilt once populated, so a stale entry would keep answering. void dropCachedSchema(Int32 schema_id) TSA_REQUIRES(mutex); + /// The subset of `clickhouse_type_annotations` that refers to the fields of `schema`. + static ClickHouseTypeAnnotations getClickHouseTypeAnnotationsOfSchema( + const Poco::JSON::Object::Ptr & schema, const ClickHouseTypeAnnotations & clickhouse_type_annotations); + bool clickHouseTypeAnnotationsUnchanged(Int32 schema_id, const ClickHouseTypeAnnotations & schema_annotations) const + TSA_REQUIRES_SHARED(mutex); + NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject( const Poco::JSON::Object::Ptr & type, String & current_full_name, ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, bool is_subfield_of_root); DataTypePtr getFieldType( const Poco::JSON::Object::Ptr & field, const String & type_key, ContextPtr context_, + const ClickHouseTypeAnnotations & clickhouse_type_annotations, bool required, String & current_full_name = default_link, bool is_subfield_of_root = false); @@ -180,6 +200,8 @@ class IcebergSchemaProcessor : private WithContext std::shared_ptr getSchemaTransformationDag( const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema, + const ClickHouseTypeAnnotations & old_annotations, + const ClickHouseTypeAnnotations & new_annotations, ContextPtr context_, Int32 old_id, Int32 new_id); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 328db41edf64d..5b082d7c31c18 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -635,6 +636,81 @@ void checkUnknownTypeAllowed(const String & column_name, const DataTypePtr & typ column_name, type->getName(), format_version); } +String getClickHouseTypePropertyKey(Int32 field_id, Int32 schema_id) +{ + return fmt::format("{}.{}.{}", Iceberg::f_clickhouse_type, field_id, schema_id); +} + +ClickHouseTypeAnnotations getClickHouseTypeAnnotations(const Poco::JSON::Object::Ptr & metadata_object, Int32 schema_id) +{ + ClickHouseTypeAnnotations annotations; + if (!metadata_object->has(Iceberg::f_properties)) + return annotations; + const auto properties = metadata_object->getObject(Iceberg::f_properties); + const String prefix = String(Iceberg::f_clickhouse_type) + "."; + /// For each field, the schema id of the entry in effect and its value. + std::map> entries; + for (const auto & [key, value] : *properties) + { + if (!key.starts_with(prefix)) + continue; + /// Only keys whose suffix is `.` are annotations + const std::string_view suffix = std::string_view(key).substr(prefix.size()); + const size_t dot = suffix.find('.'); + Int32 field_id; + Int32 entry_schema_id; + if (dot == std::string_view::npos || !tryParse(field_id, suffix.substr(0, dot)) + || !tryParse(entry_schema_id, suffix.substr(dot + 1)) || entry_schema_id > schema_id) + continue; + auto [it, inserted] = entries.try_emplace(field_id, entry_schema_id, value.extract()); + if (!inserted && it->second.first < entry_schema_id) + it->second = {entry_schema_id, value.extract()}; + } + for (auto & [field_id, entry] : entries) + { + if (!entry.second.empty()) + annotations.emplace(field_id, std::move(entry.second)); + } + return annotations; +} + +void setClickHouseTypeAnnotation(Poco::JSON::Object::Ptr metadata_object, Int32 field_id, Int32 schema_id, const String & annotation) +{ + const auto key = getClickHouseTypePropertyKey(field_id, schema_id); + const auto inherited_annotations = getClickHouseTypeAnnotations(metadata_object, schema_id - 1); + const auto inherited = inherited_annotations.find(field_id); + const String & inherited_annotation = inherited == inherited_annotations.end() ? String{} : inherited->second; + if (annotation == inherited_annotation) + { + if (metadata_object->has(Iceberg::f_properties)) + metadata_object->getObject(Iceberg::f_properties)->remove(key); + return; + } + + if (!metadata_object->has(Iceberg::f_properties)) + metadata_object->set(Iceberg::f_properties, Poco::JSON::Object::Ptr(new Poco::JSON::Object)); + metadata_object->getObject(Iceberg::f_properties)->set(key, annotation); +} + +void addIcebergTableSchemasFromMetadata( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, ContextPtr context) +{ + auto add_schema = [&](const Poco::JSON::Object::Ptr & schema) + { + schema_processor.addIcebergTableSchema( + schema, context, getClickHouseTypeAnnotations(metadata_object, schema->getValue(f_schema_id))); + }; + /// `schemas` is optional in format version 1, which keeps its only schema in the top-level `schema`. + if (!metadata_object->has(f_schemas)) + { + add_schema(parseTableSchemaV1Method(metadata_object).first); + return; + } + auto schemas = metadata_object->getArray(f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + add_schema(schemas->getObject(i)); +} + /// Returns type and required std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version) { @@ -781,6 +857,102 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite } } +namespace +{ + +String removeASCIIWhitespace(const String & s) +{ + String result; + result.reserve(s.size()); + for (char c : s) + { + if (!isWhitespaceASCII(c)) + result.push_back(c); + } + return result; +} + +/// A missing key matches only a missing key. +bool optionalBoolsEqual(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second, const char * key) +{ + const bool first_has = first->has(key); + if (first_has != second->has(key)) + return false; + return !first_has || first->getValue(key) == second->getValue(key); +} + +} + +/// Only the properties that define the type are compared. Field ids are ignored because +/// `getIcebergType` allocates them from a running counter, so a previous attempt may have written +/// the very same type with lower ids. Other keys (`doc`, defaults, the `required` that +/// `getIcebergType` puts on a list) are ignored because other writers emit or omit them freely. +bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type) +{ + if (old_type.isString() || new_type.isString()) + { + if (!old_type.isString() || !new_type.isString()) + return false; + /// Writers differ in spacing, e.g. `decimal(10,2)` and `decimal(10, 2)`. + return removeASCIIWhitespace(old_type.extract()) == removeASCIIWhitespace(new_type.extract()); + } + + if (old_type.type() != typeid(Poco::JSON::Object::Ptr) || new_type.type() != typeid(Poco::JSON::Object::Ptr)) + return false; + + auto old_object = old_type.extract(); + auto new_object = new_type.extract(); + if (!old_object || !new_object) + return false; + + if (old_object->has("precision") || new_object->has("precision")) + { + return old_object->has("precision") && new_object->has("precision") + && old_object->getValue("precision") == new_object->getValue("precision") + && old_object->getValue("scale") == new_object->getValue("scale"); + } + + if (!old_object->has(Iceberg::f_type) || !new_object->has(Iceberg::f_type)) + return false; + const auto kind = old_object->getValue(Iceberg::f_type); + if (kind != new_object->getValue(Iceberg::f_type)) + return false; + + if (kind == Iceberg::f_list) + { + return optionalBoolsEqual(old_object, new_object, Iceberg::f_element_required) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_element), new_object->get(Iceberg::f_element)); + } + + if (kind == Iceberg::f_map) + { + return optionalBoolsEqual(old_object, new_object, Iceberg::f_value_required) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_key), new_object->get(Iceberg::f_key)) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_value), new_object->get(Iceberg::f_value)); + } + + if (kind == Iceberg::f_struct) + { + auto old_fields = old_object->getArray(Iceberg::f_fields); + auto new_fields = new_object->getArray(Iceberg::f_fields); + if (!old_fields || !new_fields || old_fields->size() != new_fields->size()) + return false; + for (UInt32 i = 0; i < old_fields->size(); ++i) + { + auto old_field = old_fields->getObject(i); + auto new_field = new_fields->getObject(i); + if (!old_field || !new_field + || old_field->getValue(Iceberg::f_name) != new_field->getValue(Iceberg::f_name) + || !optionalBoolsEqual(old_field, new_field, Iceberg::f_required) + || !icebergTypesEqualIgnoringIds(old_field->get(Iceberg::f_type), new_field->get(Iceberg::f_type))) + return false; + } + return true; + } + + return false; +} + Poco::Dynamic::Var getAvroType(DataTypePtr type, Int32 field_id) { switch (type->getTypeId()) @@ -1270,7 +1442,8 @@ std::pair createEmptyMetadataFile( field->set(Iceberg::f_required, type.second); field->set(Iceberg::f_type, type.first); if (needsClickHouseTypeAnnotation(column.type)) - field->set(Iceberg::f_clickhouse_type, getClickHouseTypeAnnotationName(column.type)); + setClickHouseTypeAnnotation( + new_metadata_file_content, iter_for_initial_columns, /*schema_id=*/ 0, getClickHouseTypeAnnotationName(column.type)); column_name_to_source_id[column.name] = iter_for_initial_columns; schema_fields->add(field); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index ceda88fc76e12..292a5ec953737 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -99,10 +99,28 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( /// into v1/v2 metadata. std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version); +/// Whether two Iceberg type descriptors denote the same type, ignoring the ids of nested fields. +bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type); + /// Throws if `type` contains `Nothing` at any depth (the Iceberg `unknown` type) and /// `format_version` is below 3, the first format version that defines `unknown`. void checkUnknownTypeAllowed(const String & column_name, const DataTypePtr & type, Int64 format_version); +/// Table property `clickhouse.type..` records a type Iceberg cannot express for the field, +/// effective from that schema id until the next property of the field. An empty value records no type. +String getClickHouseTypePropertyKey(Int32 field_id, Int32 schema_id); + +/// The annotations in effect for schema `schema_id`: per field, the property with the greatest schema id not above it. +ClickHouseTypeAnnotations getClickHouseTypeAnnotations(const Poco::JSON::Object::Ptr & metadata_object, Int32 schema_id); + +/// Records `annotation` (empty for none) for `field_id` from schema `schema_id` on. The property is removed +/// when the field inherits the same annotation from a lower schema id. +void setClickHouseTypeAnnotation(Poco::JSON::Object::Ptr metadata_object, Int32 field_id, Int32 schema_id, const String & annotation); + +/// Registers every schema of the table metadata with the ClickHouse type annotations in effect for it. +void addIcebergTableSchemasFromMetadata( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, ContextPtr context); + Poco::Dynamic::Var getAvroType(DataTypePtr type, Int32 field_id); Poco::Dynamic::Var getAvroLogicalType(DataTypePtr type); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp index bb18c96c12173..e34fdfe4c95ad 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp @@ -589,14 +589,25 @@ Poco::JSON::Object::Ptr makeMetadataWithAnnotatedField( const String & name, const Poco::Dynamic::Var & iceberg_type, bool required, const String & annotation) { auto metadata = makeMetadataWithField(name, iceberg_type, required); - metadata->getArray(f_schemas)->getObject(0)->getArray(f_fields)->getObject(0)->set(f_clickhouse_type, annotation); + auto field_id = metadata->getArray(f_schemas)->getObject(0)->getArray(f_fields)->getObject(0)->getValue(f_id); + Poco::JSON::Object::Ptr properties = new Poco::JSON::Object; + properties->set(Iceberg::getClickHouseTypePropertyKey(field_id, /*schema_id=*/ 0), annotation); + metadata->set(f_properties, properties); return metadata; } +std::optional findFieldAnnotation(const Poco::JSON::Object::Ptr & metadata, const String & name, Int32 schema_id) +{ + const auto annotations = Iceberg::getClickHouseTypeAnnotations(metadata, schema_id); + const auto it = annotations.find(findCurrentFieldValue(metadata, name, f_id).convert()); + if (it == annotations.end()) + return std::nullopt; + return it->second; +} + std::optional findCurrentFieldAnnotation(const Poco::JSON::Object::Ptr & metadata, const String & name) { - auto value = findCurrentFieldValue(metadata, name, f_clickhouse_type); - return value.isEmpty() ? std::nullopt : std::optional{value.extract()}; + return findFieldAnnotation(metadata, name, metadata->getValue(f_current_schema_id)); } void expectModifyRejected( @@ -1276,12 +1287,25 @@ TEST(IcebergMetadataGenerator, ModifyColumnChangingTheSimpleAggregateFunctionRew EXPECT_TRUE(MetadataGenerator(metadata).generateModifyColumnMetadata( "s", DataTypeFactory::instance().get("SimpleAggregateFunction(max, UInt64)"), getContext().context)); - const auto after = readSchemaState(metadata); - EXPECT_EQ(after.schema_count, before.schema_count + 1); - EXPECT_NE(after.current_schema_id, before.current_schema_id); + /// The Iceberg schema is the same, so the annotation is rewritten for the current schema id. + expectSchemaUnchanged(metadata, before); EXPECT_EQ(findCurrentFieldAnnotation(metadata, "s"), "SimpleAggregateFunction(max, UInt64)"); } +TEST(IcebergMetadataGenerator, ModifyColumnWideningKeepsTheAnnotationOfTheOldSchema) +{ + tryRegisterAggregateFunctions(); + auto metadata + = makeMetadataWithAnnotatedField("x", "int", /* required */ true, "SimpleAggregateFunction(max, Int32)"); + + EXPECT_TRUE(MetadataGenerator(metadata).generateModifyColumnMetadata( + "x", DataTypeFactory::instance().get("SimpleAggregateFunction(max, Int64)"), getContext().context)); + + ASSERT_EQ(metadata->getValue(f_current_schema_id), 1); + EXPECT_EQ(findFieldAnnotation(metadata, "x", 0), "SimpleAggregateFunction(max, Int32)"); + EXPECT_EQ(findFieldAnnotation(metadata, "x", 1), "SimpleAggregateFunction(max, Int64)"); +} + TEST(IcebergMetadataGenerator, ModifyColumnAddingASimpleAggregateFunctionRecordsTheAnnotation) { tryRegisterAggregateFunctions(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index 6097db3a91404..4a89737908a5e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -10,6 +10,7 @@ #include "config.h" #if USE_AVRO #include +#include #endif #include @@ -43,13 +44,13 @@ DB::ContextMutablePtr contextWithAggregateFunctionStates(bool allow) return context; } -void expectAnnotatedSchemaRejected(const Poco::JSON::Object::Ptr & schema) +void expectAnnotatedSchemaRejected(const Poco::JSON::Object::Ptr & schema, const ClickHouseTypeAnnotations & annotations) { auto context = contextWithAggregateFunctionStates(true); IcebergSchemaProcessor processor(context); try { - processor.addIcebergTableSchema(schema, context); + processor.addIcebergTableSchema(schema, context, annotations); FAIL() << "The annotation does not describe the field type and must be rejected"; } catch (const DB::Exception & e) @@ -176,7 +177,7 @@ TEST(IcebergSchemaProcessor, UnknownFieldInSchemaProducesNullableNothing) { auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"placeholder","required":false,"type":"unknown"}]})json"); IcebergSchemaProcessor processor(getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(schema, getContext().context)); + EXPECT_NO_THROW(processor.addIcebergTableSchema(schema, getContext().context, {})); } TEST(IcebergSchemaProcessor, GetSimpleTypeUnknownThrows) @@ -200,7 +201,7 @@ TEST(IcebergSchemaProcessor, InitialSchemaTypeWithStringLiteralArgumentThrows) { auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"MyType('Hello ( world )')"}]})json"); IcebergSchemaProcessor processor(getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context), DB::Exception); + EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context, {}), DB::Exception); } /// The primitive parser must accept the same inner-whitespace spellings that the @@ -230,8 +231,8 @@ TEST(IcebergSchemaProcessor, DecimalTypeWhitespaceIsInsensitive) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0)"}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20, 0)"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context)); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context, {})); } /// A genuinely different type bound to the same schema-id must still be rejected. @@ -240,8 +241,8 @@ TEST(IcebergSchemaProcessor, RebindingSchemaIdToDifferentTypeStillRejected) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0)"}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,2)"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context), DB::Exception); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context, {}), DB::Exception); } /// The manifest header 'schema' key is only a copy of the table schema at write time; metadata.json @@ -254,9 +255,9 @@ TEST(IcebergSchemaProcessor, ConflictingManifestSchemaToleratedWhenEnabled) auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(from_metadata, getContext().context); + processor.addIcebergTableSchema(from_metadata, getContext().context, {}); EXPECT_NO_THROW(processor.addIcebergTableSchema( - from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); /// The metadata.json copy must win: the field keeps the timestamptz type. auto schema = processor.getClickHouseTableSchemaById(0); @@ -270,10 +271,10 @@ TEST(IcebergSchemaProcessor, ConflictingManifestSchemaRejectedWhenDisabled) auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(from_metadata, getContext().context); + processor.addIcebergTableSchema(from_metadata, getContext().context, {}); EXPECT_THROW( processor.addIcebergTableSchema( - from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), DB::Exception); } @@ -284,10 +285,10 @@ TEST(IcebergSchemaProcessor, ConflictingMetadataSchemaAlwaysRejected) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); + processor.addIcebergTableSchema(first, getContext().context, {}); EXPECT_THROW( processor.addIcebergTableSchema( - second, getContext().context, IcebergSchemaProcessor::SchemaSource::Metadata, /*tolerate_conflicting_manifest_schemas=*/true), + second, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::Metadata, /*tolerate_conflicting_manifest_schemas=*/true), DB::Exception); } @@ -298,7 +299,7 @@ TEST(IcebergSchemaProcessor, ManifestSchemaWithNewIdRegistersNormally) auto from_manifest = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); IcebergSchemaProcessor processor(getContext().context); EXPECT_NO_THROW(processor.addIcebergTableSchema( - from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); EXPECT_TRUE(processor.hasClickHouseTableSchemaById(5)); } @@ -312,15 +313,15 @@ TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsReplacedByMetadata) auto other = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"},{"id":2,"name":"v","required":false,"type":"int"}]})json"); auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); - processor.addIcebergTableSchema(other, getContext().context); + processor.addIcebergTableSchema(from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + processor.addIcebergTableSchema(other, getContext().context, {}); /// Populate everything derived from the schema that came from the manifest. EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Nullable(DateTime64(6))"); auto stale_dag = processor.getSchemaTransformationDagByIds(getContext().context, 0, 1); ASSERT_NE(stale_dag, nullptr); - EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata, getContext().context)); + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata, getContext().context, {})); auto schema = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(schema->size(), 1u); @@ -331,10 +332,10 @@ TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsReplacedByMetadata) /// Once metadata.json has settled the schema-id, a later divergent schema from a manifest is handled like /// any header conflicting with a known metadata.json copy: ignored when tolerated, rejected otherwise. EXPECT_NO_THROW(processor.addIcebergTableSchema( - from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); EXPECT_THROW( processor.addIcebergTableSchema( - from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), DB::Exception); EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); } @@ -347,14 +348,14 @@ TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsConfirmedByIdentical auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); auto degraded = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(from_manifest, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); - EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata, getContext().context)); + processor.addIcebergTableSchema(from_manifest, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata, getContext().context, {})); EXPECT_NO_THROW(processor.addIcebergTableSchema( - degraded, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + degraded, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); EXPECT_THROW( processor.addIcebergTableSchema( - degraded, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + degraded, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), DB::Exception); EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); } @@ -368,11 +369,11 @@ TEST(IcebergSchemaProcessor, ConflictingManifestHeadersWithoutMetadataSchemaReje auto second = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); auto other = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(other, getContext().context); - processor.addIcebergTableSchema(first, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + processor.addIcebergTableSchema(other, getContext().context, {}); + processor.addIcebergTableSchema(first, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); try { - processor.addIcebergTableSchema(second, getContext().context, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + processor.addIcebergTableSchema(second, getContext().context, {}, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); FAIL() << "expected ICEBERG_SPECIFICATION_VIOLATION"; } catch (const DB::Exception & e) @@ -389,8 +390,8 @@ TEST(IcebergSchemaProcessor, RebindingSchemaIdToRenamedFieldStillRejected) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"long"}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c9","required":false,"type":"long"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context), DB::Exception); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context, {}), DB::Exception); } /// The whitespace-insensitive comparison must reach into list/map wrappers: the nested @@ -403,8 +404,8 @@ TEST(IcebergSchemaProcessor, ListElementDecimalWhitespaceIsInsensitive) auto second = parseSchema( R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20, 0)"}}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context)); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context, {})); } /// Same for map key/value primitive types (here map). @@ -415,8 +416,8 @@ TEST(IcebergSchemaProcessor, MapKeyValueDecimalWhitespaceIsInsensitive) auto second = parseSchema( R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"map","key-id":2,"key":"decimal(20, 0)","value-id":3,"value-required":false,"value":"decimal(10, 2)"}}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context)); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context, {})); } /// The Iceberg geography/geometry primitives carry parameters too, e.g. @@ -428,8 +429,8 @@ TEST(IcebergSchemaProcessor, GeographyTypeWhitespaceIsInsensitive) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C,A)"}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C, A)"}]})json"); IcebergSchemaProcessor processor(getContext().context, /*allow_geo_parser_=*/true); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context)); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context, {})); } /// A geo type string carrying leading/trailing whitespace must map to its alias just like the @@ -441,8 +442,8 @@ TEST(IcebergSchemaProcessor, GeographyTypeEdgeWhitespaceIsInsensitive) auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":" geography(C,A) "}]})json"); auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C, A)"}]})json"); IcebergSchemaProcessor processor(getContext().context, /*allow_geo_parser_=*/true); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context)); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second, getContext().context, {})); } /// Schema-evolution path: renaming a geo field across two schema-ids while only changing the @@ -454,8 +455,8 @@ TEST(IcebergSchemaProcessor, RenameGeoFieldAcrossSchemaIdsWithWhitespaceIsRename auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"geography(C,A)"}]})json"); auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"b","required":false,"type":"geography(C, A)"}]})json"); IcebergSchemaProcessor processor(getContext().context, /*allow_geo_parser_=*/true); - processor.addIcebergTableSchema(old_schema, getContext().context); - processor.addIcebergTableSchema(new_schema, getContext().context); + processor.addIcebergTableSchema(old_schema, getContext().context, {}); + processor.addIcebergTableSchema(new_schema, getContext().context, {}); auto dag = processor.getSchemaTransformationDagByIds(getContext().context, 0, 1); ASSERT_TRUE(dag); @@ -471,7 +472,7 @@ TEST(IcebergSchemaProcessor, InitialSchemaDecimalInnerWhitespaceAccepted) { auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal( 20, 0 )"}]})json"); IcebergSchemaProcessor processor(getContext().context); - EXPECT_NO_THROW(processor.addIcebergTableSchema(schema, getContext().context)); + EXPECT_NO_THROW(processor.addIcebergTableSchema(schema, getContext().context, {})); } /// Schema-evolution across two schema-ids where a decimal widens (allowed conversion) while its @@ -482,8 +483,8 @@ TEST(IcebergSchemaProcessor, WidenDecimalAcrossSchemaIdsWithInnerWhitespace) auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(10,2)"}]})json"); auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal( 20, 2 )"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(old_schema, getContext().context); - processor.addIcebergTableSchema(new_schema, getContext().context); + processor.addIcebergTableSchema(old_schema, getContext().context, {}); + processor.addIcebergTableSchema(new_schema, getContext().context, {}); auto dag = processor.getSchemaTransformationDagByIds(getContext().context, 0, 1); ASSERT_TRUE(dag); @@ -500,8 +501,8 @@ TEST(IcebergSchemaProcessor, RebindingListElementToDifferentTypeStillRejected) auto second = parseSchema( R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20,2)"}}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(first, getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context), DB::Exception); + processor.addIcebergTableSchema(first, getContext().context, {}); + EXPECT_THROW(processor.addIcebergTableSchema(second, getContext().context, {}), DB::Exception); } /// Spacing normalization only removes whitespace adjacent to the delimiters '(', ')', '[', ']', ','. @@ -523,7 +524,7 @@ TEST(IcebergSchemaProcessor, InitialSchemaDecimalMalformedInnerTokenWhitespaceTh { auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(2 0,0)"}]})json"); IcebergSchemaProcessor processor(getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context), DB::Exception); + EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context, {}), DB::Exception); } /// Trailing garbage after the scale token must be rejected. Canonicalizing spacing does not remove @@ -539,7 +540,7 @@ TEST(IcebergSchemaProcessor, InitialSchemaDecimalTrailingGarbageInScaleThrows) { auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0 0)"}]})json"); IcebergSchemaProcessor processor(getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context), DB::Exception); + EXPECT_THROW(processor.addIcebergTableSchema(schema, getContext().context, {}), DB::Exception); } /// A new schema-id introduced during evolution is parsed at add time (getSimpleType runs on every @@ -550,8 +551,8 @@ TEST(IcebergSchemaProcessor, SchemaEvolutionDecimalTrailingGarbageInScaleThrows) auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(10,2)"}]})json"); auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,2 2)"}]})json"); IcebergSchemaProcessor processor(getContext().context); - processor.addIcebergTableSchema(old_schema, getContext().context); - EXPECT_THROW(processor.addIcebergTableSchema(new_schema, getContext().context), DB::Exception); + processor.addIcebergTableSchema(old_schema, getContext().context, {}); + EXPECT_THROW(processor.addIcebergTableSchema(new_schema, getContext().context, {}), DB::Exception); } /// A missing scale ("decimal(20,)") or a sign-only scale ("decimal(20,+)") is malformed metadata and @@ -571,10 +572,10 @@ TEST(IcebergSchemaProcessor, AnnotatedSimpleAggregateFunctionFieldNeedsNoSetting { tryRegisterAggregateFunctions(); auto schema = parseSchema( - R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long","clickhouse.type":"SimpleAggregateFunction(sum, Int64)"}]})json"); + R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long"}]})json"); auto context = contextWithAggregateFunctionStates(false); IcebergSchemaProcessor processor(context); - processor.addIcebergTableSchema(schema, context); + processor.addIcebergTableSchema(schema, context, {{1, "SimpleAggregateFunction(sum, Int64)"}}); auto columns = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(columns->size(), 1u); @@ -582,24 +583,86 @@ TEST(IcebergSchemaProcessor, AnnotatedSimpleAggregateFunctionFieldNeedsNoSetting EXPECT_EQ(columns->front().type->getName(), "SimpleAggregateFunction(sum, Int64)"); } +TEST(IcebergSchemaProcessor, ChangedAnnotationReparsesIdenticalSchema) +{ + tryRegisterAggregateFunctions(); + const std::string schema_json = R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long"}]})json"; + auto context = contextWithAggregateFunctionStates(false); + IcebergSchemaProcessor processor(context); + + processor.addIcebergTableSchema(parseSchema(schema_json), context, {{1, "SimpleAggregateFunction(sum, Int64)"}}); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "SimpleAggregateFunction(sum, Int64)"); + + processor.addIcebergTableSchema(parseSchema(schema_json), context, {}); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Int64"); + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Int64"); +} + +TEST(IcebergSchemaProcessor, SchemasKeepTheAnnotationsTheyWereRegisteredWith) +{ + tryRegisterAggregateFunctions(); + auto context = contextWithAggregateFunctionStates(false); + IcebergSchemaProcessor processor(context); + + processor.addIcebergTableSchema( + parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long"}]})json"), + context, + {{1, "SimpleAggregateFunction(sum, Int64)"}}); + processor.addIcebergTableSchema( + parseSchema( + R"json({"schema-id":1,"fields":[{"id":1,"name":"s","required":true,"type":"long"},{"id":2,"name":"t","required":false,"type":"long"}]})json"), + context, + {}); + + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "SimpleAggregateFunction(sum, Int64)"); + EXPECT_EQ(processor.getFieldCharacteristics(1, 1).type->getName(), "Int64"); + + auto dag = processor.getSchemaTransformationDagByIds(context, 0, 1); + ASSERT_EQ(dag->getInputs().size(), 1u); + EXPECT_EQ(dag->getInputs().front()->result_type->getName(), "SimpleAggregateFunction(sum, Int64)"); +} + TEST(IcebergSchemaProcessor, AnnotationNotMatchingTheFieldTypeIsRejected) { tryRegisterAggregateFunctions(); - expectAnnotatedSchemaRejected(parseSchema( - R"json({"schema-id":0,"fields":[{"id":1,"name":"u","required":true,"type":"long","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json")); + expectAnnotatedSchemaRejected( + parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"u","required":true,"type":"long"}]})json"), + {{1, "AggregateFunction(uniq, UInt64)"}}); +} + +/// Another engine promoted the field from `int` to `long`, leaving the annotation stale. +TEST(IcebergSchemaProcessor, AnnotationOfAPromotedFieldIsRejected) +{ + tryRegisterAggregateFunctions(); + expectAnnotatedSchemaRejected( + parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long"}]})json"), + {{1, "SimpleAggregateFunction(max, Int32)"}}); +} + +/// ClickHouse writes both `Int64` and `UInt64` as `long`. +TEST(IcebergSchemaProcessor, AnnotationWrittenAsTheSameIcebergTypeIsAccepted) +{ + tryRegisterAggregateFunctions(); + auto schema = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"s","required":true,"type":"long"}]})json"); + auto context = contextWithAggregateFunctionStates(false); + IcebergSchemaProcessor processor(context); + processor.addIcebergTableSchema(schema, context, {{1, "SimpleAggregateFunction(sum, UInt64)"}}); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "SimpleAggregateFunction(sum, UInt64)"); } TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSucceeds) { tryRegisterAggregateFunctions(); const std::string refused_json - = R"json({"schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"long"},{"id":2,"name":"u","required":false,"type":"binary","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json"; + = R"json({"schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"long"},{"id":2,"name":"u","required":true,"type":"binary"}]})json"; IcebergSchemaProcessor processor(getContext().context); + const ClickHouseTypeAnnotations annotations{{2, "AggregateFunction(uniq, UInt64)"}}; auto refused_context = contextWithAggregateFunctionStates(false); try { - processor.addIcebergTableSchema(parseSchema(refused_json), refused_context); + processor.addIcebergTableSchema(parseSchema(refused_json), refused_context, annotations); FAIL() << "The annotation names an aggregate function and must be refused without the setting"; } catch (const DB::Exception & e) @@ -613,8 +676,8 @@ TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSuccee auto allowed_context = contextWithAggregateFunctionStates(true); const std::string allowed_json - = R"json({"schema-id":0,"fields":[{"id":1,"name":"a2","required":false,"type":"int"},{"id":2,"name":"u","required":false,"type":"binary","clickhouse.type":"AggregateFunction(uniq, UInt64)"}]})json"; - ASSERT_NO_THROW(processor.addIcebergTableSchema(parseSchema(allowed_json), allowed_context)); + = R"json({"schema-id":0,"fields":[{"id":1,"name":"a2","required":false,"type":"int"},{"id":2,"name":"u","required":true,"type":"binary"}]})json"; + ASSERT_NO_THROW(processor.addIcebergTableSchema(parseSchema(allowed_json), allowed_context, annotations)); auto columns = processor.getClickHouseTableSchemaById(0); ASSERT_EQ(columns->size(), 2u); EXPECT_EQ(columns->front().name, "a2"); @@ -625,6 +688,34 @@ TEST(IcebergSchemaProcessor, RetryAfterRefusedAggregateFunctionStateSchemaSuccee EXPECT_FALSE(processor.tryGetColumnIDByName(0, "a").has_value()); } +TEST(IcebergSchemaProcessor, RefusedChangedAnnotationKeepsRegisteredSchema) +{ + tryRegisterAggregateFunctions(); + const std::string schema_json = R"json({"schema-id":0,"fields":[{"id":1,"name":"u","required":true,"type":"binary"}]})json"; + auto context = contextWithAggregateFunctionStates(false); + IcebergSchemaProcessor processor(context); + + processor.addIcebergTableSchema(parseSchema(schema_json), context, {}); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "String"); + + try + { + processor.addIcebergTableSchema(parseSchema(schema_json), context, {{1, "AggregateFunction(uniq, UInt64)"}}); + FAIL() << "The annotation names an aggregate function and must be refused without the setting"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::SUPPORT_IS_DISABLED) << e.message(); + } + + ASSERT_TRUE(processor.hasClickHouseTableSchemaById(0)); + auto columns = processor.getClickHouseTableSchemaById(0); + ASSERT_EQ(columns->size(), 1u); + EXPECT_EQ(columns->front().type->getName(), "String"); + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "String"); + EXPECT_EQ(processor.tryGetColumnIDByName(0, "u"), 1); +} + #if USE_AVRO TEST(IcebergSchemaProcessor, SkippingTheSchemaNeitherAppliesTheGateNorPublishesTheSchema) { @@ -633,8 +724,8 @@ TEST(IcebergSchemaProcessor, SkippingTheSchemaNeitherAppliesTheGateNorPublishesT auto metadata = parser .parse( R"json({"format-version":2,"current-schema-id":0,"schemas":[{"schema-id":0,)json" - R"json("fields":[{"id":1,"name":"u","required":false,"type":"binary",)json" - R"json("clickhouse.type":"AggregateFunction(uniq, UInt64)"}]}]})json") + R"json("fields":[{"id":1,"name":"u","required":true,"type":"binary"}]}],)json" + R"json("properties":{"clickhouse.type.1.0":"AggregateFunction(uniq, UInt64)"}})json") .extract(); auto log = getLogger("IcebergSchemaProcessorTest"); @@ -659,4 +750,38 @@ TEST(IcebergSchemaProcessor, SkippingTheSchemaNeitherAppliesTheGateNorPublishesT EXPECT_EQ(columns->front().name, "u"); EXPECT_EQ(columns->front().type->getName(), "AggregateFunction(uniq, UInt64)"); } + +TEST(IcebergSchemaProcessor, PropertyWithClickHouseTypePrefixButNoFieldIdIsNotAnAnnotation) +{ + Poco::JSON::Parser parser; + auto metadata = parser + .parse( + R"json({"properties":{"clickhouse.type.comment":"x","clickhouse.type.1":"x",)json" + R"json("clickhouse.type.1.x":"x","clickhouse.type.1.0.1":"x",)json" + R"json("clickhouse.type.1.0":"SimpleAggregateFunction(sum, Int64)"}})json") + .extract(); + + ClickHouseTypeAnnotations annotations; + ASSERT_NO_THROW(annotations = getClickHouseTypeAnnotations(metadata, 0)); + EXPECT_EQ(annotations, (ClickHouseTypeAnnotations{{1, "SimpleAggregateFunction(sum, Int64)"}})); +} + +TEST(IcebergSchemaProcessor, AnnotationInEffectIsTheOneWithTheGreatestSchemaIdNotAbove) +{ + Poco::JSON::Parser parser; + auto metadata = parser + .parse( + R"json({"properties":{"clickhouse.type.1.0":"SimpleAggregateFunction(max, Int32)",)json" + R"json("clickhouse.type.1.2":"SimpleAggregateFunction(max, Int64)","clickhouse.type.1.4":"",)json" + R"json("clickhouse.type.2.3":"SimpleAggregateFunction(sum, Int64)"}})json") + .extract(); + + EXPECT_EQ(getClickHouseTypeAnnotations(metadata, 0), (ClickHouseTypeAnnotations{{1, "SimpleAggregateFunction(max, Int32)"}})); + EXPECT_EQ(getClickHouseTypeAnnotations(metadata, 1), (ClickHouseTypeAnnotations{{1, "SimpleAggregateFunction(max, Int32)"}})); + EXPECT_EQ(getClickHouseTypeAnnotations(metadata, 2), (ClickHouseTypeAnnotations{{1, "SimpleAggregateFunction(max, Int64)"}})); + EXPECT_EQ( + getClickHouseTypeAnnotations(metadata, 3), + (ClickHouseTypeAnnotations{{1, "SimpleAggregateFunction(max, Int64)"}, {2, "SimpleAggregateFunction(sum, Int64)"}})); + EXPECT_EQ(getClickHouseTypeAnnotations(metadata, 4), (ClickHouseTypeAnnotations{{2, "SimpleAggregateFunction(sum, Int64)"}})); +} #endif diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index 84026dba4ac37..5eff8f9eb4b07 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -1257,6 +1257,72 @@ Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and | `map` | `Map` | | `struct` | `Tuple` | +### Aggregate function states {#aggregate-function-states} + +Iceberg has no aggregate-state type, so ClickHouse stores the two aggregate-state types as ordinary +Iceberg values and records the ClickHouse type name in the table property `clickhouse.type..`, +where `` is the Iceberg field id of the column. The type applies to the schemas with an id not +below ``, up to the next such property of the field; an empty value means no type. So older +schemas keep the type their data was written with. Catalogs and other engines that commit to the +table keep table properties: + +| ClickHouse type | Iceberg type | Stored as | +|---|---|---| +| `AggregateFunction(f, T...)` | `binary` | The serialized state, the same bytes `f` writes with the `-State` combinator | +| `SimpleAggregateFunction(f, T)` | whatever `T` maps to | An ordinary value of type `T` | + +Other query engines ignore the property and see a plain `binary` (or `T`-typed) column. + +Creating such a table requires +[`allow_experimental_aggregate_function_states_in_open_formats`](/reference/settings/session-settings/allow-experimental#allow_experimental_aggregate_function_states_in_open_formats), +and so does every query that reads a column whose `clickhouse.type` property names an `AggregateFunction` - an +`INSERT` or an `ALTER TABLE ... EXPORT PART` as much as a `SELECT`. A state is an opaque blob handed +to the deserializer of the aggregate function the table's metadata names, so with the setting enabled +it is that metadata, not the query, which chooses the deserializer; keep it disabled for tables from +untrusted sources, where such a field is then rejected rather than read as `String`. A +`SimpleAggregateFunction` column is not gated on read, holding ordinary values of its storage type. + +The setting is read from the query that parses the schema, so a session `SET` or a `SETTINGS` clause +on that query supplies it. The `iceberg*` table functions build a fresh storage object per query, so +for them it takes effect per query; the table engine parses the schema once and keeps it, so the +query that first touches the table after `ATTACH` decides, and that outcome holds - including for +queries that do not set it - until `DETACH TABLE` or a server restart. + +Writing goes through the ordinary Parquet writer, which takes the same setting from its format +settings. An object storage table engine freezes its format settings at `CREATE TABLE` - the server +settings plus that query's `SETTINGS` clause, session settings ignored - so for `INSERT` the setting +must also be in that clause; given only on the `INSERT`, it does not reach the writer: + +```sql +CREATE TABLE agg (k UInt32, u AggregateFunction(uniq, UInt64), s SimpleAggregateFunction(sum, UInt64)) +ENGINE = IcebergLocal('/path/to/table/') +PARTITION BY k +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; + +-- The states merge exactly as they do in an AggregatingMergeTree table. +SELECT k, uniqMerge(u), sum(s) FROM agg GROUP BY k +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; +``` + +`ALTER TABLE ... EXPORT PART` and `ALTER TABLE ... EXPORT PARTITION` from an `AggregatingMergeTree` +table into such an Iceberg table work as well, so a partition of pre-aggregated states can be moved +into the lake without finalizing it. Both take the setting off the `ALTER` query itself: + +```sql +ALTER TABLE mt EXPORT PART 'all_1_1_0' TO TABLE agg +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; + +-- EXPORT PARTITION, which only ReplicatedMergeTree implements, records the value in its manifest, +-- so every replica executing the task applies it in place of its own profile. +ALTER TABLE rmt EXPORT PARTITION ID '1' TO TABLE agg +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1; +``` + +The Parquet footer of each data file carries the same information, so `DESCRIBE file('data.parquet')` +reports the aggregate types too; that path goes through Parquet schema inference, gated by the same +setting. See +[aggregate function states in Parquet](/reference/formats/Parquet/Parquet#aggregate-function-states). + ## Schema and write compatibility limitations {#schema-and-write-compatibility-limitations} The data type mappings above apply to reads. The following limitations apply when ClickHouse creates or evolves Iceberg schemas and writes data: diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 430f3d593f656..a2dfa6fc7bc14 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3939,3 +3939,67 @@ def test_datalake_catalog_wrapper_object_storage_cluster(started_cluster, engine node2.query(f"DROP DATABASE IF EXISTS {catalog_db}") if cluster_mode == "database": node.query(f"DROP DATABASE IF EXISTS {cluster_catalog_name}") + + +def test_clickhouse_type_annotation_survives_catalog(started_cluster): + node = started_cluster.instances["node1"] + ns, tbl = f"ann_{uuid.uuid4().hex}", "t" + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + node.query( + f""" +CREATE TABLE {CATALOG_NAME}.`{ns}.{tbl}` (id Int64, s SimpleAggregateFunction(sum, Int64)) +ENGINE = IcebergS3('http://minio1:9001/warehouse-rest/{ns}/', '{minio_access_key}', '{minio_secret_key}') +SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1 + """, + settings={ + "allow_experimental_database_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "allow_experimental_aggregate_function_states_in_open_formats": 1, + }, + ) + catalog = load_catalog_impl(started_cluster) + + def metadata(): + loc = catalog.load_table(f"{ns}.{tbl}").metadata_location + bucket, key = loc[len("s3://"):].split("/", 1) + return json.loads(get_file_contents(started_cluster.minio_client, bucket, key)) + + def annotations(schema_id=None): + # The `clickhouse.type..` entry with the greatest schema id not above `schema_id` is in effect. + meta = metadata() + if schema_id is None: + schema_id = meta["current-schema-id"] + schema = next(s for s in meta["schemas"] if s["schema-id"] == schema_id) + entries = {} + for key, value in meta.get("properties", {}).items(): + parts = key.split(".") + if len(parts) == 4 and parts[:2] == ["clickhouse", "type"] and parts[2].isdigit() and parts[3].isdigit(): + field_id, entry_schema_id = int(parts[2]), int(parts[3]) + if entry_schema_id <= schema_id and entry_schema_id >= entries.get(field_id, (-1, None))[0]: + entries[field_id] = (entry_schema_id, value) + return {f["name"]: entries.get(f["id"], (None, None))[1] or None for f in schema["fields"]} + + # 1. Survives CREATE through the REST catalog? + assert annotations()["s"] == "SimpleAggregateFunction(sum, Int64)" + # 2. Survives a commit by another engine? + arrow_schema = pa.schema([pa.field("id", pa.int64(), nullable=False), + pa.field("s", pa.int64(), nullable=False)]) + catalog.load_table(f"{ns}.{tbl}").append( + pa.Table.from_pylist([{"id": 1, "s": 2}], schema=arrow_schema)) + assert annotations()["s"] == "SimpleAggregateFunction(sum, Int64)" + # 3. ClickHouse still sees the type + assert "SimpleAggregateFunction" in node.query(f"DESCRIBE {CATALOG_NAME}.`{ns}.{tbl}`") + # 4. ALTER through the catalog records and removes the annotation + alter_settings = {"allow_insert_into_iceberg": 1, "allow_experimental_aggregate_function_states_in_open_formats": 1} + node.query( + f"ALTER TABLE {CATALOG_NAME}.`{ns}.{tbl}` ADD COLUMN m SimpleAggregateFunction(max, Nullable(Int64))", + settings=alter_settings) + assert annotations()["m"] == "SimpleAggregateFunction(max, Nullable(Int64))" + schema_id_before_modify = metadata()["current-schema-id"] + node.query(f"ALTER TABLE {CATALOG_NAME}.`{ns}.{tbl}` MODIFY COLUMN s Int64", settings=alter_settings) + # Only the annotation changes, so the catalog keeps the current schema. + assert metadata()["current-schema-id"] == schema_id_before_modify + assert annotations() == {"id": None, "s": None, "m": "SimpleAggregateFunction(max, Nullable(Int64))"} + # The schema the first rows were written with keeps its annotation. + assert annotations(0) == {"id": None, "s": "SimpleAggregateFunction(sum, Int64)"} + assert node.query(f"SELECT id, s, toTypeName(s) FROM {CATALOG_NAME}.`{ns}.{tbl}`") == "1\t2\tInt64\n" diff --git a/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py index 3167b89340b8b..8db1ee477d41f 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py @@ -158,13 +158,16 @@ def test_aggregate_states_round_trip(started_cluster_iceberg_with_spark, storage f"{WAREHOUSE}/{TABLE_NAME}/", ) - fields = fields_by_name(latest_metadata(TABLE_NAME)) + metadata = latest_metadata(TABLE_NAME) + fields = fields_by_name(metadata) + properties = metadata.get("properties", {}) assert fields["k"]["type"] == "int" - assert "clickhouse.type" not in fields["k"] + assert metadata["current-schema-id"] == 0 + assert f"clickhouse.type.{fields['k']['id']}.0" not in properties assert fields["u"]["type"] == "binary" - assert fields["u"]["clickhouse.type"] == "AggregateFunction(uniq, UInt64)" + assert properties[f"clickhouse.type.{fields['u']['id']}.0"] == "AggregateFunction(uniq, UInt64)" assert fields["s"]["type"] == "long" - assert fields["s"]["clickhouse.type"] == "SimpleAggregateFunction(sum, UInt64)" + assert properties[f"clickhouse.type.{fields['s']['id']}.0"] == "SimpleAggregateFunction(sum, UInt64)" @pytest.mark.parametrize("storage_type", ["s3"]) diff --git a/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.reference b/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.reference new file mode 100644 index 0000000000000..b1a5823548d80 --- /dev/null +++ b/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.reference @@ -0,0 +1,18 @@ +widen +k Int32 +x SimpleAggregateFunction(max, Int64) +1 10 SimpleAggregateFunction(max, Int64) +2 20 SimpleAggregateFunction(max, Int64) +1 10 SimpleAggregateFunction(max, Int64) +2 20 SimpleAggregateFunction(max, Int64) +time travel +1 10 SimpleAggregateFunction(max, Int32) +drop the annotation +k Int32 +x Int64 +1 10 Int64 +2 20 Int64 +1 10 Int64 +2 20 Int64 +time travel +1 10 SimpleAggregateFunction(max, Int32) diff --git a/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.sh b/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.sh new file mode 100755 index 0000000000000..0522276126db5 --- /dev/null +++ b/tests/queries/0_stateless/05428_iceberg_aggregate_function_state_annotation_per_schema.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The ClickHouse type annotation of an Iceberg field is recorded per schema id, so after `MODIFY COLUMN` +# the older schema keeps the type its data was written with. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -euo pipefail + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}_annotation" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" + rm -rf "${TABLE_PATH}" +} +trap cleanup EXIT + +CLIENT="${CLICKHOUSE_CLIENT} --allow_experimental_aggregate_function_states_in_open_formats=1 --allow_insert_into_iceberg=1" + +${CLIENT} --query " + CREATE TABLE ${TABLE} (k Int32, x SimpleAggregateFunction(max, Int32)) + ENGINE = IcebergLocal('${TABLE_PATH}') + SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES (1, 10)" +SNAPSHOT=$(${CLIENT} --query "SELECT snapshot_id FROM system.iceberg_history WHERE database = currentDatabase() AND table = '${TABLE}'") + +check() +{ + ${CLIENT} --query "DESCRIBE TABLE ${TABLE}" | cut -f1,2 + ${CLIENT} --query "SELECT k, x, toTypeName(x) FROM ${TABLE} ORDER BY k" + ${CLIENT} --query "SELECT k, x, toTypeName(x) FROM icebergLocal('${TABLE_PATH}') ORDER BY k" + echo "time travel" + ${CLIENT} --query "SELECT k, x, toTypeName(x) FROM ${TABLE} ORDER BY k SETTINGS iceberg_snapshot_id = ${SNAPSHOT}" +} + +echo "widen" +${CLIENT} --query "ALTER TABLE ${TABLE} MODIFY COLUMN x SimpleAggregateFunction(max, Int64)" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES (2, 20)" +check + +echo "drop the annotation" +${CLIENT} --query "ALTER TABLE ${TABLE} MODIFY COLUMN x Int64" +check From d2ee51dfa1c987340bba95b9041715bb496c6c89 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:48:18 +0100 Subject: [PATCH 5/6] Fix build without Avro: move Iceberg type helpers out of `USE_AVRO` `IcebergSchemaProcessor::getFieldType` is compiled without Avro but calls `getIcebergType` and `icebergTypesEqualIgnoringIds`, which were declared and defined only under `USE_AVRO`. Neither depends on Avro, so move them out of the guard. CI report: https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2680&sha=bf6ff058da7117bbd3c4ff1bc9bbb499b2236d38&name_0=PR&name_1=Fast%20test PR: https://github.com/Altinity/ClickHouse/pull/2680 --- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 498 +++++++++--------- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 22 +- 2 files changed, 268 insertions(+), 252 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 5b082d7c31c18..2db05d4691879 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -54,10 +54,263 @@ #include #include +#include + +namespace DB::ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} + +namespace DB::Iceberg +{ + +/// Returns type and required +std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version) +{ + switch (type->getTypeId()) + { + case TypeIndex::UInt8: + { + if (isBool(type)) + return {"boolean", true}; + return {"int", true}; + } + case TypeIndex::Int8: + case TypeIndex::UInt16: + case TypeIndex::Int16: + case TypeIndex::UInt32: + case TypeIndex::Int32: + return {"int", true}; + case TypeIndex::UInt64: + case TypeIndex::Int64: + return {"long", true}; + case TypeIndex::Float32: + return {"float", true}; + case TypeIndex::Float64: + return {"double", true}; + case TypeIndex::Date: + case TypeIndex::Date32: + return {"date", true}; + case TypeIndex::DateTime: + return {"timestamp", true}; + case TypeIndex::DateTime64: + { + /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds and is + /// allowed only in format version 3. Writing it into v1/v2 metadata is invalid. + const auto scale = getDecimalScale(*type); + if (scale == 9) + { + if (format_version < 3) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Iceberg type {} requires format version 3 or higher, got version {}", + type->getName(), + format_version); + + auto date_time64 = std::static_pointer_cast(type); + if (date_time64->hasExplicitTimeZone()) + return {Iceberg::f_timestamptz_ns, true}; + return {Iceberg::f_timestamp_ns, true}; + } + if (scale <= 6) + return {Iceberg::f_timestamp, true}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + } + case TypeIndex::Time: + return {"time", true}; + case TypeIndex::Time64: + if (getDecimalScale(*type) <= 6) + return {"time", true}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + case TypeIndex::String: + return {"string", true}; + case TypeIndex::AggregateFunction: + return {Iceberg::f_binary, true}; + case TypeIndex::UUID: + return {"uuid", true}; + case TypeIndex::Nothing: + return {"unknown", false}; + case TypeIndex::Decimal32: + case TypeIndex::Decimal64: + case TypeIndex::Decimal128: + case TypeIndex::Decimal256: + { + auto precision = getDecimalPrecision(*type); + if (precision > 38) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Iceberg decimal type supports precision up to 38, got {}", precision); + return {"decimal(" + std::to_string(precision) + ", " + std::to_string(getDecimalScale(*type)) + ")", true}; + } + case TypeIndex::Tuple: + { + auto type_tuple = std::static_pointer_cast(type); + Poco::JSON::Object::Ptr result = new Poco::JSON::Object; + result->set(Iceberg::f_type, "struct"); + Poco::JSON::Array::Ptr fields = new Poco::JSON::Array; + size_t iter_names = 1; + size_t iter_fields = iter; + iter += type_tuple->getElements().size(); + for (const auto & element : type_tuple->getElements()) + { + Poco::JSON::Object::Ptr field = new Poco::JSON::Object; + field->set(Iceberg::f_id, ++iter_fields); + field->set(Iceberg::f_name, type_tuple->getNameByPosition(iter_names)); + /// Recurse on the element itself: `getNormalizedType` would rename a nested + /// tuple's elements to "1", "2", ... (`DataTypeTuple::getNormalizedType`). + auto child_type = getIcebergType(element, iter, format_version); + field->set(Iceberg::f_required, child_type.second); + field->set(Iceberg::f_type, child_type.first); + fields->add(field); + ++iter_names; + } + result->set(Iceberg::f_fields, fields); + return {result, true}; + } + case TypeIndex::Array: + { + auto type_array = std::static_pointer_cast(type); + Poco::JSON::Object::Ptr field = new Poco::JSON::Object; + + field->set(Iceberg::f_type, "list"); + field->set(Iceberg::f_element_id, ++iter); + auto child_type = getIcebergType(type_array->getNestedType(), iter, format_version); + field->set(Iceberg::f_required, false); + field->set(Iceberg::f_element, child_type.first); + field->set(Iceberg::f_element_required, child_type.second); + return {field, true}; + } + case TypeIndex::Map: + { + auto type_map = std::static_pointer_cast(type); + Poco::JSON::Object::Ptr field = new Poco::JSON::Object; + + field->set(Iceberg::f_type, "map"); + field->set(Iceberg::f_key_id, ++iter); + field->set(Iceberg::f_value_id, ++iter); + + field->set(Iceberg::f_key, getIcebergType(type_map->getKeyType(), iter, format_version).first); + auto value_type = getIcebergType(type_map->getValueType(), iter, format_version); + field->set(Iceberg::f_value, value_type.first); + field->set(Iceberg::f_value_required, value_type.second); + return {field, true}; + } + case TypeIndex::Nullable: + { + auto type_nullable = std::static_pointer_cast(type); + return {getIcebergType(type_nullable->getNestedType(), iter, format_version).first, false}; + } + case TypeIndex::Variant: + { + if (type->getCustomName() && type->getCustomName()->getName() == "Geometry") + return {Iceberg::f_geometry, false}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + } + default: + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + } +} + +namespace +{ + +String removeASCIIWhitespace(const String & s) +{ + String result; + result.reserve(s.size()); + for (char c : s) + { + if (!isWhitespaceASCII(c)) + result.push_back(c); + } + return result; +} + +/// A missing key matches only a missing key. +bool optionalBoolsEqual(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second, const char * key) +{ + const bool first_has = first->has(key); + if (first_has != second->has(key)) + return false; + return !first_has || first->getValue(key) == second->getValue(key); +} + +} + +/// Only the properties that define the type are compared. Field ids are ignored because +/// `getIcebergType` allocates them from a running counter, so a previous attempt may have written +/// the very same type with lower ids. Other keys (`doc`, defaults, the `required` that +/// `getIcebergType` puts on a list) are ignored because other writers emit or omit them freely. +bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type) +{ + if (old_type.isString() || new_type.isString()) + { + if (!old_type.isString() || !new_type.isString()) + return false; + /// Writers differ in spacing, e.g. `decimal(10,2)` and `decimal(10, 2)`. + return removeASCIIWhitespace(old_type.extract()) == removeASCIIWhitespace(new_type.extract()); + } + + if (old_type.type() != typeid(Poco::JSON::Object::Ptr) || new_type.type() != typeid(Poco::JSON::Object::Ptr)) + return false; + + auto old_object = old_type.extract(); + auto new_object = new_type.extract(); + if (!old_object || !new_object) + return false; + + if (old_object->has("precision") || new_object->has("precision")) + { + return old_object->has("precision") && new_object->has("precision") + && old_object->getValue("precision") == new_object->getValue("precision") + && old_object->getValue("scale") == new_object->getValue("scale"); + } + + if (!old_object->has(Iceberg::f_type) || !new_object->has(Iceberg::f_type)) + return false; + const auto kind = old_object->getValue(Iceberg::f_type); + if (kind != new_object->getValue(Iceberg::f_type)) + return false; + + if (kind == Iceberg::f_list) + { + return optionalBoolsEqual(old_object, new_object, Iceberg::f_element_required) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_element), new_object->get(Iceberg::f_element)); + } + + if (kind == Iceberg::f_map) + { + return optionalBoolsEqual(old_object, new_object, Iceberg::f_value_required) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_key), new_object->get(Iceberg::f_key)) + && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_value), new_object->get(Iceberg::f_value)); + } + + if (kind == Iceberg::f_struct) + { + auto old_fields = old_object->getArray(Iceberg::f_fields); + auto new_fields = new_object->getArray(Iceberg::f_fields); + if (!old_fields || !new_fields || old_fields->size() != new_fields->size()) + return false; + for (UInt32 i = 0; i < old_fields->size(); ++i) + { + auto old_field = old_fields->getObject(i); + auto new_field = new_fields->getObject(i); + if (!old_field || !new_field + || old_field->getValue(Iceberg::f_name) != new_field->getValue(Iceberg::f_name) + || !optionalBoolsEqual(old_field, new_field, Iceberg::f_required) + || !icebergTypesEqualIgnoringIds(old_field->get(Iceberg::f_type), new_field->get(Iceberg::f_type))) + return false; + } + return true; + } + + return false; +} + +} + #if USE_AVRO #include -#include #include #include #include @@ -84,7 +337,6 @@ using namespace DB; namespace DB::ErrorCodes { extern const int FILE_DOESNT_EXIST; -extern const int BAD_ARGUMENTS; extern const int ICEBERG_SPECIFICATION_VIOLATION; extern const int LOGICAL_ERROR; extern const int UNSUPPORTED_METHOD; @@ -711,248 +963,6 @@ void addIcebergTableSchemasFromMetadata( add_schema(schemas->getObject(i)); } -/// Returns type and required -std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version) -{ - switch (type->getTypeId()) - { - case TypeIndex::UInt8: - { - if (isBool(type)) - return {"boolean", true}; - return {"int", true}; - } - case TypeIndex::Int8: - case TypeIndex::UInt16: - case TypeIndex::Int16: - case TypeIndex::UInt32: - case TypeIndex::Int32: - return {"int", true}; - case TypeIndex::UInt64: - case TypeIndex::Int64: - return {"long", true}; - case TypeIndex::Float32: - return {"float", true}; - case TypeIndex::Float64: - return {"double", true}; - case TypeIndex::Date: - case TypeIndex::Date32: - return {"date", true}; - case TypeIndex::DateTime: - return {"timestamp", true}; - case TypeIndex::DateTime64: - { - /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds and is - /// allowed only in format version 3. Writing it into v1/v2 metadata is invalid. - const auto scale = getDecimalScale(*type); - if (scale == 9) - { - if (format_version < 3) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Iceberg type {} requires format version 3 or higher, got version {}", - type->getName(), - format_version); - - auto date_time64 = std::static_pointer_cast(type); - if (date_time64->hasExplicitTimeZone()) - return {Iceberg::f_timestamptz_ns, true}; - return {Iceberg::f_timestamp_ns, true}; - } - if (scale <= 6) - return {Iceberg::f_timestamp, true}; - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); - } - case TypeIndex::Time: - return {"time", true}; - case TypeIndex::Time64: - if (getDecimalScale(*type) <= 6) - return {"time", true}; - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); - case TypeIndex::String: - return {"string", true}; - case TypeIndex::AggregateFunction: - return {Iceberg::f_binary, true}; - case TypeIndex::UUID: - return {"uuid", true}; - case TypeIndex::Nothing: - return {"unknown", false}; - case TypeIndex::Decimal32: - case TypeIndex::Decimal64: - case TypeIndex::Decimal128: - case TypeIndex::Decimal256: - { - auto precision = getDecimalPrecision(*type); - if (precision > 38) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Iceberg decimal type supports precision up to 38, got {}", precision); - return {"decimal(" + std::to_string(precision) + ", " + std::to_string(getDecimalScale(*type)) + ")", true}; - } - case TypeIndex::Tuple: - { - auto type_tuple = std::static_pointer_cast(type); - Poco::JSON::Object::Ptr result = new Poco::JSON::Object; - result->set(Iceberg::f_type, "struct"); - Poco::JSON::Array::Ptr fields = new Poco::JSON::Array; - size_t iter_names = 1; - size_t iter_fields = iter; - iter += type_tuple->getElements().size(); - for (const auto & element : type_tuple->getElements()) - { - Poco::JSON::Object::Ptr field = new Poco::JSON::Object; - field->set(Iceberg::f_id, ++iter_fields); - field->set(Iceberg::f_name, type_tuple->getNameByPosition(iter_names)); - /// Recurse on the element itself: `getNormalizedType` would rename a nested - /// tuple's elements to "1", "2", ... (`DataTypeTuple::getNormalizedType`). - auto child_type = getIcebergType(element, iter, format_version); - field->set(Iceberg::f_required, child_type.second); - field->set(Iceberg::f_type, child_type.first); - fields->add(field); - ++iter_names; - } - result->set(Iceberg::f_fields, fields); - return {result, true}; - } - case TypeIndex::Array: - { - auto type_array = std::static_pointer_cast(type); - Poco::JSON::Object::Ptr field = new Poco::JSON::Object; - - field->set(Iceberg::f_type, "list"); - field->set(Iceberg::f_element_id, ++iter); - auto child_type = getIcebergType(type_array->getNestedType(), iter, format_version); - field->set(Iceberg::f_required, false); - field->set(Iceberg::f_element, child_type.first); - field->set(Iceberg::f_element_required, child_type.second); - return {field, true}; - } - case TypeIndex::Map: - { - auto type_map = std::static_pointer_cast(type); - Poco::JSON::Object::Ptr field = new Poco::JSON::Object; - - field->set(Iceberg::f_type, "map"); - field->set(Iceberg::f_key_id, ++iter); - field->set(Iceberg::f_value_id, ++iter); - - field->set(Iceberg::f_key, getIcebergType(type_map->getKeyType(), iter, format_version).first); - auto value_type = getIcebergType(type_map->getValueType(), iter, format_version); - field->set(Iceberg::f_value, value_type.first); - field->set(Iceberg::f_value_required, value_type.second); - return {field, true}; - } - case TypeIndex::Nullable: - { - auto type_nullable = std::static_pointer_cast(type); - return {getIcebergType(type_nullable->getNestedType(), iter, format_version).first, false}; - } - case TypeIndex::Variant: - { - if (type->getCustomName() && type->getCustomName()->getName() == "Geometry") - return {Iceberg::f_geometry, false}; - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); - } - default: - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); - } -} - -namespace -{ - -String removeASCIIWhitespace(const String & s) -{ - String result; - result.reserve(s.size()); - for (char c : s) - { - if (!isWhitespaceASCII(c)) - result.push_back(c); - } - return result; -} - -/// A missing key matches only a missing key. -bool optionalBoolsEqual(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second, const char * key) -{ - const bool first_has = first->has(key); - if (first_has != second->has(key)) - return false; - return !first_has || first->getValue(key) == second->getValue(key); -} - -} - -/// Only the properties that define the type are compared. Field ids are ignored because -/// `getIcebergType` allocates them from a running counter, so a previous attempt may have written -/// the very same type with lower ids. Other keys (`doc`, defaults, the `required` that -/// `getIcebergType` puts on a list) are ignored because other writers emit or omit them freely. -bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type) -{ - if (old_type.isString() || new_type.isString()) - { - if (!old_type.isString() || !new_type.isString()) - return false; - /// Writers differ in spacing, e.g. `decimal(10,2)` and `decimal(10, 2)`. - return removeASCIIWhitespace(old_type.extract()) == removeASCIIWhitespace(new_type.extract()); - } - - if (old_type.type() != typeid(Poco::JSON::Object::Ptr) || new_type.type() != typeid(Poco::JSON::Object::Ptr)) - return false; - - auto old_object = old_type.extract(); - auto new_object = new_type.extract(); - if (!old_object || !new_object) - return false; - - if (old_object->has("precision") || new_object->has("precision")) - { - return old_object->has("precision") && new_object->has("precision") - && old_object->getValue("precision") == new_object->getValue("precision") - && old_object->getValue("scale") == new_object->getValue("scale"); - } - - if (!old_object->has(Iceberg::f_type) || !new_object->has(Iceberg::f_type)) - return false; - const auto kind = old_object->getValue(Iceberg::f_type); - if (kind != new_object->getValue(Iceberg::f_type)) - return false; - - if (kind == Iceberg::f_list) - { - return optionalBoolsEqual(old_object, new_object, Iceberg::f_element_required) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_element), new_object->get(Iceberg::f_element)); - } - - if (kind == Iceberg::f_map) - { - return optionalBoolsEqual(old_object, new_object, Iceberg::f_value_required) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_key), new_object->get(Iceberg::f_key)) - && icebergTypesEqualIgnoringIds(old_object->get(Iceberg::f_value), new_object->get(Iceberg::f_value)); - } - - if (kind == Iceberg::f_struct) - { - auto old_fields = old_object->getArray(Iceberg::f_fields); - auto new_fields = new_object->getArray(Iceberg::f_fields); - if (!old_fields || !new_fields || old_fields->size() != new_fields->size()) - return false; - for (UInt32 i = 0; i < old_fields->size(); ++i) - { - auto old_field = old_fields->getObject(i); - auto new_field = new_fields->getObject(i); - if (!old_field || !new_field - || old_field->getValue(Iceberg::f_name) != new_field->getValue(Iceberg::f_name) - || !optionalBoolsEqual(old_field, new_field, Iceberg::f_required) - || !icebergTypesEqualIgnoringIds(old_field->get(Iceberg::f_type), new_field->get(Iceberg::f_type))) - return false; - } - return true; - } - - return false; -} - Poco::Dynamic::Var getAvroType(DataTypePtr type, Int32 field_id) { switch (type->getTypeId()) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index 292a5ec953737..e33a0e8839df8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -16,6 +16,20 @@ #include #include +#include + +namespace DB::Iceberg +{ + +/// Maps a ClickHouse type to an Iceberg JSON type. `timestamp_ns` / `timestamptz_ns` +/// require `format_version` >= 3; lower versions throw instead of writing v3 types +/// into v1/v2 metadata. +std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version); + +/// Whether two Iceberg type descriptors denote the same type, ignoring the ids of nested fields. +bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type); + +} #if USE_AVRO @@ -94,14 +108,6 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( const std::optional & table_uuid); -/// Maps a ClickHouse type to an Iceberg JSON type. `timestamp_ns` / `timestamptz_ns` -/// require `format_version` >= 3; lower versions throw instead of writing v3 types -/// into v1/v2 metadata. -std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version); - -/// Whether two Iceberg type descriptors denote the same type, ignoring the ids of nested fields. -bool icebergTypesEqualIgnoringIds(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type); - /// Throws if `type` contains `Nothing` at any depth (the Iceberg `unknown` type) and /// `format_version` is below 3, the first format version that defines `unknown`. void checkUnknownTypeAllowed(const String & column_name, const DataTypePtr & type, Int64 format_version); From 430f9ca5726609a26040a5006c4c045b72106bb5 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:02:04 +0100 Subject: [PATCH 6/6] Skip Iceberg min/max bounds of fields with a ClickHouse type annotation Bounds are stored in the Iceberg encoding of the field, but were decoded with the annotated ClickHouse type: - `AggregateFunction`: `ColumnAggregateFunction::insertData` treated the bound bytes written by Spark as a state pointer, so reading the table crashed in `UniquesHashSet::write`. - `SimpleAggregateFunction` over `DateTime64(3)` or `Time`: microsecond bounds were read at the annotated scale or width, so `SELECT` returned `9999-12-31 23:59:59.000` through the min/max read optimization and min/max pruning skipped files with matching rows. `ManifestFilesPruner` and `DataFileMetaInfo` now ignore bounds of fields for which `IcebergSchemaProcessor::hasClickHouseTypeAnnotation` is true. Related: https://github.com/Altinity/ClickHouse/pull/2680#issuecomment-6521266485 --- .../DataLakes/IDataLakeMetadata.cpp | 4 +- .../Iceberg/IcebergFieldParseHelpers.cpp | 2 +- .../Iceberg/ManifestFilesPruning.cpp | 4 ++ .../DataLakes/Iceberg/SchemaProcessor.cpp | 8 +++ .../DataLakes/Iceberg/SchemaProcessor.h | 2 + .../test_aggregate_function_states.py | 64 +++++++++++++++++++ ..._function_state_annotated_bounds.reference | 2 + ...gregate_function_state_annotated_bounds.sh | 29 +++++++++ 8 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.reference create mode 100755 tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.sh diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp index a3f6898562874..751be03a5f42a 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp @@ -145,7 +145,9 @@ DataFileMetaInfo::DataFileMetaInfo( auto i_bounds = value_bounds_.find(column.first); auto i_type = type_by_index.find(column.first); - if (i_bounds != value_bounds_.end() && i_type != type_by_index.end()) + /// Bounds use the Iceberg encoding of the field, which an annotated type does not necessarily share. + if (i_bounds != value_bounds_.end() && i_type != type_by_index.end() + && !schema_processor.hasClickHouseTypeAnnotation(file_schema_id, column.first)) { const auto & type = i_type->second; if (const auto type_id = type->getTypeId(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index e062057a99c6a..fb4886040ef33 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -265,7 +265,6 @@ deserializeFieldFromBinaryRepr(const std::string & str, DataTypePtr expected_typ if (WhichDataType(non_nullable_type).isDateTime64()) return deserializeDateTime64FromBinaryRepr(str, *non_nullable_type, lower_bound); - auto column = non_nullable_type->createColumn(); if (WhichDataType(non_nullable_type).isDecimal()) { if (str.empty()) @@ -288,6 +287,7 @@ deserializeFieldFromBinaryRepr(const std::string & str, DataTypePtr expected_typ /// For all other types except decimal binary representation /// matches our internal representation + auto column = non_nullable_type->createColumn(); column->insertData(str.data(), str.length()); Field result; column->get(0, result); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index c49aaf07b5e71..c1b43f20256e2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -208,6 +208,10 @@ ManifestFilesPruner::ManifestFilesPruner( if (!name_and_type.has_value()) continue; + /// Bounds use the Iceberg encoding of the field, which an annotated type does not necessarily share. + if (schema_processor.hasClickHouseTypeAnnotation(initial_schema_id, used_column_id)) + continue; + min_max_column_types.emplace(used_column_id, name_and_type->type); name_and_type->name = DB::backQuote(DB::toString(used_column_id)); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index ee790d44eb0b5..4b18056d07776 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -1092,6 +1092,14 @@ bool IcebergSchemaProcessor::isSchemaRegisteredFromMetadata(Int32 id) const return iceberg_table_schemas_by_ids.contains(id) && !manifest_sourced_schema_ids.contains(id); } +bool IcebergSchemaProcessor::hasClickHouseTypeAnnotation(Int32 schema_id, Int32 field_id) const +{ + SharedLockGuard lock(mutex); + + auto it = clickhouse_type_annotations_by_schema_ids.find(schema_id); + return it != clickhouse_type_annotations_by_schema_ids.end() && it->second.contains(field_id); +} + std::unordered_map IcebergSchemaProcessor::traverseSchema(Poco::JSON::Array::Ptr schema) { std::unordered_map result; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index 55cf70273ea02..720f782dd1f38 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -122,6 +122,8 @@ class IcebergSchemaProcessor : private WithContext Poco::JSON::Object::Ptr getIcebergTableSchemaById(Int32 id) const; bool hasClickHouseTableSchemaById(Int32 id) const; bool isSchemaRegisteredFromMetadata(Int32 id) const; + /// Whether the type of `field_id` in schema `schema_id` comes from a `clickhouse.type` table property. + bool hasClickHouseTypeAnnotation(Int32 schema_id, Int32 field_id) const; static DataTypePtr getSimpleType(const String & type_name, ContextPtr context_, bool allow_geo_parser = true); diff --git a/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py index 8db1ee477d41f..03c95fe7bd41c 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_aggregate_function_states.py @@ -8,6 +8,7 @@ from helpers.iceberg_utils import ( create_iceberg_table, default_download_directory, + default_upload_directory, get_creation_expression, get_uuid_str, ) @@ -211,3 +212,66 @@ def test_aggregate_states_read_by_spark(started_cluster_iceberg_with_spark, stor for line in expected.split("\n"): k, state_hex, sum_value = line.split("\t") assert spark_rows[int(k)] == (state_hex, int(sum_value)) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_aggregate_states_merged_after_spark_append(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_agg_states_spark_append_" + storage_type + "_" + get_uuid_str() + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + SPARK_SCHEMA, + format_version=2, + settings=STATE_SETTINGS, + additional_settings=PARQUET_STATE_SETTINGS, + ) + insert_states(instance, TABLE_NAME, "toInt64(number)") + + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"{WAREHOUSE}/{TABLE_NAME}/", + f"{WAREHOUSE}/{TABLE_NAME}/", + ) + + with open(f"{WAREHOUSE}/{TABLE_NAME}/metadata/version-hint.text", "wb") as f: + f.write(b"1") + + appended = instance.query( + "SELECT toInt32(number % 2) AS k, hex(uniqState(toUInt64(number % 23))), sum(toInt64(number))" + " FROM numbers(200, 100) GROUP BY k ORDER BY k FORMAT TSV" + ).strip() + values = ", ".join( + f"({k}, X'{state_hex}', {sum_value})" + for k, state_hex, sum_value in (line.split("\t") for line in appended.split("\n")) + ) + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES {values}") + + # The appended file must carry bounds for the column holding the states. + u_field_id = fields_by_name(latest_metadata(TABLE_NAME))["u"]["id"] + spark_files = spark.sql( + f"SELECT lower_bounds, upper_bounds FROM spark_catalog.default.{TABLE_NAME}.files WHERE lower_bounds IS NOT NULL" + ).collect() + assert any( + u_field_id in (row["lower_bounds"] or {}) and u_field_id in (row["upper_bounds"] or {}) for row in spark_files + ), f"No data file has bounds for field {u_field_id}: {spark_files}" + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"{WAREHOUSE}/{TABLE_NAME}/", + f"{WAREHOUSE}/{TABLE_NAME}/", + ) + + assert instance.query( + f"SELECT k, uniqMerge(u), sum(s) FROM {TABLE_NAME} GROUP BY k ORDER BY k", + settings=STATE_SETTINGS, + ) == instance.query( + "SELECT toInt32(number % 2) AS k, uniq(toUInt64(number % 23)), sum(number)" + " FROM numbers(300) GROUP BY k ORDER BY k" + ) diff --git a/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.reference b/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.reference new file mode 100644 index 0000000000000..b39305ce26d04 --- /dev/null +++ b/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.reference @@ -0,0 +1,2 @@ +2026-01-01 00:00:00.123 +1 diff --git a/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.sh b/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.sh new file mode 100755 index 0000000000000..3bfc8a4a60970 --- /dev/null +++ b/tests/queries/0_stateless/05429_iceberg_aggregate_function_state_annotated_bounds.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Iceberg bounds of an annotated `DateTime64(3)` field are microseconds and must not be read at scale 3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -euo pipefail + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}_annotated_bounds" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" + rm -rf "${TABLE_PATH}" +} +trap cleanup EXIT + +CLIENT="${CLICKHOUSE_CLIENT} --allow_experimental_aggregate_function_states_in_open_formats=1 --allow_insert_into_iceberg=1" + +${CLIENT} --query " + CREATE TABLE ${TABLE} (x SimpleAggregateFunction(max, DateTime64(3, 'UTC'))) + ENGINE = IcebergLocal('${TABLE_PATH}') + SETTINGS allow_experimental_aggregate_function_states_in_open_formats = 1" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES ('2026-01-01 00:00:00.123')" +${CLIENT} --query "SELECT x FROM ${TABLE}" +${CLIENT} --query "SELECT count() FROM ${TABLE} WHERE x = toDateTime64('2026-01-01 00:00:00.123', 3, 'UTC')"