diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index e67faf84b24..8938993fa5a 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -19,8 +19,10 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.FlussConnection; import org.apache.fluss.client.metadata.KvSnapshotMetadata; import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.AppendWriter; @@ -1211,19 +1213,24 @@ void testKvSnapshotLeaseAfterCoordinatorServerRestart() throws Exception { assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isPresent(); restartCoordinatorServer(zkClient); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - lease.dropLease().get(); assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isNotPresent(); } private void restartCoordinatorServer(ZooKeeperClient zkClient) throws Exception { + MetadataUpdater metadataUpdater = ((FlussConnection) conn).getMetadataUpdater(); + ServerNode cachedCoordinator = metadataUpdater.getCoordinatorServer(); FLUSS_CLUSTER_EXTENSION.stopCoordinatorServer(); waitUntil( () -> !zkClient.getCoordinatorLeaderAddress().isPresent(), Duration.ofMinutes(1), "Coordinator server node still exists in ZooKeeper"); FLUSS_CLUSTER_EXTENSION.startCoordinatorServer(); + // Election completes before the new coordinator address reaches every tablet server. + // Wait for server metadata to converge so that the client's retry can discover it. + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + // Keep the client cache stale to exercise the lease's metadata refresh and retry. + assertThat(metadataUpdater.getCoordinatorServer()).isEqualTo(cachedCoordinator); } @Test @@ -2831,6 +2838,7 @@ void testKvSnapshotLeaseForLogTable() throws Exception { TableDescriptor logTableDescriptor = TableDescriptor.builder().schema(logSchema).distributedBy(3).build(); long tableId = createTable(logTablePath, logTableDescriptor, true); + FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId); // Create a KvSnapshotLease KvSnapshotLease lease = admin.createKvSnapshotLease("test-lease-log", 60000); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 3e86eb77579..6d6adb6a5ee 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2269,6 +2269,38 @@ public class ConfigOptions { + "Downstream consumers must be upgraded to Fluss v1.0+ before enabling this option, " + "as older versions cannot parse the extended batch format."); + public static final ConfigOption TABLE_KAFKA_ENABLED = + key("table.kafka.enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether the table is exposed as a Kafka topic. Enabling validates the " + + "existing schema and conversion options without adding columns or defaults."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + key("table.kafka.convert.value.type") + .stringType() + .noDefaultValue() + .withDescription( + "Required when Kafka compatibility is enabled: binary, string, or parse_json. " + + "Binary and string target the declared _kafka_value column."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + key("table.kafka.convert.value.layout") + .stringType() + .noDefaultValue() + .withDescription( + "JSON layout: flat (the read-time default for parse_json) or nested. " + + "Nested targets _kafka_value ROW. Not allowed for binary or string."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = + key("table.kafka.convert.key.type") + .stringType() + .noDefaultValue() + .withDescription( + "Optional Kafka key conversion type. Only binary is supported and requires " + + "_kafka_key BYTES. Declaring that column implies binary without persisting a default."); + // ------------------------------------------------------------------------ // ConfigOptions for Kv // ------------------------------------------------------------------------ @@ -2905,12 +2937,15 @@ public class ConfigOptions { .withDescription( "The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma."); - public static final ConfigOption KAFKA_DATABASE = - key("kafka.database") + public static final ConfigOption KAFKA_DEFAULT_DATABASE = + key("kafka.default.database") .stringType() - .defaultValue("kafka") + .defaultValue("fluss") .withDescription( - "The database for fluss kafka. The default database is `kafka`."); + "Default database for unqualified Kafka topic names. Fully qualified names may " + + "access other databases. When Kafka is enabled, a configured database other " + + "than 'fluss' is created if missing. The native 'fluss' database is created " + + "only when the cluster has no databases."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 37d5cc9f88e..415103c46ea 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -47,6 +47,10 @@ public class FlussConfigUtils { CLIENT_OPTIONS = extractConfigOptions("client."); ALTERABLE_TABLE_OPTIONS = Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_MODE.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ed2c111c335..b1bc64dfc83 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -54,6 +54,11 @@ public TableConfig(Configuration config) { this.config = config; } + /** Whether the table explicitly enables Kafka compatibility. */ + public boolean isKafkaEnabled() { + return config.get(ConfigOptions.TABLE_KAFKA_ENABLED); + } + /** Gets the replication factor of the table. */ public int getReplicationFactor() { return config.get(ConfigOptions.TABLE_REPLICATION_FACTOR); diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java new file mode 100644 index 00000000000..f8946107020 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFields().get(position).getName()); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java new file mode 100644 index 00000000000..9ab16195fbd --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable Kafka record mapping, shared by native DDL validation and Kafka request handling. + * Resolving a contract never changes the schema or materializes configuration defaults. + */ +@Internal +public final class KafkaTableSchema { + + public static final String KEY_COLUMN = "_kafka_key"; + public static final String VALUE_COLUMN = "_kafka_value"; + public static final String HEADERS_COLUMN = "_kafka_headers"; + public static final String METADATA_COLUMN = "_kafka_metadata"; + public static final String RESCUE_COLUMN = "_kafka_rescue"; + + /** Maximum recursive depth below each top-level JSON business field. */ + public static final int MAX_JSON_SCHEMA_NESTING_DEPTH = 64; + + private static final Set RESERVED_NAMES = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + KEY_COLUMN, + VALUE_COLUMN, + HEADERS_COLUMN, + METADATA_COLUMN, + RESCUE_COLUMN))); + private static final Set OPTIONS = + new HashSet<>( + Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key())); + + /** Supported value conversion contracts; decoders are provided by the Kafka module. */ + public enum ValueType { + BINARY, + STRING, + PARSE_JSON + } + + /** JSON business fields either occupy the table row or the fixed value ROW column. */ + public enum ValueLayout { + FLAT, + NESTED + } + + private final RowType rowType; + private final ValueType valueType; + private final @Nullable ValueLayout valueLayout; + private final KafkaFieldProjection valueProjection; + private final int keyPosition; + private final int headersPosition; + private final int metadataPosition; + private final int rescuePosition; + + private KafkaTableSchema( + RowType rowType, + ValueType valueType, + @Nullable ValueLayout valueLayout, + List valuePositions, + int keyPosition, + int headersPosition, + int metadataPosition, + int rescuePosition) { + this.rowType = rowType; + this.valueType = valueType; + this.valueLayout = valueLayout; + this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); + this.keyPosition = keyPosition; + this.headersPosition = headersPosition; + this.metadataPosition = metadataPosition; + this.rescuePosition = rescuePosition; + } + + /** Returns only the persisted opt-in flag; schema compatibility does not imply enablement. */ + public static boolean isEnabled(TableDescriptor table) { + return new TableConfig(Configuration.fromMap(table.getProperties())).isKafkaEnabled(); + } + + /** Validates the Kafka contract only when the final table state enables it. */ + public static void validate(TableDescriptor table) { + if (isEnabled(table)) { + resolve(table); + } + } + + /** Checks whether the canonical topic name satisfies Fluss and Kafka name restrictions. */ + public static boolean isValidTablePath(TablePath path) { + // Fluss identifiers are already restricted to Kafka's ASCII character set, without dots. + return path != null + && path.isValid() + && TablePath.validatePrefix(path.getDatabaseName()) == null + && TablePath.validatePrefix(path.getTableName()) == null + && path.toString().length() <= 249; + } + + /** Validates the canonical topic name before persisting an enabled table. */ + public static void validateTablePath(TablePath path, TableDescriptor table) { + if (isEnabled(table) && !isValidTablePath(path)) { + throw invalid("Table path cannot be represented as a Kafka topic: " + path); + } + } + + /** Returns whether a JSON member at the value mapping level has a reserved name. */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + + /** Resolves an explicitly enabled table or rejects its invalid contract. */ + public static KafkaTableSchema resolve(TableDescriptor table) { + if (!isEnabled(table)) { + throw invalid("Kafka compatibility is disabled; table.kafka.enabled must be true."); + } + validateOptions(table); + if (table.hasPrimaryKey() + || table.isPartitioned() + || new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid( + "Kafka requires a non-partitioned Arrow log table without a primary key."); + } + Configuration options = Configuration.fromMap(table.getProperties()); + String value = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); + if (value == null) { + throw invalid("Missing required table property 'table.kafka.convert.value.type'."); + } + ValueType valueType; + try { + valueType = ValueType.valueOf(value.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid( + "Unsupported table.kafka.convert.value.type '" + + value + + "'. Expected binary, string or parse_json."); + } + String layout = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT); + ValueLayout valueLayout = null; + if (valueType == ValueType.PARSE_JSON) { + try { + valueLayout = + layout == null + ? ValueLayout.FLAT + : ValueLayout.valueOf(layout.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid("Unsupported table.kafka.convert.value.layout '" + layout + "'."); + } + } else if (layout != null) { + throw invalid("table.kafka.convert.value.layout is only supported for parse_json."); + } + + RowType rowType = table.getSchema().getRowType(); + int keyPosition = rowType.getFieldIndex(KEY_COLUMN); + if (keyPosition >= 0 && !(rowType.getTypeAt(keyPosition) instanceof BytesType)) { + throw invalid("_kafka_key must be BYTES."); + } + String keyType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE); + if (keyType != null && (!"binary".equalsIgnoreCase(keyType.trim()) || keyPosition < 0)) { + throw invalid( + "table.kafka.convert.key.type only supports binary and requires _kafka_key BYTES."); + } + int headersPosition = rowType.getFieldIndex(HEADERS_COLUMN); + if (headersPosition >= 0) { + DataType headers = rowType.getTypeAt(headersPosition); + if (!(headers instanceof MapType) + || !(((MapType) headers).getKeyType() instanceof StringType) + || !(((MapType) headers).getValueType() instanceof BytesType)) { + throw invalid("_kafka_headers must be MAP."); + } + } + int rescuePosition = rowType.getFieldIndex(RESCUE_COLUMN); + if (rescuePosition >= 0 + && (!(rowType.getTypeAt(rescuePosition) instanceof StringType) + || !rowType.getTypeAt(rescuePosition).isNullable())) { + throw invalid("_kafka_rescue must be nullable STRING."); + } + int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); + if (metadataPosition >= 0) { + validateMetadataType(rowType.getTypeAt(metadataPosition)); + } + + List valuePositions = new ArrayList<>(); + if (valueLayout == ValueLayout.FLAT) { + if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { + throw invalid( + "_kafka_value is reserved for binary, string or nested JSON; " + + "it cannot be a flat JSON business field."); + } + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (!isReservedName(rowType.getFieldNames().get(i))) { + valuePositions.add(i); + } + } + } else { + int valuePosition = rowType.getFieldIndex(VALUE_COLUMN); + if (valuePosition < 0) { + throw invalid("Missing required value column _kafka_value."); + } + DataType type = rowType.getTypeAt(valuePosition); + boolean valid = + valueType == ValueType.BINARY + ? type instanceof BytesType + : valueType == ValueType.STRING + ? type instanceof StringType + : type instanceof RowType; + if (!valid) { + throw invalid( + "_kafka_value must be " + + (valueType == ValueType.BINARY + ? "BYTES" + : valueType == ValueType.STRING ? "STRING" : "ROW") + + " for " + + valueType + + "."); + } + for (String name : rowType.getFieldNames()) { + if (!isReservedName(name)) { + throw invalid( + "Column '" + + name + + "' has no value mapping; this conversion targets _kafka_value only."); + } + } + if (type instanceof RowType) { + for (String name : ((RowType) type).getFieldNames()) { + if (isReservedName(name)) { + throw invalid( + "Reserved name '" + + name + + "' cannot be a nested JSON business field."); + } + } + } + valuePositions.add(valuePosition); + } + if (valueType == ValueType.PARSE_JSON) { + for (int position : valuePositions) { + DataType type = rowType.getTypeAt(position); + if (valueLayout == ValueLayout.NESTED) { + // The physical value container represents the JSON object root, just like + // the table row in flat layout; it must not add a business nesting level. + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field); + } + } else { + validateJsonType(type); + } + } + } + return new KafkaTableSchema( + rowType, + valueType, + valueLayout, + valuePositions, + keyPosition, + headersPosition, + metadataPosition, + rescuePosition); + } + + /** + * Validates a top-level JSON business field without parser or protocol dependencies. DDL and + * decoder construction share this contract. Each business field starts at depth zero; ROW + * fields, ARRAY elements and MAP values each advance the depth by one. Nullability and scalar + * value constraints are enforced when decoding individual records. The JSON object root + * (including the physical nested _kafka_value ROW) is not counted. + */ + public static void validateJsonType(DataType type) { + validateJsonType(type, 0); + } + + private static void validateJsonType(DataType type, int depth) { + if (depth > MAX_JSON_SCHEMA_NESTING_DEPTH) { + throw invalid( + "Kafka JSON schema exceeds the maximum nesting depth of " + + MAX_JSON_SCHEMA_NESTING_DEPTH + + "."); + } + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case BINARY: + case BYTES: + case DECIMAL: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return; + case ARRAY: + validateJsonType(((ArrayType) type).getElementType(), depth + 1); + return; + case MAP: + MapType map = (MapType) type; + if (!(map.getKeyType() instanceof StringType)) { + throw invalid( + "Kafka JSON format only supports STRING map keys, but found " + + type.asSummaryString() + + "."); + } + validateJsonType(map.getValueType(), depth + 1); + return; + case ROW: + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field, depth + 1); + } + return; + default: + throw invalid( + "Kafka JSON format does not support Fluss type " + + type.asSummaryString() + + "."); + } + } + + private static void validateOptions(TableDescriptor table) { + for (String name : table.getProperties().keySet()) { + if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { + throw invalid("Unsupported Kafka table property '" + name + "'."); + } + } + for (String name : table.getCustomProperties().keySet()) { + if (name.startsWith("table.kafka.") + || name.startsWith("kafka.") + || name.startsWith("fluss.")) { + throw invalid( + "Unsupported Kafka custom property '" + + name + + "'. Use registered table.kafka.* properties and fixed system columns."); + } + } + } + + private static void validateMetadataType(DataType type) { + String message = + "_kafka_metadata must be ROW."; + if (!(type instanceof RowType)) { + throw invalid(message); + } + RowType row = (RowType) type; + if (!row.getFieldNames().equals(Arrays.asList("timestamp", "timestamp_type", "partition")) + || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) + || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT())) { + throw invalid(message); + } + } + + /** Returns the unchanged physical schema, including declared nullability. */ + public RowType rowType() { + return rowType; + } + + /** Returns the value conversion type. */ + public ValueType valueType() { + return valueType; + } + + /** Returns the JSON layout, or null for binary/string. */ + public @Nullable ValueLayout valueLayout() { + return valueLayout; + } + + /** Returns physical value targets; nested JSON projects its single ROW container. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the key column position, or -1 when keys are not stored. */ + public int keyPosition() { + return keyPosition; + } + + /** + * Returns the headers column position, or -1 when headers are not stored. Decoding maps names + * to binary values, with the last occurrence winning for duplicate header names. + */ + public int headersPosition() { + return headersPosition; + } + + /** + * Returns the metadata column position, or -1 when absent. The ROW stores the input record + * timestamp, batch timestamp type and requested partition; log offsets are not row fields. + */ + public int metadataPosition() { + return metadataPosition; + } + + /** + * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and + * reserved members here; binary/string decoders write null to this optional column. + */ + public int rescuePosition() { + return rescuePosition; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof KafkaTableSchema)) { + return false; + } + KafkaTableSchema that = (KafkaTableSchema) other; + return rowType.equals(that.rowType) + && valueType == that.valueType + && valueLayout == that.valueLayout + && valueProjection.equals(that.valueProjection) + && keyPosition == that.keyPosition + && headersPosition == that.headersPosition + && metadataPosition == that.metadataPosition + && rescuePosition == that.rescuePosition; + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + valueType, + valueLayout, + valueProjection, + keyPosition, + headersPosition, + metadataPosition, + rescuePosition); + } + + private static InvalidTableException invalid(String message) { + return new InvalidTableException(message); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java new file mode 100644 index 00000000000..458242feadd --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the shared DDL and Kafka conversion contract. */ +class KafkaTableSchemaTest { + + @Test + void testExplicitOptInAndDisabledTables() { + TableDescriptor disabled = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + assertThat(KafkaTableSchema.isEnabled(disabled)).isFalse(); + assertThatCode(() -> KafkaTableSchema.validate(disabled)).doesNotThrowAnyException(); + assertThatThrownBy(() -> KafkaTableSchema.resolve(disabled)) + .hasMessageContaining("disabled"); + TableDescriptor arbitrary = + TableDescriptor.builder(disabled) + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .primaryKey("id") + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "future-format") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + assertThatCode(() -> KafkaTableSchema.validate(arbitrary)).doesNotThrowAnyException(); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testFixedValueColumnWithOptionalSystemFields(String format) { + DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); + for (int mask = 0; mask < 16; mask++) { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", valueType.copy(false)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 2) != 0) { + schema.column( + "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType()); + } + TableDescriptor table = enabled(schema.build(), format).build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueProjection().positions()).containsExactly(0); + assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); + assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); + assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); + assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); + assertThat(table.getProperties()) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + } + } + + @Test + void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { + RowType nested = DataTypes.ROW(DataTypes.FIELD("_kafka_key", DataTypes.STRING())); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) + .column("payload", nested) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); + assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); + assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(resolved.metadataPosition()).isEqualTo(2); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + } + + @Test + void testNestedJsonTargetsOneRow() { + RowType row = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()), + DataTypes.FIELD( + "details", + DataTypes.ROW( + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) + .column("_kafka_value", row) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); + assertThat(resolved.valueProjection().positions()).containsExactly(2); + assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); + assertThat(resolved.rescuePosition()).isZero(); + assertThat(resolved.metadataPosition()).isEqualTo(1); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonRejectsNonStringMapKeysRecursively(String layout) { + for (DataType key : Arrays.asList(DataTypes.INT(), DataTypes.CHAR(10), DataTypes.BYTES())) { + DataType invalidMap = DataTypes.MAP(key, DataTypes.STRING()); + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TableDescriptor table = jsonTable(layout, type); + assertInvalid(table, "only supports STRING map keys"); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { + for (DataType scalar : + Arrays.asList( + DataTypes.CHAR(10), + DataTypes.STRING(), + DataTypes.BOOLEAN(), + DataTypes.BINARY(4), + DataTypes.BYTES(), + DataTypes.DECIMAL(18, 2), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.DATE(), + DataTypes.TIME(3), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(9))) { + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING().copy(false), + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", scalar.copy(false))))); + TableDescriptor table = jsonTable(layout, type); + assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonBusinessNestingDepthBoundary(String layout) { + // Exercise every recursive edge, including map values (keys do not add a level). + DataType boundary = DataTypes.STRING(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + if (depth % 3 == 0) { + boundary = DataTypes.ARRAY(boundary); + } else if (depth % 3 == 1) { + boundary = DataTypes.MAP(DataTypes.STRING(), boundary); + } else { + boundary = DataTypes.ROW(DataTypes.FIELD("child", boundary)); + } + } + TableDescriptor valid = jsonTable(layout, boundary); + assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); + assertInvalid(tooDeep, "maximum nesting depth of 64"); + } + + private static TableDescriptor jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + return enabled(schema, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .build(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_value", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue" + }) + void testReservedNamesAtValueMappingLevel(String name) { + assertThat(KafkaTableSchema.isReservedName(name)).isTrue(); + TableDescriptor nested = + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD(name, DataTypes.STRING()))) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + assertInvalid(nested, "Reserved name"); + } + + @ParameterizedTest + @ValueSource(strings = {"raw", "avro", ""}) + void testRejectsUnsupportedValueTypes(String type) { + assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); + } + + @Test + void testRejectsMissingValueTypeAndTargetConflicts() { + assertInvalid( + TableDescriptor.builder() + .schema(valueSchema()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(), + "Missing required"); + assertInvalid( + enabled(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "binary") + .build(), + "Missing required value column"); + assertInvalid(enabled(valueSchema(), "string").build(), "must be STRING"); + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "must be ROW"); + assertInvalid(enabled(valueSchema(), "parse_json").build(), "flat JSON"); + assertInvalid( + enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unused", DataTypes.STRING()) + .build(), + "binary") + .build(), + "no value mapping"); + assertInvalid( + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()))) + .column("unused", DataTypes.STRING()) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "no value mapping"); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNonJsonRejectsExplicitLayout(String type) { + assertInvalid( + enabled(valueSchema(), type) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "flat") + .build(), + "only supported for parse_json"); + } + + @Test + void testInvalidLayoutAndKeyConversion() { + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "unknown") + .build(), + "value.layout"); + assertInvalid( + enabled(valueSchema(), "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .build(), + "requires _kafka_key"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build(); + assertInvalid( + enabled(schema, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build(), + "only supports binary"); + assertThat( + KafkaTableSchema.resolve( + enabled(schema, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + "binary") + .build()) + .keyPosition()) + .isEqualTo(1); + } + + @Test + void testSystemColumnTypes() { + for (DataType invalid : Arrays.asList(DataTypes.STRING(), DataTypes.BINARY(8))) { + assertInvalid(withSystem("_kafka_key", invalid), "_kafka_key must be BYTES"); + } + for (DataType invalid : + Arrays.asList( + DataTypes.BYTES(), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.MAP(DataTypes.INT(), DataTypes.BYTES()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))) { + assertInvalid(withSystem("_kafka_headers", invalid), "MAP"); + } + assertInvalid(withSystem("_kafka_rescue", DataTypes.INT()), "nullable STRING"); + assertInvalid( + withSystem("_kafka_rescue", DataTypes.STRING().copy(false)), "nullable STRING"); + } + + @Test + void testMetadataPreservesDeclaredNullability() { + RowType metadata = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + for (boolean nullable : new boolean[] {true, false}) { + DataType type = metadata.copy(nullable); + TableDescriptor table = withSystem("_kafka_metadata", type); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); + } + } + + @Test + void testMetadataRejectsInvalidStructureAndOffsetField() { + for (DataType invalid : + Arrays.asList( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(6)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.STRING()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.BIGINT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("bucket", DataTypes.INT())))) { + assertInvalid( + withSystem("_kafka_metadata", invalid), + "ROW"); + } + } + + @Test + void testUnknownAndPrototypeOptions() { + for (String key : + Arrays.asList("table.kafka.unknown", "table.kafka.convert.rescue-column")) { + assertInvalid( + enabled(valueSchema(), "binary").property(key, "anything").build(), + "Unsupported Kafka table property"); + } + for (String key : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "kafka.metadata.headers.column", + "fluss.value.format", + "table.kafka.convert.value.type")) { + assertInvalid( + enabled(valueSchema(), "binary").customProperty(key, "raw").build(), + "Unsupported Kafka custom property"); + } + assertThat( + KafkaTableSchema.resolve( + enabled(valueSchema(), "binary") + .customProperty("owner", "team") + .build())) + .isNotNull(); + } + + @Test + void testUnsupportedTableKinds() { + Schema keyed = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") + .build(); + assertInvalid(enabled(keyed, "binary").build(), "without a primary key"); + assertInvalid( + enabled(valueSchema(), "binary").partitionedBy("_kafka_value").build(), + "non-partitioned"); + assertInvalid( + enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); + } + + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + + private static TableDescriptor withSystem(String name, DataType type) { + return enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column(name, type) + .build(), + "binary") + .build(); + } + + private static Schema valueSchema() { + return Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(); + } + + private static TableDescriptor.Builder enabled(Schema schema, String type) { + return TableDescriptor.builder() + .schema(schema) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, type); + } + + private static void assertInvalid(TableDescriptor table, String message) { + assertThatThrownBy(() -> KafkaTableSchema.resolve(table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining(message); + } +} diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index f48a94ab9e9..e38487db564 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,19 @@ + + ${project.groupId} + fluss-client + ${project.version} + test + + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2bd2f5d81bc..07bdbbfe86b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,19 +19,24 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkState; /** The Kafka protocol plugin. */ public class KafkaProtocolPlugin implements NetworkProtocolPlugin { private Configuration conf; + private KafkaProduceConversionExecutor conversionExecutor; + private CompletableFuture closeFuture; @Override public String name() { @@ -40,7 +45,7 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = conf; + this.conf = new Configuration(conf); } @Override @@ -60,13 +65,35 @@ public ChannelHandler createChannelHandler( } @Override - public RequestHandler createRequestHandler(RpcGatewayService service) { - if (!(service instanceof TabletServerGateway)) { - throw new IllegalArgumentException( - "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " - + service.getClass().getSimpleName()); + public synchronized RequestHandler createRequestHandler(RpcGatewayService service) { + checkState(closeFuture == null, "Kafka protocol plugin has already been closed."); + if (conversionExecutor == null) { + // Mirror the RPC worker concurrency, with a separate queue capped at 1024 requests. + int queueCapacity = + Math.max( + 1, + Math.min( + 1024, + conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_REQUESTS))); + int threads = + Math.max( + 1, + Math.min( + queueCapacity, + conf.get(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS))); + conversionExecutor = new KafkaProduceConversionExecutor(threads, queueCapacity); + } + return new KafkaRequestHandler(conf, service, conversionExecutor); + } + + @Override + public synchronized CompletableFuture closeAsync() { + if (closeFuture == null) { + closeFuture = + conversionExecutor == null + ? CompletableFuture.completedFuture(null) + : conversionExecutor.closeAsync(); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return closeFuture; } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..dc34e6aaaca 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,61 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; +import javax.annotation.Nullable; -/** Kafka protocol implementation for request handler. */ +import java.util.Arrays; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaApiRegistry registry; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { + this(conf, service, null); + } + + /** Creates a handler with a plugin-owned executor for blocking Produce work. */ + public KafkaRequestHandler( + Configuration conf, + RpcGatewayService service, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { + KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); + this.registry = + new KafkaApiRegistry( + Arrays.asList( + new ApiVersionsHandler(this::apiRegistry), + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, topicMapper), + topicMapper), + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + (TabletServerGateway) service, + topicMapper, + new ArrowKafkaRecordTranscoder(), + conversionExecutor)))); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +81,19 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); + private KafkaApiRegistry apiRegistry() { + return registry; } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); - } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..bffc28d6de2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + private final KafkaTopicMapper topicMapper; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend, KafkaTopicMapper topicMapper) { + this.backend = checkNotNull(backend); + this.topicMapper = checkNotNull(topicMapper); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!isValidTopic(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private boolean isValidTopic(String name) { + try { + // Validate the resolved path too: a bare name's canonical form includes the configured + // default database and must fit the same Kafka name limit as an explicit name. + topicMapper.toTablePath(name); + return true; + } catch (InvalidTopicException e) { + return false; + } + } + + private MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + // The backend retains the shared table identity. Distinguish names + // only on the wire so clients can cache both and detect recreation. + .setTopicId( + topic.name() != null + && topic.name().indexOf('.') < 0 + && topicMapper.isMappedTopicId(topic.topicId()) + ? topicMapper.toAliasTopicId( + topicMapper.toTableId(topic.topicId())) + : topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..c485e618152 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.CloseableIterator; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Implements non-idempotent Kafka Produce versions 3 through 11. + * + *

Topic names cross the backend boundary unchanged. The backend resolves bare and qualified + * names and returns results keyed by the original request names. + */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + // Validate the incoming Kafka batch before copying away its offset metadata. Fluss + // assigns its own storage offsets later, independently of this input validation. + long countFromOffsets = batch.lastOffset() - batch.baseOffset() + 1; + Integer recordCount = batch.countOrNull(); + if (countFromOffsets <= 0 + || recordCount == null + || recordCount <= 0 + || countFromOffsets != recordCount) { + throw new InvalidRecordException( + "Invalid Kafka record batch offset range [" + + batch.baseOffset() + + ", " + + batch.lastOffset() + + "] for record count " + + recordCount + + "."); + } + TimestampType timestampType = timestampType(batch); + // The ordinary compressed iterator preallocates a list using the untrusted record + // count. Stream records instead and close the decompressor on success and failure. + try (CloseableIterator iterator = + batch.streamingIterator(BufferSupplier.NO_CACHING)) { + while (iterator.hasNext()) { + org.apache.kafka.common.record.Record record = iterator.next(); + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + timestampType, + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static TimestampType timestampType(RecordBatch batch) { + switch (batch.timestampType()) { + case CREATE_TIME: + return TimestampType.CREATE_TIME; + case LOG_APPEND_TIME: + return TimestampType.LOG_APPEND_TIME; + case NO_TIMESTAMP_TYPE: + return TimestampType.UNKNOWN; + default: + throw new InvalidRecordException("Unknown batch timestamp type."); + } + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..5cac0c3cca5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final Supplier registrySupplier; + + /** + * Creates an ApiVersions handler backed by the same immutable registry used for dispatch. The + * supplier is read only when handling requests, so this handler can be included in the + * registry's construction snapshot. + */ + public ApiVersionsHandler(Supplier registrySupplier) { + this.registrySupplier = checkNotNull(registrySupplier); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + KafkaApiRegistry registry = + checkNotNull(registrySupplier.get(), "Kafka API registry is not initialized."); + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..2c2ff42e117 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + + /** + * Creates a metadata backend backed by the local TabletServer gateway. + * + * @throws IllegalArgumentException if the service does not implement {@link + * TabletServerGateway} + */ + public GatewayKafkaMetadataBackend(RpcGatewayService service, KafkaTopicMapper topicMapper) { + this.service = checkNotNull(service); + checkArgument( + service instanceof TabletServerGateway, + "Kafka protocol endpoints can only be enabled on TabletServers, but the service is %s", + service.getClass().getSimpleName()); + this.gateway = (TabletServerGateway) service; + this.topicMapper = checkNotNull(topicMapper); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + return currentTablePaths(query) + .thenCompose(tablePaths -> requestFlussMetadata(query, tablePaths)); + } + + Set tablePaths = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + tablePaths.add(topicMapper.toTablePath(topic.topicName())); + } + } + return requestFlussMetadata(query, tablePaths); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set tablePaths) { + return requestFlussMetadata(query, tablePaths, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set tablePaths, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (TablePath tablePath : tablePaths) { + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTablePaths(query) + .thenCompose(currentPaths -> requestFlussMetadata(query, currentPaths, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTablePaths(KafkaMetadataQuery query) { + CompletableFuture> databasesFuture; + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + databasesFuture = + gateway.listDatabases(new ListDatabasesRequest()) + .thenApply( + response -> + new LinkedHashSet<>(response.getDatabaseNamesList())); + } else { + Set databases = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + databases.add(topicMapper.toTablePath(topic.topicName()).getDatabaseName()); + } + } + databasesFuture = CompletableFuture.completedFuture(databases); + } + return databasesFuture + .thenCompose(databases -> listTablePaths(query, databases)) + .thenApply( + currentPaths -> { + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedTables = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedTables.add( + topicMapper.toTablePath(topic.topicName())); + } + } + // Canonical names and aliases share one table identity, including + // when a missing database triggers this discovery retry. + currentPaths.retainAll(requestedTables); + } + return currentPaths; + }); + } + + private CompletableFuture> listTablePaths( + KafkaMetadataQuery query, Set databases) { + CompletableFuture> tablesFuture = + CompletableFuture.completedFuture(new LinkedHashSet<>()); + for (String database : databases) { + tablesFuture = + tablesFuture.thenCompose( + tablePaths -> + listTables(query, database) + .thenApply( + response -> { + for (String tableName : + response.getTableNamesList()) { + tablePaths.add( + TablePath.of( + database, tableName)); + } + return tablePaths; + })); + } + return tablesFuture; + } + + private CompletableFuture listTables( + KafkaMetadataQuery query, String database) { + setCurrentSession(query); + CompletableFuture tablesFuture; + try { + tablesFuture = gateway.listTables(new ListTablesRequest().setDatabaseName(database)); + } catch (Exception failure) { + tablesFuture = FutureUtils.completedExceptionally(failure); + } + return tablesFuture.exceptionally( + failure -> { + Throwable cause = unwrap(failure); + if (cause instanceof DatabaseNotExistException) { + // A database may be missing or deleted after listDatabases. Keep loading + // the remaining topics and brokers through the normal metadata request. + return new ListTablesResponse(); + } + throw new CompletionException(cause); + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByTable = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + TablePath tablePath = + TablePath.of( + table.getTablePath().getDatabaseName(), + table.getTablePath().getTableName()); + Topic topic = toKafkaTopic(table, tablePath, aliveBrokerIds); + if (topic != null) { + topicsByTable.put(tablePath, topic); + topicsById.put(topic.topicId(), topic); + } + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + // Invalid enabled tables remain visible as errors. Only disabled tables are omitted. + topics.addAll(topicsByTable.values()); + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByTable.get(topicMapper.toTablePath(reference.topicName())); + if (topic != null && matches(reference, topic)) { + topics.add( + new Topic( + reference.topicName() == null + ? topic.name() + : reference.topicName(), + topic.topicId(), + topic.error(), + topic.partitions())); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + @Nullable + private Topic toKafkaTopic( + PbTableMetadata table, TablePath tablePath, Set aliveBrokerIds) { + String topicName = tablePath.toString(); + try { + TableDescriptor descriptor = TableDescriptor.fromJsonBytes(table.getTableJson()); + if (!schemaResolver.isEnabled(descriptor)) { + return null; + } + topicMapper.toTopicName(tablePath); + schemaResolver.resolve(descriptor); + } catch (IllegalArgumentException | InvalidTopicException e) { + LOG.debug( + "Enabled table {} has an invalid Kafka contract: {}", + topicName, + e.getMessage()); + return new Topic( + topicName, + topicMapper.toTopicId(table.getTableId()), + TopicError.INVALID_TOPIC, + Collections.emptyList()); + } + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.NONE, partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null + || topicMapper + .toTablePath(reference.topicName()) + .equals(topicMapper.toTablePath(topic.name()))) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..c5dd9638dea --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final KafkaRecordTranscoder transcoder; + private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); + private final @Nullable KafkaProduceConversionExecutor conversionExecutor; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaTopicMapper topicMapper, + KafkaRecordTranscoder transcoder) { + this(service, gateway, topicMapper, transcoder, null); + } + + /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaTopicMapper topicMapper, + KafkaRecordTranscoder transcoder, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); + this.transcoder = checkNotNull(transcoder); + this.conversionExecutor = conversionExecutor; + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + if (conversionExecutor != null) { + return conversionExecutor + .submit(Thread.currentThread().getId(), () -> writeTopics(command)) + .exceptionally( + failure -> { + List results = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + results.add(failedTopic(topic, failure)); + } + return new KafkaProduceResult(results); + }); + } + return writeTopics(command); + } + + private CompletableFuture writeTopics(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + try { + checkInterrupted(); + TablePath path = topicMapper.toTablePath(topic.topicName()); + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath() + .setDatabaseName(path.getDatabaseName()) + .setTableName(path.getTableName()); + CompletableFuture metadata = gateway.getTableInfo(request); + if (conversionExecutor != null) { + // The local gateway is synchronous. Waiting here also preserves FIFO append + // order if an alternative gateway returns asynchronously; only this worker waits. + GetTableInfoResponse response = + metadata.get(Math.max(0, command.timeoutMs()), TimeUnit.MILLISECONDS); + return produceTopic(command, topic, toTableInfo(path, response)); + } + return metadata.thenCompose( + response -> produceTopic(command, topic, toTableInfo(path, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + // Admission must match Metadata, including when the schema changes between requests. + // Even a cached snapshot is selected only after fetching the authoritative descriptor. + if (!schemaResolver.isEnabled(tableInfo.toTableDescriptor())) { + throw new TableNotExistException( + "Table " + tableInfo.getTablePath() + " is not enabled for Kafka."); + } + KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + Map failures = new HashMap<>(); + for (PartitionWrite partition : topic.partitions()) { + if (partition.partitionId() < 0 + || partition.partitionId() >= tableInfo.getNumBuckets()) { + failures.put( + partition.partitionId(), + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_TOPIC_OR_PARTITION, + -1L, + "Partition is outside the table bucket range.")); + continue; + } + try { + BytesView records = + transcoder.transcode( + partition.partitionId(), partition.records(), writePlan); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } catch (Exception failure) { + failures.put( + partition.partitionId(), failedPartition(partition.partitionId(), failure)); + } + } + if (request.getBucketsReqsCount() == 0) { + return CompletableFuture.completedFuture( + toTopicResult(topic, new ProduceLogResponse(), failures)); + } + try { + checkInterrupted(); + setCurrentSession(command); + CompletableFuture appended = gateway.produceLog(request); + return appended.handle( + (response, failure) -> + failure == null + ? toTopicResult(topic, response, failures) + : failedAppend(topic, failures, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedAppend(topic, failures, failure)); + } + } + + private static TopicResult failedAppend( + TopicWrite topic, Map failures, Throwable failure) { + List results = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult local = failures.get(partition.partitionId()); + results.add(local == null ? failedPartition(partition.partitionId(), failure) : local); + } + return new TopicResult(topic.topicName(), results); + } + + private TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return tableInfoCache.getOrLoad(path, response, () -> parseTableInfo(path, response)); + } + + private static TableInfo parseTableInfo(TablePath path, GetTableInfoResponse response) { + return TableInfo.of( + path, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static TopicResult toTopicResult( + TopicWrite topic, ProduceLogResponse response, Map failures) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult localFailure = failures.get(partition.partitionId()); + if (localFailure != null) { + partitions.add(localFailure); + continue; + } + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode() && bucket.getErrorCode() != 0) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add(failedPartition(partition.partitionId(), failure)); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof RejectedExecutionException + || cause instanceof TimeoutException + || cause instanceof InterruptedException + || cause instanceof CancellationException + ? Errors.REQUEST_TIMED_OUT + : cause instanceof KafkaTopicSchemaException + || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.INVALID_RECORD + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors + .forException(cause)); + return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case DATABASE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } + + private void checkInterrupted() { + if (conversionExecutor != null && Thread.currentThread().isInterrupted()) { + throw new CancellationException("Kafka conversion worker was interrupted."); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..b0d83392d39 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the original request topic name, including any default-database alias. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order, including records with null values. */ + public List records() { + return records; + } + } + + /** + * Timestamp provenance supplied by a batch, independent of whether a record has a timestamp. + */ + @Internal + public enum TimestampType { + /** No batch timestamp type was supplied, including by the legacy record constructor. */ + UNKNOWN, + /** The timestamp was assigned when the record was created. */ + CREATE_TIME, + /** The timestamp was assigned when the batch was appended to a log. */ + LOG_APPEND_TIME + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + /** Sentinel used when the input record has no timestamp. */ + public static final long NO_TIMESTAMP = -1L; + + private final long timestamp; + private final TimestampType timestampType; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** + * Creates a copied record without batch timestamp provenance. + * + *

The type is {@link TimestampType#UNKNOWN} even when a timestamp is present. Callers + * that know the batch type should use the explicit constructor instead of inferring it from + * the timestamp value. + */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this(timestamp, TimestampType.UNKNOWN, key, value, headers); + } + + /** Creates a copied record with the timestamp type supplied by its source batch. */ + public Record( + long timestamp, + TimestampType timestampType, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.timestampType = checkNotNull(timestampType); + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the record timestamp in milliseconds, or {@link #NO_TIMESTAMP} when absent. */ + public long timestamp() { + return timestamp; + } + + /** Returns whether the input record has a timestamp, independently of its batch type. */ + public boolean hasTimestamp() { + return timestamp != NO_TIMESTAMP; + } + + /** Returns the source batch's timestamp type, never inferred from the timestamp value. */ + public TimestampType timestampType() { + return timestampType; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** + * Borrows the nullable key owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #key()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedKey() { + return key; + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** + * Borrows the nullable value owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + + /** + * Borrows the nullable header value for synchronous conversion. + * + *

The returned array is owned by this header and must not be modified or retained after + * conversion. Use {@link #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java new file mode 100644 index 00000000000..e3953f001e0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; + +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Dedicated, bounded FIFO lanes for Kafka metadata lookup, conversion, and native submission. + * + *

The caller uses its RPC processor identity as the ordering key. The server routes each + * connection to one processor, so its requests remain ordered when moved off that processor. An + * operation must invoke native append before returning, but need not wait for acknowledgements. + */ +@Internal +public final class KafkaProduceConversionExecutor { + private final ThreadPoolExecutor[] workers; + private volatile boolean closed; + private CompletableFuture closeFuture; + + /** Creates fixed single-worker lanes whose combined queue capacity is bounded. */ + public KafkaProduceConversionExecutor(int threads, int queueCapacity) { + checkArgument(threads > 0, "Conversion thread count must be positive."); + checkArgument( + queueCapacity >= threads, "Queue capacity must be at least the thread count."); + workers = new ThreadPoolExecutor[threads]; + for (int i = 0; i < threads; i++) { + int capacity = queueCapacity / threads + (i < queueCapacity % threads ? 1 : 0); + workers[i] = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(capacity), + new ExecutorThreadFactory("kafka-produce-conversion-" + i), + new ThreadPoolExecutor.AbortPolicy()); + } + } + + /** Submits ordered work without ever running rejected work on the caller's thread. */ + public CompletableFuture submit( + long orderingKey, Supplier> operation) { + ConversionTask task = new ConversionTask<>(checkNotNull(operation)); + try { + if (closed) { + throw new RejectedExecutionException("Kafka conversion executor is closed."); + } + workers[(int) Math.floorMod(orderingKey, (long) workers.length)].execute(task); + } catch (RejectedExecutionException failure) { + task.cancel(failure); + } + return task.result; + } + + /** + * Stops new work, rejects queued requests, interrupts workers, and awaits their termination. + */ + public synchronized CompletableFuture closeAsync() { + if (closeFuture != null) { + return closeFuture; + } + closed = true; + RejectedExecutionException failure = + new RejectedExecutionException("Kafka conversion executor is shutting down."); + for (ThreadPoolExecutor worker : workers) { + List queued = worker.shutdownNow(); + for (Runnable task : queued) { + ((ConversionTask) task).cancel(failure); + } + } + closeFuture = + CompletableFuture.runAsync( + () -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + try { + for (ThreadPoolExecutor worker : workers) { + if (!worker.awaitTermination( + Math.max(0L, deadline - System.nanoTime()), + TimeUnit.NANOSECONDS)) { + throw new IllegalStateException( + "Kafka conversion worker did not terminate."); + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted closing Kafka conversion workers.", + interrupted); + } + }); + return closeFuture; + } + + private static final class ConversionTask implements Runnable { + private final AtomicReference>> operation; + private final CompletableFuture result = new CompletableFuture<>(); + private final AtomicBoolean claimed = new AtomicBoolean(); + + private ConversionTask(Supplier> operation) { + this.operation = new AtomicReference<>(operation); + } + + @Override + public void run() { + if (!claimed.compareAndSet(false, true)) { + return; + } + Supplier> action = operation.getAndSet(null); + if (result.isDone()) { + return; + } + try { + checkNotNull(action.get(), "conversion result") + .whenComplete( + (value, failure) -> { + if (failure == null) { + result.complete(value); + } else { + result.completeExceptionally(failure); + } + }); + } catch (Throwable failure) { + result.completeExceptionally(failure); + } + } + + private void cancel(Throwable failure) { + if (claimed.compareAndSet(false, true)) { + operation.set(null); + result.completeExceptionally(failure); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..8f41864f85b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the original request topic name, rather than a resolved table path. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java new file mode 100644 index 00000000000..1d53a5eed8a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded cache for parsed Kafka topic {@link TableInfo} snapshots. + * + *

The cache never replaces the authoritative {@code GetTableInfo} call. Every Produce request + * still obtains a fresh response (and therefore retains its authorization and freshness semantics). + * This cache only avoids parsing an unchanged descriptor JSON response again. + * + *

An entry is reused only when all response identity fields and the complete descriptor bytes + * match. Comparing the complete descriptor is intentional: a {@code kafka.*} property change may + * alter decoding semantics without changing the schema ID. Cache slots serialize parsing for one + * table, while independent topics remain concurrent. + */ +@Internal +@ThreadSafe +final class KafkaTableInfoCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaTableInfoCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaTableInfoCache(long maximumSize) { + checkArgument(maximumSize > 0, "Kafka TableInfo cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + TableInfo getOrLoad( + TablePath tablePath, + GetTableInfoResponse response, + Supplier tableInfoLoader) { + checkNotNull(tablePath); + checkNotNull(response); + checkNotNull(tableInfoLoader); + + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.entry != null && slot.entry.matches(response)) { + return slot.entry.tableInfo; + } + TableInfo tableInfo = checkNotNull(tableInfoLoader.get()); + slot.entry = new CacheEntry(response, tableInfo); + return tableInfo; + } + } + } + + void invalidate(TablePath tablePath) { + CacheSlot slot = slots.get(checkNotNull(tablePath)); + if (slot == null) { + return; + } + synchronized (slot) { + slot.entry = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private CacheEntry entry; + } + + private static final class CacheEntry { + private final long tableId; + private final int schemaId; + private final byte[] tableJson; + private final long createdTime; + private final long modifiedTime; + private final long bucketCountEpoch; + private final String remoteDataDir; + private final TableInfo tableInfo; + + private CacheEntry(GetTableInfoResponse response, TableInfo tableInfo) { + this.tableId = response.getTableId(); + this.schemaId = response.getSchemaId(); + this.tableJson = response.getTableJson().clone(); + this.createdTime = response.getCreatedTime(); + this.modifiedTime = response.getModifiedTime(); + this.bucketCountEpoch = + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L; + this.remoteDataDir = response.hasRemoteDataDir() ? response.getRemoteDataDir() : null; + this.tableInfo = tableInfo; + } + + private boolean matches(GetTableInfoResponse response) { + return tableId == response.getTableId() + && schemaId == response.getSchemaId() + && createdTime == response.getCreatedTime() + && modifiedTime == response.getModifiedTime() + && bucketCountEpoch + == (response.hasBucketCountEpoch() + ? response.getBucketCountEpoch() + : 0L) + && Objects.equals( + remoteDataDir, + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null) + && Arrays.equals(tableJson, response.getTableJson()); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java new file mode 100644 index 00000000000..bce333819de --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.BytesType; + +/** Factory for the Kafka binary byte format. */ +@Internal +public final class BinaryKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public ValueType format() { + return ValueType.BINARY; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof BytesType)) { + throw new KafkaTopicSchemaException( + "Kafka binary format requires exactly one _kafka_value BYTES field."); + } + return bytes -> new Object[] {bytes}; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java new file mode 100644 index 00000000000..b70c85f17d4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +/** Decodes one nullable Kafka record value into its ordered Fluss field projection. */ +@Internal +public interface KafkaFieldDecoder { + + /** + * Decodes Kafka value bytes into internal Fluss field values. + * + *

The input is borrowed and must not be modified. Returned values may borrow its storage; + * callers must encode them synchronously before the input can be reused. + */ + Object[] decode(@Nullable byte[] bytes); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java new file mode 100644 index 00000000000..71c5db5f6ad --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; + +/** Creates a decoder for one Kafka value format and Fluss field projection. */ +@Internal +public interface KafkaFormatFactory { + + /** Returns the format implemented by this factory. */ + ValueType format(); + + /** Creates and validates a decoder for the projection. */ + KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java new file mode 100644 index 00000000000..3ac61403a70 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; + +import java.util.EnumMap; +import java.util.Map; + +/** Registry of built-in Kafka record format factories. */ +@Internal +public final class KafkaFormatFactoryRegistry { + + private final Map factories; + + /** Creates a registry containing all built-in formats. */ + public KafkaFormatFactoryRegistry() { + factories = new EnumMap<>(ValueType.class); + register(new BinaryKafkaFormatFactory()); + register(new StringKafkaFormatFactory()); + } + + /** Creates a decoder for the requested format and projection. */ + public KafkaFieldDecoder createDecoder(ValueType format, KafkaFieldProjection projection) { + KafkaFormatFactory factory = factories.get(format); + if (factory == null) { + throw new KafkaTopicSchemaException( + "No Kafka format factory is registered for '" + format + "'."); + } + return factory.createDecoder(projection); + } + + private void register(KafkaFormatFactory factory) { + factories.put(factory.format(), factory); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java new file mode 100644 index 00000000000..c004c968e3a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.StringType; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Factory for strict UTF-8 Kafka string decoding. */ +@Internal +public final class StringKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public ValueType format() { + return ValueType.STRING; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof StringType)) { + throw new KafkaTopicSchemaException( + "Kafka string format requires exactly one _kafka_value STRING field."); + } + final String fieldName = projection.nameAt(0); + return bytes -> new Object[] {decode(bytes, fieldName)}; + } + + private static BinaryString decode(byte[] bytes, String fieldName) { + if (bytes == null) { + return null; + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException( + "Kafka field '" + fieldName + "' is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..fbd54ef77e4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Maps canonical Kafka topic names and default-database aliases to Fluss tables. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + // ASCII "FlussA": Kafka clients require distinct IDs for distinct wire topic names. + private static final long ALIAS_TOPIC_ID_NAMESPACE = 0x466c757373410000L; + + private final String defaultDatabase; + + /** Creates a mapper using the default database configuration. */ + public KafkaTopicMapper() { + this(new Configuration()); + } + + /** Creates a mapper using the server's default database; it never creates databases. */ + public KafkaTopicMapper(Configuration config) { + this.defaultDatabase = config.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + TablePath.validateDatabaseName(defaultDatabase); + } + + /** Maps a canonical topic or an unqualified alias to a table path, without fallback lookup. */ + public TablePath toTablePath(String topicName) { + if (!isValidTopic(topicName)) { + throw new InvalidTopicException( + "Kafka topic must be a valid table or database.table name: " + topicName); + } + int separator = topicName.indexOf('.'); + TablePath path = + separator < 0 + ? TablePath.of(defaultDatabase, topicName) + : TablePath.of( + topicName.substring(0, separator), + topicName.substring(separator + 1)); + if (!isMappedTable(path)) { + throw new InvalidTopicException("Canonical Kafka topic name is invalid: " + path); + } + return path; + } + + /** Returns the fully qualified Kafka name of a representable Fluss table. */ + public String toTopicName(TablePath tablePath) { + if (!isMappedTable(tablePath)) { + throw new InvalidTopicException( + "Fluss table cannot be represented as a Kafka database.table name: " + + tablePath); + } + return tablePath.toString(); + } + + /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return KafkaTableSchema.isValidTablePath(tablePath); + } + + /** Returns whether a name uniquely represents a user table in a Fluss database. */ + public static boolean isValidTopic(String topicName) { + if (topicName == null || !Topic.isValid(topicName)) { + return false; + } + int separator = topicName.indexOf('.'); + if (separator < 0) { + return TablePath.detectInvalidName(topicName) == null + && TablePath.validatePrefix(topicName) == null; + } + if (separator == 0 || separator != topicName.lastIndexOf('.')) { + return false; + } + return KafkaTableSchema.isValidTablePath( + TablePath.of( + topicName.substring(0, separator), topicName.substring(separator + 1))); + } + + /** Maps a Fluss table ID to the canonical name's stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** + * Maps a Fluss table ID to the default-database alias's stable Kafka topic ID. + * + *

The wire ID must differ from the canonical name's ID because Kafka clients maintain a + * one-to-one ID-to-name map. It must also change on recreation so a client can accept a new + * table's lower leader epoch without first observing a missing-topic response. + */ + public Uuid toAliasTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(ALIAS_TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && (topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + || topicId.getMostSignificantBits() == ALIAS_TOPIC_ID_NAMESPACE) + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..c02879e922a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableDescriptor; + +/** Resolves the same persisted Kafka contract that native DDL validates. */ +@Internal +public final class KafkaTopicSchemaResolver { + + /** Returns the table's explicit opt-in flag without inferring it from schema compatibility. */ + public boolean isEnabled(TableDescriptor table) { + try { + return KafkaTableSchema.isEnabled(table); + } catch (IllegalArgumentException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } + } + + /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ + public KafkaTableSchema resolve(TableDescriptor table) { + try { + return KafkaTableSchema.resolve(table); + } catch (InvalidTableException | IllegalArgumentException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..767095b5902 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return writePlanCache.getOrCompile( + tableInfo, + () -> + new KafkaTopicWritePlan( + tableInfo, schemaResolver.resolve(tableInfo.toTableDescriptor()))); + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception { + return transcode(partitionId, records, prepare(tableInfo)); + } + + @Override + public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTableSchema schema = plan.topicSchema(); + KafkaFieldDecoder valueDecoder = + formats.createDecoder(schema.valueType(), schema.valueProjection()); + KafkaRowAssembler assembler = plan.rowAssembler(); + return arrowRecordEncoder.encodeStreaming( + consumer -> { + for (Record record : records) { + consumer.append( + assembler.assemble( + valueDecoder.decode(record.borrowedValue()), + record, + partitionId)); + } + }, + plan.tableInfo()); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..c84dd20d690 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + return encodeStreaming( + consumer -> { + for (GenericRow row : rows) { + consumer.append(row); + } + }, + tableInfo); + } + + BytesView encodeStreaming(RowSource rowSource, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + rowSource.produce(row -> builder.append(ChangeType.APPEND_ONLY, row)); + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } + + @FunctionalInterface + interface RowSource { + /** Produces rows synchronously and never retains the supplied consumer. */ + void produce(RowConsumer consumer) throws Exception; + } + + @FunctionalInterface + interface RowConsumer { + /** Serializes one row before returning and never retains the row. */ + void append(GenericRow row) throws Exception; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java new file mode 100644 index 00000000000..258e8c557b8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded, single-flight cache for immutable Kafka topic write plans. + * + *

The bounded map is keyed by {@link TablePath}, while each slot fences its plan with the exact + * {@link TableInfo} instance from the authoritative metadata cache. This avoids the recursive and + * allocation-heavy {@link TableInfo#hashCode()} on every Produce request. An equivalent but + * separately constructed {@code TableInfo} is deliberately treated as a cache miss. In the + * production path, {@code KafkaTableInfoCache} returns the same instance only when the complete + * {@code GetTableInfo} response is unchanged. + * + *

Consequently a delete/recreate, schema change, or property-only Kafka contract change receives + * a different instance and cannot reuse an older plan. Slots are synchronized individually so a + * concurrent cold start compiles once per table version. + */ +@Internal +@ThreadSafe +final class KafkaCompiledWritePlanCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaCompiledWritePlanCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaCompiledWritePlanCache(long maximumSize) { + checkArgument( + maximumSize > 0, "Kafka compiled write plan cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + KafkaTopicWritePlan getIfPresent(TableInfo tableInfo) { + checkNotNull(tableInfo); + CacheSlot slot = slots.get(tableInfo.getTablePath()); + if (slot == null) { + return null; + } + synchronized (slot) { + if (slots.get(tableInfo.getTablePath()) != slot) { + return null; + } + return slot.tableInfo == tableInfo ? slot.plan : null; + } + } + + KafkaTopicWritePlan getOrCompile( + TableInfo tableInfo, Supplier planCompiler) { + checkNotNull(tableInfo); + checkNotNull(planCompiler); + + TablePath tablePath = tableInfo.getTablePath(); + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.tableInfo == tableInfo && slot.plan != null) { + return slot.plan; + } + KafkaTopicWritePlan plan = checkNotNull(planCompiler.get()); + slot.tableInfo = tableInfo; + slot.plan = plan; + return plan; + } + } + } + + void invalidate(TableInfo tableInfo) { + checkNotNull(tableInfo); + TablePath tablePath = tableInfo.getTablePath(); + CacheSlot slot = slots.get(tablePath); + if (slot == null) { + return; + } + synchronized (slot) { + if (slot.tableInfo != tableInfo) { + return; + } + slot.tableInfo = null; + slot.plan = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private TableInfo tableInfo; + private KafkaTopicWritePlan plan; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..6c87d80e273 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Validates the table contract and prepares a shareable plan once per topic write. */ + default KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return new KafkaTopicWritePlan( + tableInfo, new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor())); + } + + /** Transcodes one requested partition using an already prepared plan. */ + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + return transcode(partitionId, records, plan.tableInfo()); + } + + /** Transcodes records from the requested Kafka partition using the target table schema. */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..b5cb170b483 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded values and system columns using the shared fixed-column contract. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTableSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTableSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** + * Assembles one Fluss row from a record and its requested partition for immediate synchronous + * encoding. + * + *

The row borrows decoded values, keys and header payloads; it must not modify them or + * retain them after conversion. + */ + public GenericRow assemble(Object[] valueValues, Record record, int partitionId) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + setProjectedFields(row, topicSchema.valueProjection(), valueValues); + if (topicSchema.keyPosition() >= 0) { + int position = topicSchema.keyPosition(); + byte[] key = record.borrowedKey(); + if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka key cannot populate NOT NULL _kafka_key."); + } + row.setField(position, key); + } + if (topicSchema.headersPosition() >= 0) { + int position = topicSchema.headersPosition(); + row.setField( + position, + toHeaders( + record.headers(), (MapType) topicSchema.rowType().getTypeAt(position))); + } + if (topicSchema.metadataPosition() >= 0) { + int position = topicSchema.metadataPosition(); + RowType type = (RowType) topicSchema.rowType().getTypeAt(position); + if (!record.hasTimestamp() && !type.getTypeAt(0).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate NOT NULL _kafka_metadata.timestamp."); + } + row.setField( + position, + GenericRow.of( + record.hasTimestamp() + ? TimestampLtz.fromEpochMillis(record.timestamp()) + : null, + timestampTypeId(record.timestampType()), + partitionId)); + } + // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. + return row; + } + + private static int timestampTypeId(TimestampType type) { + // Preserve batch provenance even when a record's timestamp is absent. The legacy record + // constructor supplies UNKNOWN, so a present timestamp alone cannot imply CreateTime. + switch (type) { + case UNKNOWN: + return -1; + case CREATE_TIME: + return 0; + case LOG_APPEND_TIME: + return 1; + default: + throw new KafkaRecordEncodingException("Unsupported Kafka timestamp type: " + type); + } + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values) { + if (values.length != projection.size()) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields for a projection of " + + projection.size() + + "."); + } + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericMap toHeaders(List headers, MapType type) { + Map values = new LinkedHashMap<>(); + for (RecordHeader header : headers) { + values.put(BinaryString.fromString(header.name()), header.borrowedValue()); + } + // Validate the final map: an overwritten null must not reject a non-null last value. + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null && !type.getKeyType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header name cannot populate NOT NULL _kafka_headers map keys."); + } + if (entry.getValue() == null && !type.getValueType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header value cannot populate NOT NULL _kafka_headers map values."); + } + } + return new GenericMap(values); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java new file mode 100644 index 00000000000..d297713d5ba --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.metadata.TableInfo; + +import javax.annotation.concurrent.ThreadSafe; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. + */ +@Internal +@ThreadSafe +public final class KafkaTopicWritePlan { + private final TableInfo tableInfo; + private final KafkaTableSchema topicSchema; + private final KafkaRowAssembler rowAssembler; + + KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { + this.tableInfo = checkNotNull(tableInfo); + this.topicSchema = checkNotNull(topicSchema); + // Validate supported value formats when compiling the plan. + if (topicSchema.valueType() != ValueType.BINARY + && topicSchema.valueType() != ValueType.STRING) { + throw new KafkaRecordEncodingException( + "Unsupported Kafka value conversion type: " + topicSchema.valueType()); + } + this.rowAssembler = new KafkaRowAssembler(topicSchema); + } + + /** Returns the authoritative table snapshot used to compile this plan. */ + public TableInfo tableInfo() { + return tableInfo; + } + + KafkaTableSchema topicSchema() { + return topicSchema; + } + + KafkaRowAssembler rowAssembler() { + return rowAssembler; + } +} diff --git a/fluss-kafka/src/test/client-matrix/README.md b/fluss-kafka/src/test/client-matrix/README.md new file mode 100644 index 00000000000..3275dc7d843 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/README.md @@ -0,0 +1,109 @@ +# Isolated Kafka producer compatibility matrix + +This test facility runs real Kafka producers against the three-node Fluss test +cluster and reads their records through the native Fluss client. Each selected +client runs in a separate JVM whose classpath contains only a standalone probe, +one `kafka-clients` JAR and that release's runtime libraries. Prepared classpaths +also include a matching SLF4J Simple binding so client network-thread failures +appear in the log. The probe +checks `AppInfoParser.getVersion()` and records its code-source JAR, Java version, +classpath and results. No production dependency is replaced. + +## Requirements and preparation + +Use Java 11 for the reactor and child JVMs, Python 3.8+, curl and Maven. Initial +artifact preparation needs access to Maven Central; subsequent reactor runs can +be offline once the project's own dependencies have been cached. + +The FIP target is every formal Java client release from **0.11.0.0 through 4.3.1**. +[Apache Kafka downloads](https://kafka.apache.org/community/downloads/) records +both boundary releases. The preparation script reads the actual formal versions +from [official Maven metadata](https://repo.maven.apache.org/maven2/org/apache/kafka/kafka-clients/maven-metadata.xml), +fetches each client POM and its runtime JARs, verifies Central SHA-1 sidecars, and +records local SHA-256 hashes. Test dependencies from the published POM are excluded. +It fails explicitly on an unfamiliar runtime dependency instead of guessing its +transitive dependency closure. + +Kafka **2.6.0** needs one explicit exception: its generated ApiVersions messages +load Jackson `JsonNode`, although its published POM declares `jackson-databind` +as `provided`. This is recorded in +[KAFKA-10378](https://issues.apache.org/jira/browse/KAFKA-10378). The script supplies +the release's exact Jackson **2.10.2** databind, core and annotations JARs, all +checksum-verified. These and the diagnostic binding are listed separately under +`test_runtime_additions` in the preparation report. Other provided dependencies +and published test dependencies are not implicitly added. These additions affect +only the isolated test processes, not Fluss production dependencies. + +Start with the boundaries and representative cached releases: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix \ + --versions 0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1 + +# Set JAVA_HOME to a Java 11 installation before running Maven. +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +After the boundary run, prepare and execute the complete formal interval: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix --versions all + +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +Do not remove `-am`: server, client, RPC and test artifacts must come from this +worktree's reactor, not another branch's installed SNAPSHOTs. A normal build +without `kafka.client.matrix.file` still runs an isolated smoke test using the +module's actual declared client version; that single-version run is not a matrix. + +## Coverage and result interpretation + +For each selected version the probe uses PLAINTEXT and +`enable.idempotence=false`, negotiates APIs through the real producer, and tests +`acks=0`, `acks=1` and `acks=all`. The same producer alternates canonical and bare +names for the same table, checking acknowledged topic names and offsets. Native +readback verifies all 12 records, their real storage offsets, keys, last-header +wins semantics and null binary rescue fields. Bucket leaders are ready before +unacknowledged writes begin. Producer connections stay open until the parent +confirms native readback; a stdin handshake then permits the probe to close. This +avoids racing unacknowledged appends against connection-close cancellation. + +Preparation is not execution. `preparation.json` lists every requested version as +prepared, unavailable or not executed, including exact artifacts and any errors. +The script deletes the previous executable manifest at startup, including when +metadata retrieval or parsing fails. It only publishes `clients.properties` when +all requested versions are prepared. If downloads are blocked, keep that failure +report and make an explicit, separately recorded subset request for the available +versions; never call the subset a full interval pass. + +Runtime evidence lives under: + +- `fluss-kafka/target/client-matrix-results//command.txt` +- `fluss-kafka/target/client-matrix-results//client.log` +- `fluss-kafka/target/client-matrix-results//result.txt` +- `fluss-kafka/target/surefire-reports/TEST-org.apache.fluss.kafka.KafkaClientMatrixITCase.xml` + +Each invocation deletes the old result and log, writes `FAIL` with the error on +failure, and only writes `PASS` after successful native readback. A client failure +fails its parameterized test; other prepared versions still run. A missing JAR, +version mismatch, process timeout or unsupported JDK is a failure, not a skipped +compatibility result. Archive the preparation report, manifest, Maven log, runtime +logs and current Surefire XML together before `clean` or another matrix run. +The POM runs the selected tests in both Surefire phases; count distinct versions +once and require both executions to succeed. + +This is a bounded Produce compatibility acceptance flow, not proof of every +operation in the FIP. Leadership, lifecycle, min-ISR, invalid-record atomicity and +null-value filtering run separately in the cumulative integration tests using the +module client. They are not yet repeated for every matrix version. SASL is reserved +for PR19. Idempotence, other client SDKs, pre-releases, versions outside the target +interval, and persisted per-row `_kafka_metadata.offset` are not covered here. diff --git a/fluss-kafka/src/test/client-matrix/prepare.py b/fluss-kafka/src/test/client-matrix/prepare.py new file mode 100644 index 00000000000..448949bf003 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/prepare.py @@ -0,0 +1,162 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Prepare isolated, checksum-verified client classpaths from Maven Central.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +CENTRAL = "https://repo.maven.apache.org/maven2/" +NS = {"m": "http://maven.apache.org/POM/4.0.0"} +RUNTIME = {"slf4j-api", "lz4", "lz4-java", "snappy-java", "zstd-jni"} + + +def fetch(relative, destination, refresh=False): + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and not refresh: + return + temporary = destination.with_name(destination.name + ".download") + try: + subprocess.run(["curl", "--fail", "--location", "--silent", "--show-error", + "--connect-timeout", "10", "--max-time", "120", "--retry", "2", + CENTRAL + relative, "--output", str(temporary)], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + + +def jar(cache, group, artifact, version): + relative = "/".join([group.replace(".", "/"), artifact, version, + artifact + "-" + version + ".jar"]) + target = cache / relative + fetch(relative, target) + checksum = target.with_suffix(".jar.sha1") + fetch(relative + ".sha1", checksum) + expected = checksum.read_text().strip().split()[0].lower() + actual = hashlib.sha1(target.read_bytes()).hexdigest() + if actual != expected: + raise ValueError("Maven Central checksum mismatch: " + str(target)) + return target.resolve() + + +def prepare(output, version): + cache = output / "artifacts" + relative = "org/apache/kafka/kafka-clients/" + version + "/kafka-clients-" + version + ".pom" + pom = cache / relative + fetch(relative, pom) + root = ET.parse(pom).getroot() + paths = [jar(cache, "org.apache.kafka", "kafka-clients", version)] + dependencies = [] + test_runtime = [] + for dep in root.findall("m:dependencies/m:dependency", NS): + scope = dep.findtext("m:scope", "compile", NS) + group = dep.findtext("m:groupId", namespaces=NS) + artifact = dep.findtext("m:artifactId", namespaces=NS) + dep_version = dep.findtext("m:version", namespaces=NS) + # Kafka 2.6.0 generated ApiVersions messages require JsonNode even for + # byte-array producers, but its POM marks databind provided (KAFKA-10378). + # Supply the release's exact Jackson version and its two runtime dependencies. + if version == "2.6.0" and group == "com.fasterxml.jackson.core" and artifact == "jackson-databind": + if scope != "provided" or dep_version != "2.10.2": + raise ValueError("Unexpected Kafka 2.6.0 Jackson dependency: " + ET.tostring(dep).decode()) + for jackson in ("jackson-databind", "jackson-core", "jackson-annotations"): + paths.append(jar(cache, group, jackson, dep_version)) + test_runtime.append({"coordinate": ":".join([group, jackson, dep_version]), + "reason": "Kafka 2.6.0 provided Jackson closure; KAFKA-10378"}) + continue + if scope not in ("compile", "runtime") or dep.findtext("m:optional", "false", NS) == "true": + continue + # These published Kafka runtime libraries are standalone JARs. Fail on a new + # dependency rather than silently constructing an incomplete classpath. + if artifact not in RUNTIME or not dep_version or "${" in dep_version: + raise ValueError("Unsupported dependency; resolve its runtime closure explicitly: " + ET.tostring(dep).decode()) + paths.append(jar(cache, group, artifact, dep_version)) + dependencies.append(":".join([group, artifact, dep_version])) + if group == "org.slf4j" and artifact == "slf4j-api": + # Without a binding, Kafka's network-thread failures disappear into NOP logging. + paths.append(jar(cache, group, "slf4j-simple", dep_version)) + test_runtime.append({"coordinate": ":".join([group, "slf4j-simple", dep_version]), + "reason": "Isolated client diagnostics; matches published slf4j-api"}) + return paths, dependencies, test_runtime + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--versions", default="0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1", + help="Comma-separated formal releases, or 'all' for the FIP interval") + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + manifest = output / "clients.properties" + manifest.unlink(missing_ok=True) + metadata_path = output / "maven-metadata.xml" + try: + fetch("org/apache/kafka/kafka-clients/maven-metadata.xml", metadata_path, refresh=True) + except Exception as exc: + (output / "preparation.json").write_text(json.dumps( + {"status": "METADATA_UNAVAILABLE_NOT_EXECUTED", "error": str(exc)}, indent=2) + "\n") + return 1 + root = ET.parse(metadata_path).getroot() + versions = [v.text for v in root.findall("versioning/versions/version") + if re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}", v.text)] + def number(version): + return tuple(int(x) for x in version.split(".")) + eligible = sorted([v for v in versions if number("0.11.0.0") <= number(v) <= number("4.3.1")], key=number) + requested = eligible if args.versions == "all" else args.versions.split(",") + report = {"source": CENTRAL + "org/apache/kafka/kafka-clients/maven-metadata.xml", + "prepared_at": datetime.now(timezone.utc).isoformat(), + "metadata_updated": root.findtext("versioning/lastUpdated"), + "formal_releases_in_scope": eligible, "requested": requested, + "not_requested": [v for v in eligible if v not in requested], + "clients": {v: {"status": "NOT_EXECUTED"} for v in requested}} + entries = [] + for version in requested: + try: + if version not in eligible: + raise ValueError("Version is absent from the official formal-release interval") + paths, dependencies, test_runtime = prepare(output, version) + entries.append(version + "=" + os.pathsep.join(str(p) for p in paths)) + report["clients"][version] = {"status": "PREPARED_NOT_EXECUTED", "dependencies": dependencies, + "test_runtime_additions": test_runtime, + "jars": [{"path": str(p), "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in paths]} + except Exception as exc: + detail = getattr(exc, "stderr", b"") + report["clients"][version] = {"status": "UNAVAILABLE_NOT_EXECUTED", + "error": str(exc) + " " + (detail.decode(errors="replace") if isinstance(detail, bytes) else str(detail))} + print(version, report["clients"][version]["status"], flush=True) + (output / "preparation.json").write_text(json.dumps(report, indent=2) + "\n") + # Do not publish a partial executable matrix when a requested version is missing. + manifest.unlink(missing_ok=True) + if any(v["status"] == "UNAVAILABLE_NOT_EXECUTED" for v in report["clients"].values()): + return 1 + manifest.write_text("\n".join(entries) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java new file mode 100644 index 00000000000..dd378dece44 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.matrix.KafkaClientProbe; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.utils.AppInfoParser; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs actual, version-checked Kafka clients in isolated JVMs and verifies native Fluss storage. + */ +class KafkaClientMatrixITCase { + private static final String DATABASE = "client_matrix"; + private static final Path OUTPUT = + Paths.get(System.getProperty("project.basedir"), "target", "client-matrix-results"); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + static Stream clients() throws Exception { + return installations().stringPropertyNames().stream().sorted(); + } + + private static Properties installations() throws Exception { + Properties properties = new Properties(); + String manifest = System.getProperty("kafka.client.matrix.file"); + if (manifest != null) { + try (InputStream input = Files.newInputStream(Paths.get(manifest))) { + properties.load(input); + } + assertThat(properties).as("The requested matrix must not be empty").isNotEmpty(); + } else { + // Normal builds still exercise isolation. Other versions require an explicit manifest. + String classpath = + Arrays.stream( + System.getProperty("java.class.path") + .split(Pattern.quote(File.pathSeparator))) + .filter( + entry -> + Paths.get(entry) + .getFileName() + .toString() + .matches( + "(kafka-clients|slf4j-api|zstd-jni|snappy-java|lz4|lz4-java)-.+\\.jar")) + .collect(Collectors.joining(File.pathSeparator)); + properties.setProperty(AppInfoParser.getVersion(), classpath); + } + return properties; + } + + @ParameterizedTest(name = "Kafka client {0}") + @MethodSource("clients") + void testIsolatedClientWithAllAckModesAndBothNames(String version) throws Exception { + Path output = OUTPUT.resolve(version); + Files.createDirectories(output); + Files.deleteIfExists(output.resolve("result.txt")); + Files.deleteIfExists(output.resolve("client.log")); + Files.deleteIfExists(output.resolve("command.txt")); + try { + runClient(version, output); + } catch (Exception | AssertionError failure) { + Files.write( + output.resolve("result.txt"), + Arrays.asList("FAIL " + version, failure.toString()), + StandardCharsets.UTF_8); + throw failure; + } + } + + private static void runClient(String version, Path output) throws Exception { + String classpath = installations().getProperty(version); + List jars = Arrays.asList(classpath.split(Pattern.quote(File.pathSeparator))); + assertThat(jars).allSatisfy(jar -> assertThat(Paths.get(jar)).isRegularFile()); + assertThat( + jars.stream() + .filter( + jar -> + Paths.get(jar) + .getFileName() + .toString() + .startsWith("kafka-clients-")) + .collect(Collectors.toList())) + .singleElement() + .satisfies( + jar -> + assertThat(Paths.get(jar).getFileName().toString()) + .isEqualTo("kafka-clients-" + version + ".jar")); + // Export just the probe bytecode, never the server/test classpath, to the child JVM. + String resource = KafkaClientProbe.class.getName().replace('.', '/') + ".class"; + Path probe = output.resolve("probe").resolve(resource); + Files.createDirectories(probe.getParent()); + try (InputStream input = + KafkaClientProbe.class.getClassLoader().getResourceAsStream(resource)) { + assertThat(input).isNotNull(); + Files.copy(input, probe, StandardCopyOption.REPLACE_EXISTING); + } + String prefix = "client_" + version.replace('.', '_'); + List paths = new ArrayList<>(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, prefix + "_" + acks); + paths.add(path); + admin.createTable(path, descriptor(), false).get(); + CLUSTER.waitAndGetLeaderReplica( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + } + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + List command = + Arrays.asList( + Paths.get(System.getProperty("java.home"), "bin", "java") + .toString(), + "-cp", + output.resolve("probe") + File.pathSeparator + classpath, + KafkaClientProbe.class.getName(), + version, + node.host() + ":" + node.port(), + DATABASE, + prefix); + Files.write(output.resolve("command.txt"), command, StandardCharsets.UTF_8); + Path log = output.resolve("client.log"); + Process process = + new ProcessBuilder(command) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); + String text = ""; + while (process.isAlive() && System.nanoTime() < deadline) { + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + if (text.contains("WRITES_COMPLETE")) { + break; + } + process.waitFor(100, TimeUnit.MILLISECONDS); + } + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(text) + .as("Client %s log: %s", version, log) + .contains( + "VERSION=" + version, + "JAR=", + "JAVA=", + "ACKS=0:SENT", + "ACKS=1:SENT", + "ACKS=all:SENT", + "WRITES_COMPLETE"); + assertThat(process.isAlive()) + .as("Client must stay connected during native readback") + .isTrue(); + for (String acks : Arrays.asList("0", "1", "all")) { + assertNativeReadback( + connection, + TablePath.of(DATABASE, prefix + "_" + acks), + version, + acks); + } + process.getOutputStream().write('\n'); + process.getOutputStream().flush(); + assertThat(process.waitFor(30, TimeUnit.SECONDS)) + .as("Client log: %s", log) + .isTrue(); + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Client %s: %s", version, text).isZero(); + assertThat(text).contains("NATIVE_READBACK_CONFIRMED"); + Files.write( + output.resolve("result.txt"), + Collections.singletonList( + "PASS " + + version + + " PLAINTEXT non-idempotent; acks=0,1,all; same producer full/alias; 12 native rows and offsets verified"), + StandardCharsets.UTF_8); + } finally { + if (process.isAlive()) { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + } + } + } finally { + for (TablePath path : paths) { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertNativeReadback( + Connection connection, TablePath path, String version, String acks) throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 4 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + assertThat(row.getBytes(0)) + .isEqualTo( + (version + ":" + acks + ":" + count) + .getBytes(StandardCharsets.UTF_8)); + assertThat(row.getBytes(1)).isEqualTo("key".getBytes(StandardCharsets.UTF_8)); + assertThat(row.getMap(2).size()).isEqualTo(1); + assertThat(row.getMap(2).keyArray().getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getMap(2).valueArray().getBytes(0)).containsExactly((byte) 2); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; + } + } + assertThat(count) + .as("Native readback for client %s, acks %s", version, acks) + .isEqualTo(4); + } + } + + private static TableDescriptor descriptor() { + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index f4a255b013d..f745fe7fe8b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; @@ -71,7 +72,8 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler(new TestingTabletGatewayService()).processRequest(request); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + new KafkaRequestHandler(new Configuration(), service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java deleted file mode 100644 index 7f502494f2e..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for Kafka configuration. */ -public class KafkaConfigsTest { - @Test - public void testFromMap() throws Exception { - Map map = new HashMap<>(); - map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); - map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DATABASE.key(), "fluss"); - Configuration configuration = Configuration.fromMap(map); - - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("fluss"); - } - - @Test - public void testFromDefault() throws Exception { - Configuration configuration = Configuration.fromMap(new HashMap<>()); - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("kafka"); - } -} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..b1359f1c705 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,961 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListDatabasesResponse; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.utils.LogContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = topicsRequest(version, "kafka.topic"); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + } + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("kafka.topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("kafka.topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = + handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { + RowType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + if (legacyOffset) { + metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + // Also model an invalid historical descriptor that native DDL would now reject. + service.putTable( + "fluss.metadata_events", + 777L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType) + .build()) + .build()); + short expected = legacyOffset ? Errors.INVALID_TOPIC_EXCEPTION.code() : Errors.NONE.code(); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest( + version, + "metadata_events", + "fluss.metadata_events", + "kafka.topic"), + version); + for (String name : Arrays.asList("metadata_events", "fluss.metadata_events")) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.partitions()).hasSize(legacyOffset ? 0 : 2); + } + assertThat(named.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics().find("fluss.metadata_events").errorCode()) + .isEqualTo(expected); + assertThat(all.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("metadata_events"); + } + } + + @Test + public void testRejectMalformedNamesWithoutHidingValidTopics() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + List invalidNames = Arrays.asList(".topic", "kafka.", "kafka.topic.extra"); + List names = new ArrayList<>(invalidNames); + names.add("kafka.topic"); + MetadataResponse response = + handle( + service, + new MetadataRequest.Builder(names, false).build((short) 11), + (short) 11); + for (String name : invalidNames) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + + @Test + public void testMetadataV12IsRejectedBeforeCallingGateway() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) + .build((short) 12); + + MetadataResponse response = handle(service, request, (short) 12); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(service.gatewayCalls).isZero(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMissingDatabaseRetainsBrokersAndAliases(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Collections.singletonList("missing_db.topic"), + Arrays.asList( + "missing_db.topic", "topic", "fluss.topic", "kafka.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (names.contains("topic")) { + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find(name); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, 777L) + : Uuid.ZERO_UUID); + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, 777L) + : Uuid.ZERO_UUID); + } + } + } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDatabaseDeletedDuringDiscoveryRetainsOtherTopics(boolean failedFuture) { + for (short version = 0; version <= 11; version++) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("deleted_db.topic", 223L); + service.deleteDatabaseBeforeListing = "deleted_db"; + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, allTopicsRequest(version), version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failNextMetadataAsMissing = true; + service.listTablesFailure = new IllegalStateException("metadata unavailable"); + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); + } + + @ParameterizedTest + @ValueSource(strings = {"fluss", "custom_db"}) + public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultDatabase) { + Configuration config = new Configuration(); + if (!"fluss".equals(defaultDatabase)) { + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, defaultDatabase); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + String canonical = defaultDatabase + ".topic"; + service.putTable(canonical, 777L); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest(version, "topic", canonical, "sales.topic"), + version, + config); + assertThat(named.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("topic", canonical, "sales.topic"); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("topic"); + MetadataResponseTopic qualified = named.data().topics().find(canonical); + assertThat(alias.partitions()).isEqualTo(qualified.partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(qualified.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); + } + MetadataResponse all = handle(service, allTopicsRequest(version), version, config); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains(canonical, "sales.topic") + .doesNotContain("topic") + .doesNotHaveDuplicates(); + } + } + + @Test + public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, "topic", "kafka.topic"), + (short) 11, + config); + assertThat(response.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.requestedTables) + .containsExactlyInAnyOrder("missing_default.topic", "kafka.topic"); + assertThat(service.listedDatabases).containsExactlyInAnyOrder("missing_default", "kafka"); + } + + @Test + public void testUnauthorizedAliasesRemainHiddenWithAndWithoutRetry() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.hiddenTopics.add("fluss.topic"); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Arrays.asList("topic", "fluss.topic", "sales.topic"), + Arrays.asList( + "topic", "fluss.topic", "sales.topic", "missing_db.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + MetadataResponseTopic visible = response.data().topics().find("sales.topic"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + if (names.contains("missing_db.topic")) { + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("sales.topic") + .doesNotContain("topic", "fluss.topic"); + } + } + + @Test + public void testPersistedEnablementControlsBothAliasesAndDiscovery() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor missingFlag = + TableDescriptor.builder() + .schema(defaultDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + TableDescriptor disabledInvalid = + TableDescriptor.builder(defaultDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") + .build(); + for (TableDescriptor disabled : Arrays.asList(missingFlag, disabledInvalid)) { + service.putTable("fluss.topic", 777L, disabled); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + assertThat(named.errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("fluss.topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(named.data().topics()) + .allSatisfy( + topic -> { + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + }); + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("topic", "fluss.topic"); + } + } + service.putTable("fluss.topic", 777L); + MetadataResponse enabled = + handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); + assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 777L)); + assertThat(enabled.data().topics().find("fluss.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + service.putTable("fluss.topic", 777L, disabledInvalid); + assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + + @Test + public void testResolvedAliasNameLimitDoesNotHideValidTopic() { + String database = String.join("", Collections.nCopies(125, "d")); + String table = String.join("", Collections.nCopies(125, "t")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, table, "kafka.topic"), + (short) 11, + config); + assertThat(response.errors()).containsEntry(table, Errors.INVALID_TOPIC_EXCEPTION); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.requestedTables).containsExactly("kafka.topic"); + + // A historical enabled table with an unrepresentable canonical name must be visible as + // an error during discovery rather than silently classified as disabled. + service.putTable(database + "." + table, 777L); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11, config).errors()) + .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientCanCacheCanonicalNameAndAliasTogether(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "fluss.topic"), version), false, 1L); + // A producer discovers the alias after already caching the canonical name. Nonzero + // duplicate IDs would make Kafka's MetadataSnapshot throw while building topicNames. + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic"), version), true, 2L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic", "fluss.topic"), version), + false, + 3L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + assertThat(client.topicIds()).containsOnlyKeys("topic", "fluss.topic"); + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 777L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 777L), "topic"); + client.close(); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientRefreshesRecreatedAliasWithLowerLeaderEpoch(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + MetadataResponse original = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + original.data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(1).setLeaderEpoch(5))); + client.updateWithCurrentRequestVersion(original, false, 1L); + for (String name : Arrays.asList("topic", "fluss.topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()).isEqualTo(1); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(5); + } + + // The client never observes the missing table between deletion and recreation. + service.removeTable("fluss.topic"); + service.putTable("fluss.topic", 778L); + MetadataResponse recreated = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + recreated + .data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(2).setLeaderEpoch(0))); + client.updateWithCurrentRequestVersion(recreated, false, 2L); + for (String name : Arrays.asList("fluss.topic", "topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()) + .as("Leader after recreating %s", name) + .isEqualTo(2); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(0); + } + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 778L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 778L), "topic") + .doesNotContainKeys( + new Uuid(0x466c757373000000L, 777L), new Uuid(0x466c757373410000L, 777L)); + client.close(); + } + + private static MetadataRequest topicsRequest(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static TableDescriptor defaultDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return topicsRequest((short) 11, topicName); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + return handle(service, requestBody, version, new Configuration()); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, + MetadataRequest requestBody, + short version, + Configuration config) { + KafkaRequestHandler handler = new KafkaRequestHandler(config, service); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Set databases = new LinkedHashSet<>(); + private final Set hiddenTopics = new LinkedHashSet<>(); + private final Map tables = new LinkedHashMap<>(); + private final Map descriptors = new LinkedHashMap<>(); + private int gatewayCalls; + private int listDatabasesCalls; + private final Set requestedTables = new LinkedHashSet<>(); + private final Set listedDatabases = new LinkedHashSet<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + private boolean failListTablesAsFuture; + private String deleteDatabaseBeforeListing; + private RuntimeException listTablesFailure; + + private TestingMetadataGatewayService() { + putTable("kafka.topic", 123L); + putTable("kafka.other", 124L); + } + + @Override + public CompletableFuture listDatabases( + ListDatabasesRequest request) { + gatewayCalls++; + listDatabasesCalls++; + assertThat(currentListenerName()).isEqualTo("KAFKA"); + return CompletableFuture.completedFuture( + new ListDatabasesResponse().addAllDatabaseNames(databases)); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + gatewayCalls++; + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getDatabaseName(); + listedDatabases.add(database); + if (database.equals(deleteDatabaseBeforeListing)) { + databases.remove(database); + tables.keySet().removeIf(name -> name.startsWith(database + ".")); + } + RuntimeException failure = listTablesFailure; + if (failure == null && !databases.contains(database)) { + failure = new DatabaseNotExistException("Database does not exist."); + } + if (failure != null) { + if (failListTablesAsFuture) { + return FutureUtils.completedExceptionally(new CompletionException(failure)); + } + throw failure; + } + String prefix = request.getDatabaseName() + "."; + List names = + tables.keySet().stream() + .filter(name -> !hiddenTopics.contains(name)) + .filter(name -> name.startsWith(prefix)) + .map(name -> name.substring(prefix.length())) + .collect(Collectors.toList()); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(names)); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + gatewayCalls++; + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + requestedTables.add(topicName); + // Native Metadata and listTables omit tables without DESCRIBE permission. + if (hiddenTopics.contains(topicName)) { + continue; + } + Long tableId = tables.get(topicName); + if (tableId == null) { + throw new TableNotExistException("Table does not exist: " + topicName); + } + topics.add( + tableMetadata( + topicName, + tableId, + !"kafka.topic".equals(topicName) || topicLeaderAvailable, + "kafka.topic".equals(topicName) + ? topicIsr + : new int[] {1, 2}, + "kafka.topic".equals(topicName) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson(descriptors.get(topicName).toJsonBytes())); + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + putTable(topic, tableId, defaultDescriptor()); + } + + private void putTable(String topic, long tableId, TableDescriptor descriptor) { + databases.add(topic.substring(0, topic.indexOf('.'))); + tables.put(topic, tableId); + descriptors.put(topic, descriptor); + } + + private void removeTable(String topic) { + tables.remove(topic); + descriptors.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath() + .setDatabaseName(topic.substring(0, topic.indexOf('.'))) + .setTableName(topic.substring(topic.indexOf('.') + 1))) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java new file mode 100644 index 00000000000..3a5e582ee71 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ +public class KafkaMetadataITCase { + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(2) + .setClusterConf(clusterConfiguration()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long existing = createTable(CLUSTER, TablePath.of("existing_db", "events"), descriptor); + long deleted = createTable(CLUSTER, TablePath.of("deleted_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(existing); + CLUSTER.waitUntilTableReady(deleted); + dropDatabase(CLUSTER, "deleted_db"); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse missing = query(node, named(version, "deleted_db.events")); + assertThat(missing.brokers()).hasSize(2); + assertThat(missing.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + + MetadataResponse mixed = + query(node, named(version, "deleted_db.events", "existing_db.events")); + assertThat(mixed.brokers()).hasSize(2); + assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); + assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(mixed.data().topics().find("existing_db.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); + + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("existing_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @Test + public void testAliasesAndOptionalMetadataAcrossDatabases() throws Exception { + long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(other); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse before = query(node, named(version, "events", "sales.events")); + assertThat(before.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(before.data().topics().find("sales.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + // Exercise PR03's optional metadata column through the same alias and discovery paths. + TableDescriptor withMetadata = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), withMetadata); + CLUSTER.waitUntilTableReady(tableId); + for (short version = 0; version <= 11; version++) { + for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { + // The missing database forces the same discovery retry that used to discard + // aliases. + MetadataResponse response = + query( + broker, + named( + version, + "events", + "fluss.events", + "sales.events", + "fluss.missing", + "missing_db.events")); + assertThat(response.brokers()).hasSize(2); + assertThat(response.brokers()).extracting(Node::port).contains(broker.port()); + for (String name : Arrays.asList("fluss.events", "sales.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("sales.events").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, other) + : Uuid.ZERO_UUID); + MetadataResponseTopic alias = response.data().topics().find("events"); + MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); + assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(alias.partitions()).isEqualTo(canonical.partitions()).hasSize(2); + assertThat(alias.partitions()) + .allSatisfy( + partition -> { + assertThat(partition.leaderId()).isIn(0, 1); + assertThat(partition.replicaNodes()) + .containsExactlyInAnyOrder(0, 1); + assertThat(partition.isrNodes()) + .containsExactlyInAnyOrder(0, 1); + }); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, tableId) + : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(canonical.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(response.data().topics().find("sales.events").topicId()); + } + for (String missing : Arrays.asList("fluss.missing", "missing_db.events")) { + assertThat(response.data().topics().find(missing).errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + MetadataResponse all = query(broker, allTopics(version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + } + + @Test + public void testPersistedEnablementChangesAreVisibleThroughBothNames() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + TableDescriptor disabled = + TableDescriptor.builder(topicDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + TableDescriptor unset = + TableDescriptor.builder() + .schema(topicDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + long tableId = createTable(CLUSTER, path, disabled); + long unsetId = createTable(CLUSTER, TablePath.of("sales", "unset"), unset); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(unsetId); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + setEnabled(CLUSTER, path, false); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertExposure(node, false, tableId); + } + + @Test + public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + long tableId = createTable(CLUSTER, path, topicDescriptor()); + long validId = createTable(CLUSTER, TablePath.of("sales", "valid"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(validId); + TableRegistration registration = CLUSTER.getZooKeeperClient().getTable(path).get(); + Map invalidProperties = new HashMap<>(registration.properties); + invalidProperties.put(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "string"); + // Bypass current DDL validation to model historical persisted metadata with a damaged + // contract. Metadata must expose the enabled table's error, not hide it as disabled. + CLUSTER.getZooKeeperClient() + .updateTable( + path, + registration.newProperties( + invalidProperties, registration.customProperties)); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code())); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "events", "fluss.events", "sales.valid")); + for (String topic : Arrays.asList("events", "fluss.events")) { + assertThat(named.data().topics().find(topic).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(named.data().topics().find(topic).partitions()).isEmpty(); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.valid"); + assertThat(all.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(all.data().topics().find("sales.valid").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + } + + @Test + public void testCustomDefaultDatabaseAliasAndNoRecreationByMetadata() throws Exception { + Configuration config = clusterConfiguration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try { + long nativeDefaultId = + createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); + cluster.waitUntilTableReady(nativeDefaultId); + ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + // PR03 creates the configured database at startup, before any Metadata request. + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isTrue(); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = query(node, named(version, "events", "fluss.events")); + assertThat(response.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long customId = + createTable( + cluster, TablePath.of("custom_default", "events"), topicDescriptor()); + cluster.waitUntilTableReady(customId); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query( + node, + named(version, "events", "custom_default.events", "fluss.events")); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = response.data().topics().find("events"); + assertThat(alias.partitions()) + .isEqualTo( + response.data() + .topics() + .find("custom_default.events") + .partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, customId) + : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(response.data().topics().find("custom_default.events").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); + } + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("custom_default.events", "fluss.events"); + } + dropDatabase(cluster, "custom_default"); + MetadataResponse deleted = query(node, named((short) 11, "events", "fluss.events")); + assertThat(deleted.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(deleted.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + } finally { + cluster.close(); + } + } + + private static void assertExposure(ServerNode node, boolean enabled, long tableId) + throws Exception { + short expected = enabled ? Errors.NONE.code() : Errors.UNKNOWN_TOPIC_OR_PARTITION.code(); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(expected)); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query(node, named(version, "events", "fluss.events", "sales.unset")); + for (String name : Arrays.asList("events", "fluss.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.topicId()) + .isEqualTo( + enabled && version >= 10 + ? new Uuid( + name.indexOf('.') >= 0 + ? 0x466c757373000000L + : 0x466c757373410000L, + tableId) + : Uuid.ZERO_UUID); + } + assertThat(response.data().topics().find("sales.unset").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (enabled) { + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events"); + } else { + assertThat(query(node, allTopics(version)).data().topics()).isEmpty(); + } + } + } + + private static void setEnabled(FlussClusterExtension cluster, TablePath path, boolean enabled) + throws Exception { + cluster.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MetadataRequest named(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static MetadataRequest allTopics(short version) { + return new MetadataRequest( + new MetadataRequestData().setTopics(version == 0 ? Collections.emptyList() : null), + version); + } + + private static TableDescriptor topicDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + } + + private static Configuration clusterConfiguration() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 2); + conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); + return conf; + } + + private static MetadataResponse query(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, request.version(), "metadata-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + try (Socket socket = new Socket(node.host(), node.port())) { + socket.setSoTimeout(10000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] responseBytes = new byte[input.readInt()]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java new file mode 100644 index 00000000000..d32fa77059b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -0,0 +1,746 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ +class KafkaProduceAppendITCase { + private static final String DATABASE = "produce_default"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_" + format + "_" + acks); + String topic = path.toString(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + // Node discovery does not wait for table replicas. With acks=0 the producer + // cannot observe or retry a write rejected before the leader is ready. + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks)) { + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 123L, + KEY, + VALUE, + record.headers())) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(topic); + assertThat(second.topic()).isEqualTo(path.getTableName()); + if (!acks.equals("0")) { + assertThat(first.offset()).isZero(); + assertThat(second.offset()).isEqualTo(1L); + } + producer.flush(); + assertReadback(connection, path, format, VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValuesArePersistedForAllAckModes(String format) throws Exception { + List values = Arrays.asList(null, new byte[0], VALUE, null); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_null_" + format + "_" + acks); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks, 0)) { + for (int i = 0; i < values.size(); i++) { + ProducerRecord record = + new ProducerRecord<>( + i % 2 == 0 ? path.toString() : path.getTableName(), + 0, + 123L, + KEY, + values.get(i), + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata metadata = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(metadata.offset()).isEqualTo(i); + } + } + producer.flush(); + assertReadback(connection, path, format, values); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testAllNullAndMixedBatchesPreserveOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_null_batches_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceOffsets(query(socket, recordBatch(null, null), names), names, 0L); + assertProduceOffsets( + query(socket, recordBatch(VALUE, null, new byte[0], null), names), + names, + 2L); + assertProduceOffsets(query(socket, names), names, 6L); + assertReadback( + connection, + path, + format, + Arrays.asList(null, null, VALUE, null, new byte[0], null, VALUE)); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_not_null_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format, false), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceErrors( + query(socket, recordBatch(null, null), names), + names, + Errors.INVALID_RECORD); + assertProduceErrors( + query(socket, recordBatch(VALUE, null, VALUE), names), + names, + Errors.INVALID_RECORD); + assertProduceOffsets(query(socket, names), names, 0L); + assertReadback(connection, path, format, VALUE, 1); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNullAndEmptyValuesPreserveKeysAndOffsets() throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String format : new String[] {"binary", "string"}) { + TablePath path = TablePath.of(DATABASE, "null_values_" + format); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + String topic = path.toString(); + assertThat( + producer.send( + new ProducerRecord( + topic, + 0, + 123L, + new byte[] {(byte) 0xff}, + null)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, new byte[0])) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(1L); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, VALUE)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 3 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + if (count == 0) { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(3)).isEqualTo(new byte[] {(byte) 0xff}); + } else { + byte[] expected = count == 1 ? new byte[0] : VALUE; + if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo( + new String( + expected, StandardCharsets.UTF_8)); + } + assertThat(row.isNullAt(3)).isTrue(); + } + assertThat(row.getMap(1).size()).isZero(); + assertThat(row.isNullAt(2)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; + } + } + assertThat(count).isEqualTo(3); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testSameNamedTablesAcrossDatabases() throws Exception { + Map tables = new LinkedHashMap<>(); + tables.put(TablePath.of("produce_first", "events"), VALUE); + tables.put( + TablePath.of("produce_second", "events"), + "second database".getBytes(StandardCharsets.UTF_8)); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + try { + for (TablePath path : tables.keySet()) { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) + .get(); + admin.createTable(path, descriptor("binary"), false).get(); + } + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all")) { + for (Map.Entry entry : tables.entrySet()) { + String topic = entry.getKey().toString(); + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + entry.getValue(), + Arrays.asList( + new RecordHeader("source", entry.getValue()), + new RecordHeader("source", null))); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isZero(); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + } + for (Map.Entry entry : tables.entrySet()) { + assertReadback(connection, entry.getKey(), "binary", entry.getValue()); + } + } + } finally { + for (TablePath path : tables.keySet()) { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testPipelinedProducePreservesAppendOrder() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_pipelined"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + List values = new ArrayList<>(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + byte[] value = new byte[i % 2 == 0 ? 16384 : 1]; + Arrays.fill(value, (byte) i); + values.add(value); + writes.add( + producer.send( + new ProducerRecord<>( + path.toString(), 0, (long) i, KEY, value))); + } + for (int i = 0; i < writes.size(); i++) { + assertThat(writes.get(i).get(30, TimeUnit.SECONDS).offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); + count++; + } + } + assertThat(count).isEqualTo(values.size()); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testAliasesShareOffsetsAndObservePersistedEnablementOnOneConnection() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_enablement"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.getTableName(), path.toString()}; + assertProduceOffsets(query(socket, names), names, 0L, 1L); + setEnabled(path, false); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setEnabled(path, true); + assertProduceOffsets(query(socket, names), names, 2L, 3L); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList( + ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertReadback(connection, path, "binary", VALUE, 4); + admin.dropTable(path, true).get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Exception { + Configuration config = clusterConfig(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "produce_missing_db"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try (Connection connection = ConnectionFactory.createConnection(cluster.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + // Startup creates the configured default; Produce must not recreate a deleted one. + assertThat(admin.databaseExists("produce_missing_db").get()).isTrue(); + admin.dropDatabase("produce_missing_db", false, false).get(); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + TablePath existing = TablePath.of("fluss", "events"); + long tableId = createTable(cluster, existing, descriptor("binary")); + cluster.waitUntilTableReady(tableId); + ServerNode node = cluster.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + String[] missing = { + "events", + "produce_missing_db.events", + "another_missing_db.events", + "fluss.missing" + }; + assertProduceErrors( + query(socket, missing), missing, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 0L); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 1L); + try (KafkaProducer producer = producer(node, "all", 0)) { + assertThat( + producer.send( + new ProducerRecord<>( + existing.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader( + "source", VALUE), + new RecordHeader( + "source", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + } + assertReadback(connection, existing, "binary", VALUE, 3); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); + } + } finally { + cluster.close(); + } + } + + private static void setEnabled(TablePath path, boolean enabled) throws Exception { + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MemoryRecords recordBatch(byte[]... values) { + SimpleRecord[] records = new SimpleRecord[values.length]; + for (int i = 0; i < values.length; i++) { + records[i] = + new SimpleRecord( + 123L, + KEY, + values[i], + new Header[] { + new RecordHeader("source", VALUE), new RecordHeader("source", null) + }); + } + return MemoryRecords.withRecords(Compression.NONE, records); + } + + private static ProduceResponse query(Socket socket, String... names) throws Exception { + return query(socket, recordBatch(VALUE), names); + } + + private static ProduceResponse query(Socket socket, MemoryRecords records, String... names) + throws Exception { + List topics = new ArrayList<>(); + for (String name : names) { + topics.add( + new TopicProduceData() + .setName(name) + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords(records)))); + } + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + topics.iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] response = new byte[input.readInt()]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void assertProduceOffsets( + ProduceResponse response, String[] names, long... offsets) { + assertProduceErrors(response, names, Errors.NONE); + for (int i = 0; i < names.length; i++) { + assertThat( + response.data() + .responses() + .find(names[i]) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(offsets[i]); + } + } + + private static void assertProduceErrors( + ProduceResponse response, String[] names, Errors error) { + assertThat(response.data().responses()) + .extracting(topic -> topic.name()) + .containsExactly(names); + for (String name : names) { + List partitions = + response.data().responses().find(name).partitionResponses(); + assertThat(partitions).hasSize(1); + assertThat(partitions.get(0).index()).isZero(); + assertThat(partitions.get(0).errorCode()).isEqualTo(error.code()); + if (error != Errors.NONE) { + assertThat(partitions.get(0).baseOffset()).isEqualTo(-1L); + } + } + } + + private static void assertReadback( + Connection connection, TablePath path, String format, byte[] expectedValue) + throws Exception { + assertReadback(connection, path, format, expectedValue, 2); + } + + private static void assertReadback( + Connection connection, + TablePath path, + String format, + byte[] expectedValue, + int expectedCount) + throws Exception { + assertReadback(connection, path, format, Collections.nCopies(expectedCount, expectedValue)); + } + + private static void assertReadback( + Connection connection, TablePath path, String format, List expectedValues) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < expectedValues.size() && System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(count); + byte[] expectedValue = expectedValues.get(count); + InternalRow row = record.getRow(); + if (expectedValue == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expectedValue); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); + } + assertThat(row.getBytes(3)).isEqualTo(KEY); + assertThat(row.getMap(1).size()).isEqualTo(1); + assertThat(row.getMap(1).keyArray().getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getMap(1).valueArray().isNullAt(0)).isTrue(); + assertThat(row.isNullAt(2)).isTrue(); + InternalRow metadata = row.getRow(4, 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()) + .isEqualTo(123L); + assertThat(metadata.getInt(1)).isZero(); + assertThat(metadata.getInt(2)).isZero(); + count++; + } + } + assertThat(count) + .as("Native readback for %s (%s)", path, format) + .isEqualTo(expectedValues.size()); + } + } + + private static TableDescriptor descriptor(String format) { + return descriptor(format, true); + } + + private static TableDescriptor descriptor(String format, boolean nullableValue) { + boolean binary = format.equals("binary"); + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (binary ? DataTypes.BYTES() : DataTypes.STRING()) + .copy(nullableValue)) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + } + + private static KafkaProducer producer(String acks) { + return producer(acks, 16384); + } + + private static KafkaProducer producer(String acks, int batchSize) { + return producer(CLUSTER.getTabletServerNodes("KAFKA").get(0), acks, batchSize); + } + + private static KafkaProducer producer( + ServerNode node, String acks, int batchSize) { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..1a801cd55ed --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,829 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.message.ProduceResponseData.TopicProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecordBatch; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.utils.Crc32C; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(record.hasTimestamp()).isTrue(); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testBareAndQualifiedTopicNamesRemainUnchanged(short version) { + MemoryRecords nullableValues = + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord(123L, new byte[] {1}, null), + new SimpleRecord(456L, null, new byte[0])); + ProduceRequest request = + requestForTopics( + version, + (short) 1, + topic("orders", partition(0, nullableValues)), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + assertThat(topic.partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(0); + } + List copied = + command.topics().get(0).partitions().get(0).records(); + assertThat(copied).hasSize(2); + // The protocol command preserves null and empty values for the adapter. + assertThat(copied.get(0).value()).isNull(); + assertThat(copied.get(1).value()).isEmpty(); + List results = + new ArrayList<>(successful(command).join().topics()); + Collections.reverse(results); + return CompletableFuture.completedFuture( + new KafkaProduceResult(results)); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (TopicProduceResponse topic : response.data().responses()) { + assertThat(topic.partitionResponses()) + .extracting( + PartitionProduceResponse::index, + PartitionProduceResponse::errorCode, + PartitionProduceResponse::baseOffset) + .containsExactly(tuple(0, Errors.NONE.code(), 42L)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"orders", "fluss.orders"}) + void testLocalAndBackendErrorsAreIsolatedByRequestTopicName(String invalidTopic) { + String validTopic = invalidTopic.equals("orders") ? "fluss.orders" : "orders"; + ProduceResponse response = + dispatch( + requestForTopics( + (short) 11, + (short) 1, + topic(invalidTopic, partition(0, MemoryRecords.EMPTY)), + topic(validTopic, partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly(validTopic, "sales.orders"); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Arrays.asList( + new TopicResult( + "sales.orders", + Collections.singletonList( + new PartitionResult( + 0, + Errors + .TOPIC_AUTHORIZATION_FAILED, + -1L, + null))), + new TopicResult( + validTopic, + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + 42L, + null)))))); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly(invalidTopic, validTopic, "sales.orders"); + assertTopicError(response, invalidTopic, Errors.INVALID_RECORD); + assertTopicError(response, validTopic, Errors.NONE); + assertTopicError(response, "sales.orders", Errors.TOPIC_AUTHORIZATION_FAILED); + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testRequestAndBackendFailuresPreserveTopicNames(short version) { + for (boolean invalidAcks : Arrays.asList(false, true)) { + ProduceResponse response = + dispatch( + requestForTopics( + version, + invalidAcks ? (short) 2 : (short) 1, + topic("orders", partition(0, records())), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(invalidAcks).isFalse(); + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally(new TimeoutException("append failed")); + return failed; + }); + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactlyInAnyOrder("orders", "fluss.orders", "sales.orders"); + Errors error = invalidAcks ? Errors.INVALID_REQUIRED_ACKS : Errors.REQUEST_TIMED_OUT; + for (String name : Arrays.asList("orders", "fluss.orders", "sales.orders")) { + assertTopicError(response, name, error); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { + MemoryRecords createTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L, + 456L); + MemoryRecords appendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 123L, + 789L, + 987L); + MemoryRecords missingCreateTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + MemoryRecords missingAppendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, createTime), + partition(1, appendTime), + partition(2, missingCreateTime), + partition(3, missingAppendTime)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE, Errors.NONE, Errors.NONE, Errors.NONE); + for (MemoryRecords input : + Arrays.asList(createTime, appendTime, missingCreateTime, missingAppendTime)) { + ByteBuffer buffer = input.buffer().duplicate(); + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + } + + List partitions = + copied.get().topics().get(0).partitions(); + assertThat(partitions).hasSize(4); + assertThat(partitions.get(0).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 456L); + // The same numeric timestamp can have a different batch type. + assertThat(partitions.get(1).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 123L); + for (int i = 0; i < partitions.size(); i++) { + TimestampType expectedType = + i % 2 == 0 ? TimestampType.CREATE_TIME : TimestampType.LOG_APPEND_TIME; + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestampType) + .containsExactly(expectedType, expectedType); + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::hasTimestamp) + .containsExactly(i < 2, i < 2); + if (i >= 2) { + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(RecordBatch.NO_TIMESTAMP, RecordBatch.NO_TIMESTAMP); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testMultipleBatchesPerPartitionRemainInvalid(CompressionType compressionType) { + MemoryRecords first = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L); + MemoryRecords second = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 456L, + 123L); + ByteBuffer combined = ByteBuffer.allocate(first.sizeInBytes() + second.sizeInBytes()); + combined.put(first.buffer().duplicate()).put(second.buffer().duplicate()).flip(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.readableRecords(combined)), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { + byte[] value = new byte[32 * 1024]; + Arrays.fill(value, (byte) 7); + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord( + 123L, + new byte[] {1}, + value, + new Header[] { + new RecordHeader("header", new byte[] {2, 3}), + new RecordHeader("header", null) + }), + new SimpleRecord(456L, null, new byte[0])); + AtomicReference copied = new AtomicReference<>(); + + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE); + ByteBuffer input = records.buffer().duplicate(); + while (input.hasRemaining()) { + input.put((byte) 0); + } + + List result = + copied.get().topics().get(0).partitions().get(0).records(); + assertThat(result).hasSize(2); + assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(result.get(0).key()).containsExactly((byte) 1); + assertThat(result.get(0).value()).containsExactly(value); + assertThat(result.get(0).headers()) + .extracting(KafkaProduceCommand.RecordHeader::name) + .containsExactly("header", "header"); + assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); + assertThat(result.get(0).headers().get(1).value()).isNull(); + assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(result.get(1).key()).isNull(); + assertThat(result.get(1).value()).isEmpty(); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testIncorrectRecordCountDoesNotSuppressValidPartition(CompressionType compressionType) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + ByteBuffer bytes = malformed.buffer(); + bytes.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, 32); + bytes.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 31); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testInvalidBatchOffsetRangeDoesNotSuppressValidPartition(CompressionType compressionType) { + for (int[] invalidHeader : + new int[][] {{1, -1}, {1, 9}, {1, Integer.MAX_VALUE}, {0, 0}, {-1, 0}}) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + int recordCount = invalidHeader[0]; + int lastOffsetDelta = invalidHeader[1]; + malformed.buffer().putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, recordCount); + malformed.buffer().putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testValidBatchUsesOffsetAssignedByBackend(CompressionType compressionType) { + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1}), + new SimpleRecord(new byte[] {2}), + new SimpleRecord(new byte[] {3})); + long backendBaseOffset = 1L << 40; + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + assertThat(command.topics().get(0).partitions().get(0).records()) + .hasSize(3); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + backendBaseOffset, + null)))))); + }); + assertErrors(response, Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(backendBaseOffset); + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher( + command -> { + assertThat( + command.topics() + .get(0) + .partitions() + .get(0) + .records() + .get(0) + .timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + return pending; + }) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = + new KafkaApiRegistry(Collections.singletonList(new ProduceHandler(backend))); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + return requestForTopics(version, acks, topic("topic", partitions)); + } + + private static ProduceRequest requestForTopics( + short version, short acks, TopicProduceData... topics) { + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Arrays.asList(topics).iterator())), + version); + } + + private static TopicProduceData topic(String name, PartitionProduceData... partitions) { + return new TopicProduceData().setName(name).setPartitionData(Arrays.asList(partitions)); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static MemoryRecords timestampedRecords( + CompressionType compressionType, + org.apache.kafka.common.record.TimestampType timestampType, + long logAppendTime, + long... timestamps) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.of(compressionType).build(), + timestampType, + 0L, + logAppendTime)) { + for (long timestamp : timestamps) { + builder.append(new SimpleRecord(timestamp, new byte[] {1}, new byte[] {2})); + } + return builder.build(); + } + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } + + private static void assertTopicError(ProduceResponse response, String topic, Errors error) { + assertThat(response.data().responses().find(topic).partitionResponses()) + .extracting(PartitionProduceResponse::index, PartitionProduceResponse::errorCode) + .containsExactly(tuple(0, error.code())); + } + + private static void updateBatchChecksum(MemoryRecords records) { + ByteBuffer bytes = records.buffer(); + int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; + bytes.putInt( + DefaultRecordBatch.CRC_OFFSET, + (int) Crc32C.compute(bytes, checksumStart, bytes.limit() - checksumStart)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java new file mode 100644 index 00000000000..5d016ac5c9b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -0,0 +1,1074 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.rebalance.RebalancePlanForBucket; +import org.apache.fluss.cluster.rebalance.RebalanceProgress; +import org.apache.fluss.cluster.rebalance.RebalanceStatus; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.RebalanceTask; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.groups.Tuple; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +/** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ +class KafkaProduceRegressionITCase { + private static final String DATABASE = "fluss"; + private static final short METADATA_VERSION = 11; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_lifecycle"); + String topic = path.toString(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + MetadataResponse initial = waitForTopic(topic); + Uuid initialId = initial.data().topics().find(topic).topicId(); + assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); + assertKafkaListenerEndpoints(initial); + assertThat(fetchMetadata(topic).data().topics().find(topic).topicId()) + .isEqualTo(initialId); + + admin.dropTable(path, false).get(); + retry( + Duration.ofMinutes(1), + () -> + assertThat(fetchMetadata(topic).errors()) + .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); + + createTable(admin, path, 3); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic recreated = + fetchMetadata(topic).data().topics().find(topic); + assertThat(recreated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(recreated.topicId()) + .isNotEqualTo(Uuid.ZERO_UUID) + .isNotEqualTo(initialId); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_failover"); + String topic = path.toString(); + int stoppedLeader = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + stoppedLeader = initial.leaderId(); + try (KafkaProducer producer = producer("1", true)) { + assertThat(send(producer, topic, "before_failover").offset()).isNotNegative(); + CLUSTER.stopTabletServer(stoppedLeader); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic metadata = + fetchMetadata(topic).data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = metadata.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(initial.leaderId()); + assertThat(partition.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + // Reuse the same producer to verify routing recovers after leader disconnect. + // acks=1 permits data loss and offset reuse after failover; this verifies + // continued routing, not durability of the first acknowledged record. + assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); + } + } finally { + try { + if (stoppedLeader >= 0 && CLUSTER.getTabletServerById(stoppedLeader) == null) { + CLUSTER.startTabletServer(stoppedLeader); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_online_leader_change"); + String topic = path.toString(); + String rebalanceId = "kafka-online-leader-change"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3, metadataSchema(), 1); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + int oldLeader = initial.leaderId(); + int newLeader = + initial.replicaNodes().stream() + .filter(id -> id != oldLeader) + .findFirst() + .get(); + List nodes = CLUSTER.getTabletServerNodes("KAFKA"); + ServerNode oldNode = + nodes.stream().filter(node -> node.id() == oldLeader).findFirst().get(); + + // Keep this socket open across the handoff to verify the old leader stays + // reachable. + try (Socket oldLeaderSocket = connect(oldNode); + KafkaProducer producer = producer("all", true)) { + MetadataResponse before = + (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); + assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, 0, 123L, "before_handoff").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0)); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); + double retriesBefore = producerMetric(producer, "record-retry-total"); + double disconnectsBefore = producerMetric(producer, "connection-close-total"); + + rebalanceLeader(bucket, initial, newLeader, rebalanceId); + waitForLeaderChange( + admin, topic, 0, newLeader, initial.leaderEpoch(), rebalanceId); + CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); + assertThat(CLUSTER.getTabletServerNodes("KAFKA")) + .containsExactlyInAnyOrderElementsOf(nodes); + + ProduceResponse rejected = + (ProduceResponse) + sendRequest(oldLeaderSocket, produceRequest(topic, "rejected")); + assertThat(rejected.data().responses().find(topic).partitionResponses()) + .singleElement() + .satisfies( + partition -> { + assertThat(partition.index()).isZero(); + assertThat(partition.errorCode()) + .isEqualTo(Errors.NOT_LEADER_OR_FOLLOWER.code()); + assertThat(partition.baseOffset()).isEqualTo(-1L); + }); + + // partitionsFor uses the already populated producer cache. Separate raw + // Metadata requests above must not refresh this route before the next send. + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, 0, 456L, "after_handoff").offset()) + .isEqualTo(1L); + assertThat(producerMetric(producer, "record-retry-total")) + .isGreaterThan(retriesBefore); + assertThat(producerMetric(producer, "connection-close-total")) + .isEqualTo(disconnectsBefore); + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(newLeader); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0), + tuple(1L, "after_handoff", 456L, 0, 0)); + } + } finally { + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + TablePath replicated = TablePath.of(DATABASE, "acks_all_replicated"); + TablePath underReplicated = TablePath.of(DATABASE, "acks_all_under_replicated"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, replicated, 3); + createTable(admin, underReplicated, 1); + try (KafkaProducer producer = producer("all", false)) { + assertThat(send(producer, replicated.toString(), "replicated").offset()) + .isZero(); + assertThatThrownBy(() -> send(producer, underReplicated.toString(), "rejected")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class); + + setMinIsr(admin, 1); + assertThat(send(producer, underReplicated.toString(), "accepted").offset()) + .isZero(); + // Offset zero and native readback also prove the rejected write was not stored. + assertValues(connection, underReplicated, "accepted"); + } + } finally { + try { + setMinIsr(admin, 2); + } finally { + admin.dropTable(replicated, true).get(); + admin.dropTable(underReplicated, true).get(); + } + } + } + } + + @Test + void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_lifecycle"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + assertThat(aliasId).isNotEqualTo(Uuid.ZERO_UUID).isNotEqualTo(fullId); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, full, "full").offset()).isZero(); + assertThat(send(producer, alias, "alias").offset()).isEqualTo(1L); + setTableEnabled(admin, path, false); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + assertThat(send(producer, alias, "enabled").offset()).isEqualTo(2L); + Map properties = + admin.getTableInfo(path).get().getProperties().toMap(); + assertThatThrownBy( + () -> + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "string")), + false) + .get()) + .isInstanceOf(ExecutionException.class); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .isEqualTo(properties); + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "BINARY")), + false) + .get(); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .containsEntry( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "BINARY"); + assertThat(send(producer, full, "property").offset()).isEqualTo(3L); + assertValues(connection, path, "full", "alias", "enabled", "property"); + admin.dropTable(path, false).get(); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + createTable(admin, path, 3); + assertThat(topicId(full)).isNotEqualTo(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(alias)).isNotEqualTo(aliasId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isNotEqualTo(topicId(alias)); + assertThat(send(producer, full, "recreated_full").offset()).isZero(); + assertThat(send(producer, alias, "recreated_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "recreated_full", "recreated_alias"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testSameProducerAliasesUseAddedMetadataColumn() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_schema_alter"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableInfo before = admin.getTableInfo(path).get(); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + try (KafkaProducer producer = producer("all", true)) { + // Populate both name mappings and the write plan before changing the schema. + assertThat(send(producer, full, "old_full").offset()).isZero(); + assertThat(send(producer, alias, "old_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "old_full", "old_alias"); + + admin.alterTable( + path, + Collections.singletonList( + TableChange.addColumn( + "_kafka_metadata", + metadataType(), + null, + TableChange.ColumnPosition.last())), + false) + .get(); + TableInfo after = admin.getTableInfo(path).get(); + assertThat(after.getTableId()).isEqualTo(before.getTableId()); + assertThat(after.getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(after.getProperties().toMap()) + .isEqualTo(before.getProperties().toMap()); + assertThat(after.getSchema().getColumns()).hasSize(2); + assertThat(after.getSchema().getColumn("_kafka_metadata").getDataType()) + .isEqualTo(metadataType()); + + // Neither producer recreation nor a Kafka Metadata refresh should be needed. + assertThat(send(producer, full, 0, 789L, "new_full").offset()).isEqualTo(2L); + assertThat(send(producer, alias, 0, 987L, "new_alias").offset()).isEqualTo(3L); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + // Read the new batches using the new native schema, independently of the + // native client's earlier cached table handle. + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + 0, + 2L, + after.getSchemaId(), + tuple(2L, "new_full", 789L, 0, 0), + tuple(3L, "new_alias", 987L, 0, 0)); + } + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { + TablePath path = TablePath.of(DATABASE, "never_enabled"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.getTableName(), "accepted").offset()).isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception { + TablePath path = TablePath.of(DATABASE, "atomic_conversion"); + byte[] key = new byte[] {1}; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_key", + DataTypes.BYTES().copy(false)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + try (KafkaProducer producer = producer("all", true)) { + int leader = producer.partitionsFor(path.toString()).get(0).leader().id(); + ServerNode node = + CLUSTER.getTabletServerNodes("KAFKA").stream() + .filter(n -> n.id() == leader) + .findFirst() + .get(); + try (Socket socket = connect(node)) { + ProduceResponse rejected = + (ProduceResponse) + sendRequest( + socket, + produceRequest( + path.toString(), + new SimpleRecord( + key, + "valid_prefix" + .getBytes( + StandardCharsets + .UTF_8)), + new SimpleRecord( + null, + "invalid_key" + .getBytes( + StandardCharsets + .UTF_8)))); + assertThat(rejected.errorCounts()).containsEntry(Errors.INVALID_RECORD, 1); + assertThat( + rejected.data() + .responses() + .find(path.toString()) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(-1L); + } + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + key, + "accepted" + .getBytes( + StandardCharsets + .UTF_8))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testReplicatedOffsetsAndMetadataSurviveLeaderLossAndRestart() throws Exception { + TablePath path = TablePath.of(DATABASE, "durable_restart"); + String topic = path.toString(); + String rebalanceId = "kafka-recovered-leader"; + int partition = 1; + int stopped = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3, metadataSchema(), 2); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), partition); + MetadataResponsePartition initial = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + for (int replica : initial.replicaNodes()) { + CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); + } + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, topic, partition, 123L, "before").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition)); + stopped = initial.leaderId(); + CLUSTER.stopTabletServer(stopped); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition) + .leaderId()) + .isNotEqualTo(initial.leaderId())); + assertThat(send(producer, topic, partition, 456L, "after_loss").offset()) + .isEqualTo(1L); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition)); + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); + // Make the recovered replica serve reads and writes, including the record + // produced while it was offline. ISR membership alone does not prove readback. + MetadataResponsePartition current = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + rebalanceLeader(bucket, current, stopped, rebalanceId); + waitForLeaderChange( + admin, topic, partition, stopped, current.leaderEpoch(), rebalanceId); + assertThat( + send( + producer, + path.getTableName(), + partition, + 789L, + "after_restart") + .offset()) + .isEqualTo(2L); + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition), + tuple(2L, "after_restart", 789L, 0, partition)); + } + } + } finally { + if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + } + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void setTableEnabled(Admin admin, TablePath path, boolean enabled) + throws Exception { + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled))), + false) + .get(); + } + + private static Uuid topicId(String topic) { + return waitForTopic(topic).data().topics().find(topic).topicId(); + } + + private static void assertRejectedNames(TablePath path, Errors error) throws Exception { + retry( + Duration.ofMinutes(1), + () -> { + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + MetadataResponse response = + (MetadataResponse) + sendRequest(socket, metadataRequest(topic)); + assertThat(response.errors()).containsEntry(topic, error); + } + } + } + }); + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + ProduceResponse response = + (ProduceResponse) + sendRequest(socket, produceRequest(topic, "rejected")); + assertThat(response.errorCounts()).containsEntry(error, 1); + } + } + } + } + + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + createTable( + admin, + path, + replicas, + Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(), + 1); + } + + private static void createTable( + Admin admin, TablePath path, int replicas, Schema schema, int buckets) + throws Exception { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema(schema) + .distributedBy(buckets) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + } + + private static Schema metadataSchema() { + return Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType()) + .build(); + } + + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + + private static void setMinIsr(Admin admin, int minIsr) throws Exception { + admin.alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), + Integer.toString(minIsr), + AlterConfigOpType.SET))) + .get(30, TimeUnit.SECONDS); + retry( + Duration.ofMinutes(1), + () -> + assertThat(CLUSTER.getTabletServers()) + .allSatisfy( + server -> + assertThat( + server.getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(minIsr))); + } + + private static void assertValues(Connection connection, TablePath path, String... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + List values = new ArrayList<>(); + retry( + Duration.ofSeconds(30), + () -> { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(values.size()); + values.add( + new String( + record.getRow().getBytes(0), StandardCharsets.UTF_8)); + } + assertThat(values).containsExactly(expected); + }); + } + } + + private static void assertRecordsWithMetadata( + Connection connection, + TablePath path, + int partition, + long startOffset, + int schemaId, + Tuple... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribe(partition, startOffset); + List actual = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (actual.size() < expected.length && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getSchemaId()) + .as("Schema at offset %s", record.logOffset()) + .isEqualTo(schemaId); + InternalRow row = record.getRow(); + assertThat(row.getFieldCount()).isEqualTo(2); + assertThat(row.isNullAt(1)).isFalse(); + InternalRow metadata = row.getRow(1, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.isNullAt(2)).isFalse(); + actual.add( + tuple( + record.logOffset(), + new String(row.getBytes(0), StandardCharsets.UTF_8), + metadata.getTimestampLtz(0, 3).getEpochMillisecond(), + metadata.getInt(1), + metadata.getInt(2))); + } + } + assertThat(actual) + .as("Native metadata readback for %s bucket %s", path, partition) + .containsExactly(expected); + } + } + + private static MetadataResponse waitForTopic(String topic) { + return waitForTopic(topic, 1); + } + + private static MetadataResponse waitForTopic(String topic, int partitions) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchMetadata(topic); + MetadataResponseTopic metadata = response.data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(metadata.partitions()) + .hasSize(partitions) + .allSatisfy( + partition -> assertThat(partition.leaderId()).isNotEqualTo(-1)); + result.set(response); + }); + return result.get(); + } + + private static void assertKafkaListenerEndpoints(MetadataResponse response) { + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private static MetadataResponse fetchMetadata(String topic) throws Exception { + try (Socket socket = connect(CLUSTER.getTabletServerNodes("KAFKA").get(0))) { + return (MetadataResponse) sendRequest(socket, metadataRequest(topic)); + } + } + + private static MetadataRequest metadataRequest(String topic) { + return new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + } + + private static ProduceRequest produceRequest(String topic, String value) { + return produceRequest(topic, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + } + + private static ProduceRequest produceRequest(String topic, SimpleRecord... values) { + MemoryRecords records = MemoryRecords.withRecords(Compression.NONE, values); + ProduceRequestData.TopicProduceData topicData = + new ProduceRequestData.TopicProduceData() + .setName(topic) + .setPartitionData( + Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(0) + .setRecords(records))); + return new ProduceRequest( + new ProduceRequestData() + .setAcks((short) 1) + .setTimeoutMs(5000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 11); + } + + private static Socket connect(ServerNode node) throws Exception { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + return socket; + } catch (Exception failure) { + socket.close(); + throw failure; + } + } + + private static AbstractResponse sendRequest(Socket socket, AbstractRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(request.apiKey(), request.version(), "regression-test", 1); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + // The caller owns the socket and may send another request on the same connection. + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void rebalanceLeader( + TableBucket bucket, + MetadataResponsePartition initial, + int newLeader, + String rebalanceId) + throws Exception { + List replicas = new ArrayList<>(initial.replicaNodes()); + Collections.swap(replicas, 0, replicas.indexOf(newLeader)); + RebalancePlanForBucket plan = + new RebalancePlanForBucket( + bucket, initial.leaderId(), newLeader, initial.replicaNodes(), replicas); + RebalanceTask task = + new RebalanceTask( + rebalanceId, + RebalanceStatus.NOT_STARTED, + Collections.singletonMap(bucket, plan)); + // Submit a deterministic leader-only plan through the coordinator event thread. + // All replicas and their client connections remain online during the real handoff. + CLUSTER.getZooKeeperClient().registerRebalanceTask(task); + CLUSTER.getCoordinatorServer() + .getCoordinatorEventProcessor() + .getCoordinatorEventManager() + .put(new RecoverRebalanceEvent(task)); + } + + private static void waitForLeaderChange( + Admin admin, + String topic, + int partition, + int newLeader, + int oldEpoch, + String rebalanceId) { + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()).isGreaterThan(oldEpoch); + }); + } + + private static double producerMetric(KafkaProducer producer, String name) { + return producer.metrics().entrySet().stream() + .filter( + entry -> + entry.getKey().group().equals("producer-metrics") + && entry.getKey().name().equals(name)) + .mapToDouble(entry -> ((Number) entry.getValue().metricValue()).doubleValue()) + .findFirst() + .getAsDouble(); + } + + private static RecordMetadata send( + KafkaProducer producer, String topic, String value) throws Exception { + return producer.send( + new ProducerRecord<>( + topic, 0, null, value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static RecordMetadata send( + KafkaProducer producer, + String topic, + int partition, + long timestamp, + String value) + throws Exception { + return producer.send( + new ProducerRecord<>( + topic, + partition, + timestamp, + null, + value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static KafkaProducer producer(String acks, boolean retryOnFailure) { + Map config = new HashMap<>(); + config.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(","))); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.RETRIES_CONFIG, retryOnFailure ? 20 : 0); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, (int) Duration.ofHours(1).toMillis()); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index bf8e23b4356..9642ed9bfb1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,88 +17,254 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; -import java.util.Map; +import java.nio.ByteBuffer; +import java.util.Collections; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @Test - public void testKafkaApiVersionsNotSupported() { + public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + short unsupportedVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + try { + // An unknown request version must be rejected without parsing its body schema. + processWireRequest( + handler, + requests, + channel, + unsupportedVersion, + new ApiVersionsRequest.Builder().build((short) 4), + 41); + ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); + assertThat(unsupportedResponse.data().errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + // The version error only advertises the range needed to retry ApiVersions. + assertThat(unsupportedResponse.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + + ApiVersionsRequestData data = + new ApiVersionsRequestData() + .setClientSoftwareName("test-client") + .setClientSoftwareVersion("1.0"); + data.unknownTaggedFields().add(new RawTaggedField(42, new byte[] {1, 2, 3})); + processWireRequest( + handler, + requests, + channel, + (short) 4, + new ApiVersionsRequest(data, (short) 4), + 42); + ApiVersionsResponse response = parseWireResponse(channel, (short) 4, 42); + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + assertThat(channel.isActive()).isTrue(); + assertThat(requests.requestsCount()).isZero(); + Object extraResponse = channel.readOutbound(); + assertThat(extraResponse).isNull(); + } finally { + channel.finishAndReleaseAll(); + } + } + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); ChannelHandlerContext ctx = new TestingChannelHandlerContext(); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, - (short) (latestVersion + 1), // unsupported version - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); - ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); + return parseApiVersionsResponse(request); } - @Test - public void testKafkaApiVersionsRequest() { + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), + tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + } + + @ParameterizedTest + @CsvSource({ + "3, invalid client name, 1.0", + "3, client, ''", + "4, invalid client name, 1.0", + "4, client, ''" + }) + public void testInvalidApiVersionsRequest( + short version, String softwareName, String softwareVersion) { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName(softwareName) + .setClientSoftwareVersion(softwareVersion), + version, + version) + .build(version); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, - latestVersion, - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + assertThat(response.data().apiKeys()).isEmpty(); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); + } + + private static void processWireRequest( + KafkaRequestHandler handler, + RequestChannel requests, + EmbeddedChannel channel, + short version, + ApiVersionsRequest body, + int correlationId) { + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, version, "test-client", correlationId); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + channel.writeInbound(buffer); + KafkaRequest request = (KafkaRequest) requests.pollRequest(1000); + assertThat(request).isNotNull(); + try { + assertThat(request.apiVersion()).isEqualTo(version); + handler.processRequest(request); + assertThat(request.future()).isDone(); + channel.runPendingTasks(); + assertThat(buffer.refCnt()).isZero(); + } finally { + request.releaseBuffer(); + } + } + + private static ApiVersionsResponse parseWireResponse( + EmbeddedChannel channel, short version, int correlationId) { + ByteBuf buffer = channel.readOutbound(); + assertThat(buffer).isNotNull(); + try { + ByteBuffer bytes = buffer.nioBuffer(); + assertThat(ResponseHeader.parse(bytes, (short) 0).correlationId()) + .isEqualTo(correlationId); + ApiVersionsResponseData data = + new ApiVersionsResponseData(new ByteBufferAccessor(bytes), version); + assertThat(bytes.hasRemaining()).isFalse(); + return new ApiVersionsResponse(data); + } finally { + buffer.release(); + } } private static KafkaRequest newRequest( @@ -134,6 +300,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(new Configuration(), service); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java new file mode 100644 index 00000000000..9d7912b5b42 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Tests registry-backed capability advertisement by {@link ApiVersionsHandler}. */ +class ApiVersionsHandlerTest { + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { + AtomicReference registryReference = new AtomicReference<>(); + ApiVersionsHandler handler = + new ApiVersionsHandler( + () -> { + assertThat(registryReference.get()).isNotNull(); + return registryReference.get(); + }); + TestingApiHandler metadataHandler = + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true)); + KafkaApiRegistry registry = + new KafkaApiRegistry( + Arrays.asList( + handler, + new ProduceHandler( + command -> { + throw new AssertionError( + "Capability advertisement must not invoke the Produce backend."); + }), + metadataHandler, + new TestingApiHandler( + new KafkaApiSpec( + ApiKeys.FETCH, (short) 0, (short) 3, false)))); + registryReference.set(registry); + + assertThat(registry.lookup(ApiKeys.API_VERSIONS)).isSameAs(handler); + assertThat(registry.lookup(ApiKeys.METADATA)).isSameAs(metadataHandler); + assertThat(registry.lookup(ApiKeys.FETCH)).isNull(); + + ApiVersionsResponse response = + (ApiVersionsResponse) + handler.handle(null, new ApiVersionsRequest.Builder().build(version)) + .join(); + + assertThat(response.data().errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), + tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + } + + private static final class TestingApiHandler implements KafkaApiHandler { + private final KafkaApiSpec spec; + + private TestingApiHandler(KafkaApiSpec spec) { + this.spec = spec; + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, AbstractRequest request) { + throw new AssertionError( + "Capability advertisement must not invoke other API handlers."); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java new file mode 100644 index 00000000000..039f0ebcfd5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -0,0 +1,1065 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.AuthorizationException; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.protocol.Errors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests append admission, error isolation, session propagation and asynchronous acknowledgements. + */ +class GatewayKafkaProduceBackendTest { + + @Test + void testAllNullValuesUseNativeAppendAndAuthorization() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) + .join(); + assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.writeChecks).isEqualTo(1); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); + service.denyWrite = true; + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0)))) + .join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.appends).hasSize(1); + assertThat(service.writeChecks).isEqualTo(2); + } + + @ParameterizedTest + @ValueSource(strings = {"non-null", "all-null", "mixed"}) + void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) + throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + PartitionWrite partition = + recordsKind.equals("non-null") + ? good(0) + : recordsKind.equals("all-null") + ? nullValue(0) + : new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + good(0).records().get(0))); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", partition)); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + int appendedBeforeRevocation = service.appends.size(); + service.denyWrite = true; + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(2); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(service.appends).hasSize(appendedBeforeRevocation); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { + TestingProduceService service = new TestingProduceService(); + for (String name : new String[] {"invalid", "missing"}) { + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka." + name, nullValue(0)))) + .join(), + name.equals("invalid") + ? Errors.INVALID_TOPIC_EXCEPTION + : Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(service.writeChecks).isZero(); + assertThat(service.append).isNull(); + } + + @Test + void testMixedNullValuesPreserveRecordOrderAndNativeOffsets() throws Exception { + TestingProduceService service = new TestingProduceService(); + List seen = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, table) -> { + seen.addAll(records); + return transcoder.transcode(partitionId, records, table); + }); + Record nullRecord = nullValue(0).records().get(0); + Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); + Record valid = good(0).records().get(0); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic( + "kafka.topic", + new PartitionWrite( + 0, + Arrays.asList(nullRecord, empty, valid))))) + .join(); + assertErrors(result, Errors.NONE); + assertThat(seen).containsExactly(nullRecord, empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + } + + private static PartitionWrite nullValue(int partition) { + return new PartitionWrite( + partition, + Collections.singletonList( + new Record(-1L, new byte[] {(byte) 0xff}, null, Collections.emptyList()))); + } + + @Test + void testAcksAllWaitsForNativeAppendResponse() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.append.getAcks()).isEqualTo(-1); + assertThat(service.append.getTimeoutMs()).isEqualTo(4321); + assertThat(service.append.getTableId()).isEqualTo(42L); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotDone(); + service.pendingAppend.complete(success(0)); + assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + } + + @Test + void testMapsSynchronousAppendFailure() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.throwAppend = true; + assertErrors( + backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception { + TestingProduceService service = new TestingProduceService(); + PartitionWrite malformed = + new PartitionWrite( + 0, + Arrays.asList( + good(0).records().get(0), + new Record( + 1L, + null, + new byte[] {(byte) 0xff}, + Collections.emptyList()))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.topic", malformed, good(1), good(8)))) + .join(); + assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + } + + @Test + void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { + for (boolean async : new boolean[] {false, true}) { + TestingProduceService service = new TestingProduceService(); + service.asyncMissing = async; + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.invalid", good(0)), + topic("kafka.missing", good(0)), + topic("kafka.topic", good(0)))) + .join(); + assertThat(result.topics()).hasSize(3); + assertThat(result.topics().get(0).partitions().get(0).error()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION); + assertThat(result.topics().get(1).partitions().get(0).error()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); + } + } + + @Test + void testNoAppendWhenAllPartitionsFailConversion() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); + assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append).isNull(); + } + + @Test + void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Exception { + org.apache.fluss.rpc.protocol.Errors[] flussErrors = { + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER, + org.apache.fluss.rpc.protocol.Errors.CORRUPT_RECORD_EXCEPTION + }; + Errors[] kafkaErrors = { + Errors.NOT_ENOUGH_REPLICAS, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + Errors.REQUEST_TIMED_OUT, + Errors.NOT_LEADER_OR_FOLLOWER, + Errors.CORRUPT_MESSAGE + }; + for (int i = 0; i < flussErrors.length; i++) { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = + CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setErrorCode(flussErrors[i].code()), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode(0) + .setBaseOffset(17L)))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) -1, + topic("kafka.topic", good(0), good(1), good(2)))) + .join(); + assertErrors(result, kafkaErrors[i], Errors.NONE, Errors.UNKNOWN_SERVER_ERROR); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + + @Test + void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) 0, topic("kafka.topic", good(0)))); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); + assertErrors(result.join(), Errors.NONE); + assertThat(service.append.getAcks()).isZero(); + } + + @Test + void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exception { + TestingProduceService service = new TestingProduceService(); + List transcodedPaths = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, tableInfo) -> { + transcodedPaths.add(tableInfo.getTablePath()); + return transcoder.transcode(partitionId, records, tableInfo); + }); + + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("kafka.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + + assertThat(service.metadataPaths) + .containsExactly(TablePath.of("kafka", "topic"), TablePath.of("other", "topic")); + assertThat(transcodedPaths).containsExactlyElementsOf(service.metadataPaths); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 43L); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("kafka.topic", "other.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + } + + @Test + void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic(".", good(0)), + topic("kafka.topic.extra", good(0)), + topic("kafka.__internal", good(0)), + topic("kafka.topic", good(0)))) + .join(); + + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.NONE); + assertThat(service.metadataPaths).containsExactly(TablePath.of("kafka", "topic")); + assertThat(service.appends).hasSize(1); + } + + @Test + void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() throws Exception { + TestingProduceService service = new TestingProduceService(); + ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + List prepared = new ArrayList<>(); + List converted = new ArrayList<>(); + List convertedPartitions = new ArrayList<>(); + KafkaRecordTranscoder transcoder = + new KafkaRecordTranscoder() { + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + KafkaTopicWritePlan plan = arrow.prepare(tableInfo); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode( + int partitionId, List records, TableInfo tableInfo) + throws Exception { + return arrow.transcode(partitionId, records, tableInfo); + } + + @Override + public BytesView transcode( + int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + converted.add(plan); + convertedPartitions.add(partitionId); + return arrow.transcode(partitionId, records, plan); + } + }; + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); + assertThat(service.metadataPaths).hasSize(2); + assertThat(prepared).hasSize(2); + assertThat(prepared.get(1)).isSameAs(prepared.get(0)); + assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + assertThat(convertedPartitions).containsExactly(0, 1, 0); + + service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.metadataPaths).hasSize(3); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally( + new org.apache.fluss.exception.AuthorizationException("revoked")); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(2); + } + + @Test + void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 4); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingAppend = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + service.appended = new CountDownLatch(2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture first = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + backend.write(command((short) -1, topic("kafka.topic", good(1)))); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))) + .join(); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(service.appends) + .extracting(request -> request.getBucketsReqsList().get(0).getBucketId()) + .containsExactly(0, 1); + assertThat(first).isNotDone(); + assertThat(second).isNotDone(); + service.pendingAppend.complete( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(17L), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setBaseOffset(18L)))); + assertErrors(first.get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(second.get(10, TimeUnit.SECONDS), Errors.NONE); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + List snapshots = new ArrayList<>(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, table) -> { + snapshots.add(table); + return transcoder.transcode(partitionId, records, table); + }, + executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + TableDescriptor changedSchema = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableId(84L) + .setSchemaId(2) + .setBucketCountEpoch(7L) + .setTableJson(changedSchema.toJsonBytes())); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertThat(snapshots).hasSize(3); + assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); + assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); + assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); + assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getRowType().getTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 84L); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally(new TableNotExistException("deleted")); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(3); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + service.appended = new CountDownLatch(1); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture result = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotCancelled(); + service.pendingAppend.complete(success(0)); + KafkaProduceResult completed = result.get(10, TimeUnit.SECONDS); + assertErrors(completed, Errors.NONE); + assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + CompletableFuture running = backend.write(command); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = backend.write(command); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertErrors(running.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors(queued.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + service.pendingMetadata.complete(metadata(false)); + assertThat(service.appends).isEmpty(); + assertThat(service.metadataPaths).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testAliasesSharePlansAndAuthorizationButRetainRequestNames() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 4); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder, executor); + try { + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic", good(0)))) + .get(10, TimeUnit.SECONDS); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("topic", "kafka.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("kafka", "topic"), TablePath.of("kafka", "topic")); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(0)).isSameAs(transcoder.prepared.get(1)); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDisabledAndResetFlagsRejectBothNamesBeforePrepare(boolean allNull) throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + PartitionWrite partition = allNull ? nullValue(0) : good(0); + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), Errors.NONE); + for (boolean reset : new boolean[] {false, true}) { + TableDescriptor enabled = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + java.util.Map properties = + new java.util.HashMap<>(enabled.getProperties()); + if (reset) { + properties.remove(ConfigOptions.TABLE_KAFKA_ENABLED.key()); + } else { + properties.put(ConfigOptions.TABLE_KAFKA_ENABLED.key(), "false"); + } + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + enabled.withProperties(properties).toJsonBytes())); + int prepared = transcoder.prepared.size(); + int appended = service.appends.size(); + int authorized = service.writeChecks; + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, partition))).join(), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(transcoder.prepared).hasSize(prepared); + assertThat(service.appends).hasSize(appended); + assertThat(service.writeChecks).isEqualTo(authorized); + service.pendingMetadata = CompletableFuture.completedFuture(metadata(false)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", partition))).join(), + Errors.NONE); + assertThat(transcoder.prepared.get(prepared)) + .isNotSameAs(transcoder.prepared.get(prepared - 1)); + } + assertThat(service.metadataPaths).hasSize(7); + } + + @Test + void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + assertErrors( + backend.write(command((short) 1, topic("topic", good(0)))).join(), Errors.NONE); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + for (TableDescriptor invalid : + Arrays.asList( + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build())) { + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(invalid.toJsonBytes())); + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } + } + assertThat(service.appends).hasSize(1); + assertThat(transcoder.prepared).hasSize(1); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "STRING") + .build() + .toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.NONE); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(1)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(1).tableInfo().getSchemaId()).isEqualTo(1); + assertThat(service.appends).hasSize(2); + assertThat(service.metadataPaths).hasSize(8); + } + + @Test + void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { + TestingProduceService service = new TestingProduceService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_db"); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + new KafkaTopicMapper(config), + new ArrowKafkaRecordTranscoder()); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("missing_db.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.NONE); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("missing_db", "topic"), + TablePath.of("missing_db", "topic"), + TablePath.of("other", "topic")); + assertThat(service.appends).extracting(ProduceLogRequest::getTableId).containsExactly(43L); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testInvalidRecordDoesNotAppendValidPrefixWithMetadata(boolean malformedUtf8) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema(schema) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + Record invalid = + malformedUtf8 + ? new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList()) + : good(0).records().get(0); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 2, Arrays.asList(valid, invalid))))) + .join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 1, Collections.singletonList(valid))))) + .join(), + Errors.NONE); + assertThat(service.appends).hasSize(1); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String format) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor notNull = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (format.equals("binary") + ? DataTypes.BYTES() + : DataTypes.STRING()) + .copy(false)) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(notNull.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + + Record valid = good(0).records().get(0); + PartitionWrite invalid = + new PartitionWrite(0, Arrays.asList(valid, nullValue(0).records().get(0), valid)); + assertErrors( + backend.write(command((short) 1, topic("topic", invalid, good(1)))).join(), + Errors.INVALID_RECORD, + Errors.NONE); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + } + + @Test + void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor unsupportedMetadata = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()), + DataTypes.FIELD( + "offset", DataTypes.BIGINT()))) + .build()) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(unsupportedMetadata.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + TableDescriptor json = + TableDescriptor.builder(original) + .schema(Schema.newBuilder().column("business", DataTypes.STRING()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertThat(service.writeChecks).isZero(); + } + + private static final class RecordingTranscoder implements KafkaRecordTranscoder { + private final ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + private final List prepared = new ArrayList<>(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo table) { + KafkaTopicWritePlan plan = arrow.prepare(table); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo table) + throws Exception { + return arrow.transcode(partitionId, records, table); + } + } + + private static KafkaTopicMapper mapper() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka"); + return new KafkaTopicMapper(config); + } + + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { + return new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder()); + } + + private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { + return new KafkaProduceCommand( + acks, 4321, Arrays.asList(topics), "KAFKA", InetAddress.getLoopbackAddress()); + } + + private static TopicWrite topic(String name, PartitionWrite... partitions) { + return new TopicWrite(name, Arrays.asList(partitions)); + } + + private static PartitionWrite good(int id) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record(123L, null, new byte[] {65}, Collections.emptyList()))); + } + + private static GetTableInfoResponse metadata(boolean invalid) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); + if (!invalid) { + descriptor.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string"); + } + return new GetTableInfoResponse() + .setTableId(42L) + .setSchemaId(1) + .setTableJson(descriptor.build().toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static ProduceLogResponse success(int id) { + return new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(id) + .setBaseOffset(17L))); + } + + private static void assertErrors(KafkaProduceResult result, Errors... errors) { + List partitions = result.topics().get(0).partitions(); + assertThat(partitions).extracting(PartitionResult::error).containsExactly(errors); + for (PartitionResult partition : partitions) { + if (partition.error() != Errors.NONE) { + assertThat(partition.baseOffset()).isEqualTo(-1L); + } + } + } + + private static class TestingProduceService extends TestingTabletGatewayService { + private final List metadataPaths = new ArrayList<>(); + private final List appends = new ArrayList<>(); + private ProduceLogRequest append; + private CompletableFuture pendingAppend; + private CompletableFuture pendingMetadata; + private boolean throwAppend; + private boolean asyncMissing; + private int writeChecks; + private boolean denyWrite; + private CountDownLatch metadataEntered; + private CountDownLatch appended; + private Thread metadataThread; + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getTablePath().getDatabaseName(); + String name = request.getTablePath().getTableName(); + metadataPaths.add(TablePath.of(database, name)); + metadataThread = Thread.currentThread(); + if (metadataEntered != null) { + metadataEntered.countDown(); + } + if (database.equals("missing_db")) { + throw new DatabaseNotExistException("missing_db"); + } + if (name.equals("missing")) { + if (!asyncMissing) { + throw new TableNotExistException("missing"); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new TableNotExistException("missing")); + return failed; + } + return pendingMetadata == null + ? CompletableFuture.completedFuture( + GatewayKafkaProduceBackendTest.metadata(name.equals("invalid")) + .setTableId(database.equals("other") ? 43L : 42L)) + : pendingMetadata; + } + + private void checkWriteAuthorization(long tableId) { + assertThat(tableId).isPositive(); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + writeChecks++; + if (denyWrite) { + throw new AuthorizationException("Write denied"); + } + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + checkWriteAuthorization(request.getTableId()); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + append = request; + appends.add(request); + if (throwAppend) { + throw new IllegalStateException("append failed"); + } + if (appended != null) { + appended.countDown(); + } + return pendingAppend == null + ? CompletableFuture.completedFuture( + success(request.getBucketsReqsList().get(0).getBucketId())) + : pendingAppend; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java new file mode 100644 index 00000000000..6243bb8a900 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests timestamp provenance and ownership of protocol-independent Produce commands. */ +class KafkaProduceCommandTest { + + @ParameterizedTest + @ValueSource(longs = {-1L, 0L, 123L}) + void testLegacyConstructorDoesNotInferBatchType(long timestamp) { + Record record = new Record(timestamp, null, null, Collections.emptyList()); + + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + assertThat(record.timestampType()).isEqualTo(TimestampType.UNKNOWN); + assertThat(record.key()).isNull(); + assertThat(record.value()).isNull(); + } + + @ParameterizedTest + @EnumSource(TimestampType.class) + void testTimestampPresenceDoesNotChangeExplicitBatchType(TimestampType timestampType) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, 123L}) { + Record record = + new Record(timestamp, timestampType, null, null, Collections.emptyList()); + + assertThat(record.timestampType()).isEqualTo(timestampType); + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + } + } + + @Test + void testExplicitBatchTypeMustNotBeNull() { + assertThatThrownBy(() -> new Record(123L, null, null, null, Collections.emptyList())) + .isInstanceOf(NullPointerException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBothConstructorsOwnTheirRecordData(boolean explicitTimestampType) { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + List headers = + new ArrayList<>( + Arrays.asList( + new RecordHeader("header", headerValue), + new RecordHeader("header", null))); + Record record = + explicitTimestampType + ? new Record(123L, TimestampType.LOG_APPEND_TIME, key, value, headers) + : new Record(123L, key, value, headers); + key[0] = 4; + value[0] = 5; + headerValue[0] = 6; + headers.clear(); + record.key()[0] = 7; + record.value()[0] = 8; + record.headers().get(0).value()[0] = 9; + + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()) + .extracting(RecordHeader::name) + .containsExactly("header", "header"); + assertThat(record.headers().get(0).value()).containsExactly((byte) 3); + assertThat(record.headers().get(1).value()).isNull(); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()) + .isEqualTo( + explicitTimestampType + ? TimestampType.LOG_APPEND_TIME + : TimestampType.UNKNOWN); + assertThatThrownBy(() -> record.headers().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { + List records = + new ArrayList<>( + Arrays.asList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + null, + Collections.emptyList()), + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList()), + new Record( + Record.NO_TIMESTAMP, null, null, Collections.emptyList()))); + List partitions = + new ArrayList<>(Collections.singletonList(new PartitionWrite(2, records))); + List topics = + new ArrayList<>(Collections.singletonList(new TopicWrite("db.topic", partitions))); + KafkaProduceCommand command = + new KafkaProduceCommand((short) -1, 1000, topics, "KAFKA", null); + records.clear(); + partitions.clear(); + topics.clear(); + + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(1000); + assertThat(command.topics()).extracting(TopicWrite::topicName).containsExactly("db.topic"); + assertThat(command.topics().get(0).partitions()) + .extracting(PartitionWrite::partitionId) + .containsExactly(2); + List copied = command.topics().get(0).partitions().get(0).records(); + assertThat(copied) + .extracting(Record::timestampType) + .containsExactly( + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.UNKNOWN); + assertThat(copied) + .extracting(Record::timestamp) + .containsExactly(123L, 123L, Record.NO_TIMESTAMP); + assertThatThrownBy(() -> command.topics().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> command.topics().get(0).partitions().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testNullValuesPreserveRecordIdentityAndTimestampProvenance() { + Record nullRecord = + new Record( + 123L, + TimestampType.CREATE_TIME, + new byte[] {1}, + null, + Collections.emptyList()); + Record retained = + new Record( + 456L, + TimestampType.LOG_APPEND_TIME, + new byte[] {2}, + new byte[] {3}, + Collections.emptyList()); + Record empty = + new Record( + Record.NO_TIMESTAMP, + TimestampType.CREATE_TIME, + null, + new byte[0], + Collections.emptyList()); + List source = new ArrayList<>(Arrays.asList(nullRecord, retained, empty)); + PartitionWrite partition = new PartitionWrite(0, source); + source.clear(); + + assertThat(partition.records()).containsExactly(nullRecord, retained, empty); + assertThat(partition.records().get(0)).isSameAs(nullRecord); + assertThat(partition.records().get(0).value()).isNull(); + assertThat(partition.records().get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(0).timestamp()).isEqualTo(123L); + assertThat(partition.records().get(1)).isSameAs(retained); + assertThat(partition.records().get(1).timestampType()) + .isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(partition.records().get(1).timestamp()).isEqualTo(456L); + assertThat(partition.records().get(2).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(2).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.records().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + RecordHeader header = new RecordHeader("header", headerValue); + List headers = new ArrayList<>(); + headers.add(header); + Record record = new Record(1L, key, value, headers); + + assertThat(record.borrowedKey()).isNotSameAs(key); + assertThat(record.borrowedValue()).isNotSameAs(value); + assertThat(header.borrowedValue()).isNotSameAs(headerValue); + key[0] = 10; + value[0] = 20; + headerValue[0] = 30; + headers.clear(); + + assertThat(record.borrowedKey()).containsExactly((byte) 1); + assertThat(record.borrowedValue()).containsExactly((byte) 2); + assertThat(header.borrowedValue()).containsExactly((byte) 3); + assertThat(record.headers()).containsExactly(header); + assertThat(record.borrowedKey()).isSameAs(record.borrowedKey()); + assertThat(record.borrowedValue()).isSameAs(record.borrowedValue()); + assertThat(header.borrowedValue()).isSameAs(header.borrowedValue()); + + record.key()[0] = 40; + record.value()[0] = 50; + header.value()[0] = 60; + assertThat(record.key()).containsExactly((byte) 1).isNotSameAs(record.borrowedKey()); + assertThat(record.value()).containsExactly((byte) 2).isNotSameAs(record.borrowedValue()); + assertThat(header.value()).containsExactly((byte) 3).isNotSameAs(header.borrowedValue()); + } + + @Test + void testBorrowedPayloadsPreserveNullAndEmptyBytes() { + Record nullRecord = new Record(1L, null, null, Collections.emptyList()); + assertThat(nullRecord.borrowedKey()).isNull(); + assertThat(nullRecord.borrowedValue()).isNull(); + assertThat(new RecordHeader("header", null).borrowedValue()).isNull(); + + Record emptyRecord = new Record(1L, new byte[0], new byte[0], Collections.emptyList()); + assertThat(emptyRecord.borrowedKey()).isEmpty(); + assertThat(emptyRecord.borrowedValue()).isEmpty(); + assertThat(new RecordHeader("header", new byte[0]).borrowedValue()).isEmpty(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java new file mode 100644 index 00000000000..88e95f420b6 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests bounded admission, ordering, independent lanes and executor shutdown. */ +class KafkaProduceConversionExecutorTest { + @Test + void testBoundedLanePreservesOrderAndDoesNotBlockAnotherLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 2); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + List order = new ArrayList<>(); + AtomicReference worker = new AtomicReference<>(); + AtomicBoolean rejectedRan = new AtomicBoolean(); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + worker.set(Thread.currentThread()); + started.countDown(); + await(release); + order.add(1); + return CompletableFuture.completedFuture(1); + }); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + executor.submit( + 0, + () -> { + order.add(2); + return CompletableFuture.completedFuture(2); + }); + CompletableFuture rejected = + executor.submit( + 0, + () -> { + rejectedRan.set(true); + return CompletableFuture.completedFuture(3); + }); + assertThatThrownBy(rejected::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(rejectedRan).isFalse(); + assertThat(worker.get()).isNotSameAs(Thread.currentThread()); + assertThat(second).isNotDone(); + assertThat( + executor.submit(1, () -> CompletableFuture.completedFuture(4)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(4); + release.countDown(); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + assertThat(second.get(10, TimeUnit.SECONDS)).isEqualTo(2); + assertThat(order).containsExactly(1, 2); + } finally { + release.countDown(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testPendingAcknowledgementDoesNotOccupyWorker() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CompletableFuture acknowledgement = new CompletableFuture<>(); + CountDownLatch submitted = new CountDownLatch(1); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + submitted.countDown(); + return acknowledgement; + }); + assertThat(submitted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(2)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(2); + assertThat(first).isNotDone(); + acknowledgement.complete(1); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testCloseInterruptsWorkerCompletesQueuedRequestsAndRejectsNewWork() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CountDownLatch started = new CountDownLatch(1); + AtomicBoolean queuedRan = new AtomicBoolean(); + CompletableFuture running = + executor.submit( + 0, + () -> { + started.countDown(); + await(new CountDownLatch(1)); + return CompletableFuture.completedFuture(null); + }); + try { + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = + executor.submit( + 0, + () -> { + queuedRan.set(true); + return CompletableFuture.completedFuture(null); + }); + CompletableFuture closed = executor.closeAsync(); + assertThat(executor.closeAsync()).isSameAs(closed); + closed.get(10, TimeUnit.SECONDS); + assertThatThrownBy(running::join).hasRootCauseInstanceOf(InterruptedException.class); + assertThatThrownBy(queued::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(queuedRan).isFalse(); + assertThatThrownBy( + () -> + executor.submit( + 0, + () -> CompletableFuture.completedFuture(null)) + .join()) + .hasCauseInstanceOf(RejectedExecutionException.class); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testFailedOperationDoesNotStopLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + try { + CompletableFuture failed = + executor.submit( + 0, + () -> { + throw new IllegalStateException("conversion failed"); + }); + assertThatThrownBy(failed::join).hasCauseInstanceOf(IllegalStateException.class); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(1)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Test worker was not released."); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(failure); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java new file mode 100644 index 00000000000..3ee3a125495 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaTableInfoCacheTest { + + private static final TablePath TABLE_PATH = TablePath.of("kafka", "orders"); + + @Test + void testReusesUnchangedResponseAndReloadsPropertyOnlyChange() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse firstResponse = response(1L, 1, descriptor("v1")); + + TableInfo first = + cache.getOrLoad( + TABLE_PATH, + firstResponse, + () -> load(TABLE_PATH, firstResponse, loadCount)); + GetTableInfoResponse equivalentResponse = response(1L, 1, descriptor("v1")); + TableInfo reused = + cache.getOrLoad( + TABLE_PATH, + equivalentResponse, + () -> load(TABLE_PATH, equivalentResponse, loadCount)); + + assertThat(reused).isSameAs(first); + assertThat(loadCount).hasValue(1); + + // A Kafka contract property may change without a schema ID or timestamp change. + GetTableInfoResponse changedResponse = response(1L, 1, descriptor("v2")); + TableInfo changed = + cache.getOrLoad( + TABLE_PATH, + changedResponse, + () -> load(TABLE_PATH, changedResponse, loadCount)); + + assertThat(changed).isNotSameAs(first); + assertThat(loadCount).hasValue(2); + } + + @Test + void testReloadsDeleteAndRecreateAndSupportsExplicitInvalidation() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse original = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, original, () -> load(TABLE_PATH, original, loadCount)); + + GetTableInfoResponse recreated = response(2L, 1, descriptor("v1")); + TableInfo second = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(second).isNotSameAs(first); + + cache.invalidate(TABLE_PATH); + TableInfo third = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(third).isNotSameAs(second); + assertThat(loadCount).hasValue(3); + } + + @Test + void testReloadsWhenOnlyBucketCountEpochChanges() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loads = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + response.setBucketCountEpoch(7L); + TableInfo changed = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + assertThat(changed).isNotSameAs(first); + assertThat(changed.getBucketCountEpoch()).isEqualTo(7L); + assertThat(loads).hasValue(2); + } + + @Test + void testLoadFailureIsNotCached() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger attempts = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + + assertThatThrownBy( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + attempts.incrementAndGet(); + throw new IllegalStateException("broken descriptor"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("broken descriptor"); + + TableInfo loaded = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, attempts)); + assertThat(loaded.getTableId()).isEqualTo(1L); + assertThat(attempts).hasValue(2); + } + + @Test + void testConcurrentColdLoadIsSingleFlight() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + }))); + } + loaderStarted.await(); + releaseLoader.countDown(); + + TableInfo first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(loadCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightLoadToResurrect() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future loading = + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + })); + loaderStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(TABLE_PATH)); + releaseLoader.countDown(); + + TableInfo loadedBeforeInvalidation = loading.get(); + invalidating.get(); + TableInfo loadedAfterInvalidation = + cache.getOrLoad( + TABLE_PATH, response, () -> load(TABLE_PATH, response, loadCount)); + + assertThat(loadedAfterInvalidation).isNotSameAs(loadedBeforeInvalidation); + assertThat(loadCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(2); + for (int i = 0; i < 10; i++) { + TablePath path = TablePath.of("kafka", "topic_" + i); + GetTableInfoResponse response = response(i + 1L, 1, descriptor("v1")); + cache.getOrLoad(path, response, () -> toTableInfo(path, response)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static TableInfo load( + TablePath tablePath, GetTableInfoResponse response, AtomicInteger loadCount) { + loadCount.incrementAndGet(); + return toTableInfo(tablePath, response); + } + + private static TableInfo toTableInfo(TablePath tablePath, GetTableInfoResponse response) { + return TableInfo.of( + tablePath, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static GetTableInfoResponse response( + long tableId, int schemaId, TableDescriptor descriptor) { + return new GetTableInfoResponse() + .setTableId(tableId) + .setSchemaId(schemaId) + .setTableJson(descriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static TableDescriptor descriptor(String version) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + if ("v2".equals(version)) { + builder.property(ConfigOptions.TABLE_KAFKA_ENABLED, false); + } + return builder.build(); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test loader.", e); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java new file mode 100644 index 00000000000..b4b028ba0c5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests value decoders against the shared fixed-column table contract. */ +class KafkaFormatFactoryRegistryTest { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + + @Test + void testBinaryBorrowsOpaqueBytesAndPreservesNullAndEmpty() { + KafkaFieldDecoder decoder = + formats.createDecoder(ValueType.BINARY, projection(DataTypes.BYTES())); + byte[] opaque = new byte[] {(byte) 0xff, 0}; + assertThat(decoder.decode(opaque)[0]).isSameAs(opaque); + assertThat(opaque).containsExactly((byte) 0xff, (byte) 0); + assertThat(decoder.decode(null)).containsExactly((Object) null); + byte[] empty = new byte[0]; + assertThat(decoder.decode(empty)[0]).isSameAs(empty); + } + + @Test + void testStringRejectsMalformedUtf8AndCanBeReused() { + KafkaFieldDecoder decoder = + formats.createDecoder(ValueType.STRING, projection(DataTypes.STRING())); + for (byte[] invalid : + new byte[][] { + {(byte) 0xc3, 0x28}, + {(byte) 0xc0, (byte) 0x80}, + {(byte) 0xed, (byte) 0xa0, (byte) 0x80}, + {(byte) 0xe2, (byte) 0x82} + }) { + byte[] original = invalid.clone(); + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("_kafka_value") + .hasMessageContaining("UTF-8"); + assertThat(invalid).isEqualTo(original); + byte[] valid = "键".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(valid)).containsExactly(BinaryString.fromString("键")); + assertThat(valid).isEqualTo("键".getBytes(StandardCharsets.UTF_8)); + } + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThat(decoder.decode(new byte[0])).containsExactly(BinaryString.fromString("")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesUseThePublicValueProjection(boolean string) { + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()) + .column(KafkaTableSchema.VALUE_COLUMN, type) + .column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string ? "string" : "binary") + .build(); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + assertThat(schema.valueProjection().positions()).containsExactly(1); + KafkaFieldDecoder decoder = + formats.createDecoder(schema.valueType(), schema.valueProjection()); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(payload)) + .containsExactly(string ? BinaryString.fromString("payload") : payload); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectIncompatibleProjections(boolean string) { + ValueType format = string ? ValueType.STRING : ValueType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + assertThatThrownBy( + () -> + formats.createDecoder( + format, + projection( + string ? DataTypes.BYTES() : DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(string ? "STRING" : "BYTES"); + KafkaFieldProjection multiple = + new KafkaFieldProjection( + DataTypes.ROW( + DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type), + DataTypes.FIELD(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES())), + Arrays.asList(0, 1)); + KafkaFieldProjection empty = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.emptyList()); + for (KafkaFieldProjection invalid : Arrays.asList(multiple, empty)) { + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("exactly one"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectKeyAndArbitraryValueColumnNames(boolean string) { + ValueType format = string ? ValueType.STRING : ValueType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + for (String name : Arrays.asList(KafkaTableSchema.KEY_COLUMN, "body")) { + KafkaFieldProjection invalid = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(name, type)), + Collections.singletonList(0)); + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + } + } + + @Test + void testJsonHasNoDecoderUntilItsImplementationIsRegistered() { + assertThatThrownBy( + () -> + formats.createDecoder( + ValueType.PARSE_JSON, projection(DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("PARSE_JSON"); + } + + private static KafkaFieldProjection projection(DataType type) { + return new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.singletonList(0)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..42c45627319 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + private final KafkaTopicMapper mapper = new KafkaTopicMapper(); + + @Test + public void testQualifiedNamesAcrossDatabases() { + for (String database : Arrays.asList("sales", "archive", "sales_1-2026")) { + TablePath tablePath = TablePath.of(database, "orders_1-2026"); + String topicName = database + ".orders_1-2026"; + assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); + assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); + assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); + } + } + + @Test + void testDefaultDatabaseAliases() { + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("fluss", "orders")); + assertThat(mapper.toTopicName(mapper.toTablePath("orders"))).isEqualTo("fluss.orders"); + assertThat(mapper.toTablePath("kafka.orders")).isEqualTo(TablePath.of("kafka", "orders")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper custom = new KafkaTopicMapper(config); + assertThat(custom.toTablePath("orders")).isEqualTo(TablePath.of("sales", "orders")); + assertThat(custom.toTablePath("warehouse.orders")) + .isEqualTo(TablePath.of("warehouse", "orders")); + assertThat(custom.toTopicName(custom.toTablePath("orders"))).isEqualTo("sales.orders"); + config.setString("kafka.database", "legacy"); + assertThat(new KafkaTopicMapper(config).toTablePath("orders")) + .isEqualTo(TablePath.of("sales", "orders")); + } + + @Test + void testInvalidDefaultDatabaseAndCanonicalAliasLength() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "bad.name"); + assertThatThrownBy(() -> new KafkaTopicMapper(config)).hasMessageContaining("invalid"); + config.set( + ConfigOptions.KAFKA_DEFAULT_DATABASE, + String.join("", Collections.nCopies(128, "d"))); + KafkaTopicMapper longDatabase = new KafkaTopicMapper(config); + assertThatThrownBy( + () -> + longDatabase.toTablePath( + String.join("", Collections.nCopies(128, "t")))) + .isInstanceOf(InvalidTopicException.class); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + ".orders", + "sales.", + "sales.orders.extra", + "sales..orders", + "sales.bad name", + " sales.orders", + "sales.orders ", + "sales/orders", + "__system.orders", + "sales.__internal", + "数据库.orders", + "sales.订单" + }) + public void testRejectsInvalidTopicNames(String topicName) { + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(topicName)) + .isInstanceOf(InvalidTopicException.class); + } + + @Test + public void testNameLengthLimits() { + for (TablePath valid : + Arrays.asList( + TablePath.of(repeat('a', 200), "orders"), + TablePath.of("sales", repeat('a', 200)), + TablePath.of(repeat('a', 124), repeat('b', 124)))) { + assertThat(mapper.toTablePath(mapper.toTopicName(valid))).isEqualTo(valid); + } + for (TablePath invalid : + Arrays.asList( + TablePath.of(repeat('a', 201), "orders"), + TablePath.of("sales", repeat('a', 201)), + TablePath.of(repeat('a', 124), repeat('b', 125)))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testRejectsUnrepresentableTablePaths() { + for (TablePath invalid : + Arrays.asList( + null, + TablePath.of(null, "orders"), + TablePath.of("sales", null), + TablePath.of("", "orders"), + TablePath.of("sales", ""), + TablePath.of("sales.region", "orders"), + TablePath.of("sales", "orders.v1"), + TablePath.of("__system", "orders"), + TablePath.of("sales", "__internal"), + TablePath.of("sales", "bad name"))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testTopicIdMapping() { + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testAliasTopicIdsKeepTableIdentityAndDetectRecreation() { + for (long tableId : new long[] {0L, 123L, Long.MAX_VALUE}) { + Uuid canonical = mapper.toTopicId(tableId); + Uuid alias = mapper.toAliasTopicId(tableId); + assertThat(alias) + .isNotIn(canonical, Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID) + .isEqualTo(new KafkaTopicMapper().toAliasTopicId(tableId)); + assertThat(mapper.isMappedTopicId(alias)).isTrue(); + assertThat(mapper.toTableId(alias)).isEqualTo(mapper.toTableId(canonical)); + } + assertThat(mapper.toAliasTopicId(124L)).isNotEqualTo(mapper.toAliasTopicId(123L)); + assertThatThrownBy(() -> mapper.toAliasTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testRejectsInvalidTopicIds() { + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static String repeat(char character, int count) { + return String.join("", Collections.nCopies(count, String.valueOf(character))); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java new file mode 100644 index 00000000000..b0dd5a0e146 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.matrix; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.utils.AppInfoParser; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Standalone client process: deliberately depends only on the JDK and the selected Kafka client. + */ +public final class KafkaClientProbe { + private KafkaClientProbe() {} + + /** Writes through a real endpoint; a failed check terminates the isolated process. */ + public static void main(String[] args) throws Exception { + String expectedVersion = args[0]; + String actualVersion = AppInfoParser.getVersion(); + if (!expectedVersion.equals(actualVersion)) { + throw new IllegalStateException( + "Expected " + expectedVersion + " but loaded " + actualVersion); + } + System.out.println("VERSION=" + actualVersion); + System.out.println( + "JAR=" + KafkaProducer.class.getProtectionDomain().getCodeSource().getLocation()); + System.out.println("JAVA=" + System.getProperty("java.version")); + System.out.println("CLASSPATH=" + System.getProperty("java.class.path")); + List> producers = new ArrayList<>(); + try { + for (String acks : Arrays.asList("0", "1", "all")) { + Map config = new HashMap<>(); + config.put("bootstrap.servers", args[1]); + config.put("key.serializer", ByteArraySerializer.class); + config.put("value.serializer", ByteArraySerializer.class); + config.put("enable.idempotence", false); + config.put("security.protocol", "PLAINTEXT"); + config.put("acks", acks); + config.put("max.in.flight.requests.per.connection", 1); + config.put("request.timeout.ms", 10000); + config.put("max.block.ms", 30000); + config.put("retries", 3); + config.put("compression.type", "none"); + config.put("linger.ms", 0); + String alias = args[3] + "_" + acks; + KafkaProducer producer = new KafkaProducer<>(config); + producers.add(producer); + for (int i = 0; i < 4; i++) { + String topic = i % 2 == 0 ? args[2] + "." + alias : alias; + RecordMetadata result = + producer.send( + new ProducerRecord<>( + topic, + 0, + 123L, + "key".getBytes(StandardCharsets.UTF_8), + (actualVersion + ":" + acks + ":" + i) + .getBytes(StandardCharsets.UTF_8), + Arrays.asList( + new RecordHeader( + "source", new byte[] {1}), + new RecordHeader( + "source", new byte[] {2})))) + .get(30, TimeUnit.SECONDS); + if (!result.topic().equals(topic) + || (!acks.equals("0") && result.offset() != i)) { + throw new IllegalStateException("Unexpected acknowledgement " + result); + } + } + producer.flush(); + System.out.println("ACKS=" + acks + ":SENT"); + } + // acks=0 only confirms a socket write. Keep connections open until the + // parent verifies native storage, rather than racing cancellation on close. + System.out.println("WRITES_COMPLETE"); + if (System.in.read() != '\n') { + throw new IllegalStateException("Native readback was not confirmed by the parent"); + } + System.out.println("NATIVE_READBACK_CONFIRMED"); + } finally { + for (KafkaProducer producer : producers) { + producer.close(); + } + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c7ff17eda3 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,501 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.AlterTableRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbAddColumn; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.json.DataTypeJsonSerde; +import org.apache.fluss.utils.json.JsonSerdeUtils; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateDatabaseRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newGetTableInfoRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newTableExistsRequest; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies Kafka DDL contracts through real coordinator RPC and persisted metadata. */ +class KafkaTableMappingITCase { + + private static final Configuration CONFIG = new Configuration(); + + static { + CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka_default"); + CONFIG.setString("kafka.database", "legacy_database"); + } + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder().setNumOfTabletServers(1).setClusterConf(CONFIG).build(); + + private CoordinatorGateway gateway; + + @BeforeAll + static void verifyConfiguredDatabaseCreatedOnStartup() throws Exception { + assertThat( + CLUSTER.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("kafka_default")) + .get() + .isExists()) + .isTrue(); + } + + @BeforeEach + void setup() { + gateway = CLUSTER.newCoordinatorClient(); + } + + @Test + void testConfiguredKafkaDatabaseAndQualifiedAccess() throws Exception { + for (String database : Arrays.asList("kafka", "legacy_database")) { + assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) + .isFalse(); + } + assertThat( + gateway.databaseExists(newDatabaseExistsRequest(DEFAULT_DATABASE)) + .get() + .isExists()) + .isTrue(); + KafkaTopicMapper mapper = new KafkaTopicMapper(CONFIG); + TablePath sales = mapper.toTablePath("sales.orders"); + TablePath warehouse = mapper.toTablePath("warehouse.orders"); + long salesId = createTable(CLUSTER, sales, enabled().build()); + long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); + assertThat(salesId).isNotEqualTo(warehouseId); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("kafka_default", "orders")); + // The cluster fixture removes custom databases between tests; recreate this table's + // database through the usual DDL helper after checking startup creation in @BeforeAll. + createTable(CLUSTER, mapper.toTablePath("orders"), enabled().build()); + assertThat( + gateway.databaseExists(newDatabaseExistsRequest("kafka_default")) + .get() + .isExists()) + .isTrue(); + assertThat(KafkaTableSchema.isEnabled(stored(mapper.toTablePath("orders")))).isTrue(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); + assertThat(info(aliasMapper.toTablePath("orders")).getTableId()) + .isEqualTo(info(sales).getTableId()); + assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); + TablePath path = TablePath.of("ddl_json", layout + "_invalid"); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + TableDescriptor descriptor = jsonTable(layout, invalid).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertUnchanged(path, before); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonComplexSchemaRoundtripAndEnable(String layout) throws Exception { + TablePath path = TablePath.of("ddl_json", layout + "_complex"); + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD("amount", DataTypes.DECIMAL(18, 2)), + DataTypes.FIELD("time", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("_kafka_key", DataTypes.BYTES())))); + TableDescriptor descriptor = + jsonTable(layout, type).property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, descriptor); + GetTableInfoResponse before = info(path); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor actual = stored(path); + assertThat(actual.getSchema()).isEqualTo(descriptor.getSchema()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + Map expected = new HashMap<>(kafkaProperties(descriptor)); + expected.put("table.kafka.enabled", "true"); + assertThat(kafkaProperties(actual)).isEqualTo(expected); + addColumn(path, "_kafka_rescue", DataTypes.STRING()); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(stored(path).getSchema().getRowType().getTypeAt(0)) + .isEqualTo(descriptor.getSchema().getRowType().getTypeAt(0)); + assertThat(KafkaTableSchema.resolve(stored(path)).rescuePosition()).isEqualTo(1); + assertThat(kafkaProperties(stored(path))).isEqualTo(expected); + } + + @Test + void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception { + TablePath path = TablePath.of("ddl_json", "alter_business_type"); + createTable(CLUSTER, path, jsonTable("flat", DataTypes.STRING()).build()); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "bad", invalid)) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + addColumn( + path, "good", DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT()))); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "ordinary", invalid); + GetTableInfoResponse disabled = info(path); + assertThat(stored(path).getSchema().getRowType().getFieldNames()).contains("ordinary"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, disabled); + } + + private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { + GetTableInfoResponse after = info(path); + assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); + assertThat(after.getSchemaId()).isEqualTo(before.getSchemaId()); + } + + private static TableDescriptor.Builder jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + TableDescriptor.Builder builder = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); + // Omitted flat layout must remain omitted through CREATE and ALTER. + if ("nested".equals(layout)) { + builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout); + } + return builder; + } + + @Test + void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { + TablePath path = TablePath.of("ddl", "roundtrip"); + Schema schema = + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column( + "payload", + DataTypes.ROW(DataTypes.FIELD("nested", DataTypes.STRING()))) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + createTable(CLUSTER, path, descriptor); + TableDescriptor stored = stored(path); + assertThat(stored.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(stored)).isEqualTo(kafkaProperties(descriptor)); + assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + .containsExactly(1); + } + + @Test + void testEnableDisableAndRejectInvalidProperties() throws Exception { + TablePath path = TablePath.of("ddl", "enable_disable"); + TableDescriptor disabled = + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, disabled); + TableDescriptor before = stored(path); + assertThat(KafkaTableSchema.isEnabled(before)).isFalse(); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor enabled = stored(path); + assertThat(KafkaTableSchema.isEnabled(enabled)).isTrue(); + Map expected = new HashMap<>(before.getProperties()); + expected.put("table.kafka.enabled", "true"); + assertThat(enabled.getProperties()).isEqualTo(expected); + assertThat(enabled.getSchema()).isEqualTo(before.getSchema()); + Map invalid = new HashMap<>(); + invalid.put("table.kafka.convert.value.type", "string"); + invalid.put("table.kafka.convert.key.type", "binary"); + assertThatThrownBy(() -> set(path, invalid)) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + assertThatThrownBy(() -> reset(path, "table.kafka.convert.value.type")) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + reset(path, "table.kafka.enabled"); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + assertThat(stored(path).getProperties()).doesNotContainKey("table.kafka.enabled"); + // Disabled tables can stage incompatible Kafka conversion properties without Kafka + // validation. + set(path, Collections.singletonMap("table.kafka.convert.value.type", "string")); + TableDescriptor staged = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(staged); + } + + @Test + void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exception { + TablePath path = TablePath.of("ddl", "schema_change"); + createTable(CLUSTER, path, enabled().build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_headers", DataTypes.STRING())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(info(path).getTableJson()).isEqualTo(before.getTableJson()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + addColumn(path, "_kafka_metadata", metadataType()); + assertThat(KafkaTableSchema.resolve(stored(path)).metadataPosition()).isEqualTo(2); + TableDescriptor beforeFailedRescue = stored(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(beforeFailedRescue); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "unmapped_business_field", DataTypes.INT()); + assertThat(stored(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped_business_field"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + + @Test + void testMetadataCanBeCreatedAndEnabledWithoutOffset() throws Exception { + Schema schema = + Schema.newBuilder() + .column("_kafka_metadata", metadataType()) + .column("_kafka_value", DataTypes.BYTES()) + .build(); + TableDescriptor descriptor = enabled().schema(schema).build(); + TablePath created = TablePath.of("ddl", "metadata_create"); + createTable(CLUSTER, created, descriptor); + assertThat(stored(created).getSchema()).isEqualTo(schema); + + TablePath altered = TablePath.of("ddl", "metadata_enable"); + createTable( + CLUSTER, + altered, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + int schemaId = info(altered).getSchemaId(); + set(altered, Collections.singletonMap("table.kafka.enabled", "true")); + assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); + assertThat(stored(altered).getSchema()).isEqualTo(schema); + assertThat(KafkaTableSchema.resolve(stored(altered)).metadataPosition()).isZero(); + } + + @Test + void testMetadataOffsetFieldRejectedOnCreateAndEnable() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl", true)).get(); + TablePath path = TablePath.of("ddl", "metadata_with_offset"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled().schema(schema).build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining( + "ROW"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + createTable( + CLUSTER, + path, + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining( + "ROW"); + assertThat(stored(path)).isEqualTo(before); + } + + @Test + void testCanonicalNameCheckedOnCreateAndEnable() throws Exception { + String database = String.join("", Collections.nCopies(128, "d")); + TablePath path = TablePath.of(database, String.join("", Collections.nCopies(128, "t"))); + gateway.createDatabase(newCreateDatabaseRequest(database, true)).get(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, enabled().build(), false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("Kafka topic"); + createTable( + CLUSTER, + path, + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(before); + } + + private static DataType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + + private GetTableInfoResponse info(TablePath path) throws Exception { + return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); + } + + private TableDescriptor stored(TablePath path) throws Exception { + return TableDescriptor.fromJsonBytes(info(path).getTableJson()); + } + + private void set(TablePath path, Map properties) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + properties, + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private void reset(TablePath path, String property) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(property), + Collections.emptyList(), + false)) + .get(); + } + + private void addColumn(TablePath path, String name, DataType type) throws Exception { + PbAddColumn column = + new PbAddColumn() + .setColumnName(name) + .setDataTypeJson( + JsonSerdeUtils.writeValueAsBytes(type, DataTypeJsonSerde.INSTANCE)) + .setColumnPositionType(0); + AlterTableRequest request = + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.emptyList(), + Collections.singletonList(column), + false); + gateway.alterTable(request).get(); + } + + private static Map kafkaProperties(TableDescriptor table) { + Map properties = new HashMap<>(); + table.getProperties() + .forEach( + (key, value) -> { + if (key.startsWith("table.kafka.")) { + properties.put(key, value); + } + }); + return properties; + } + + private static TableDescriptor.Builder enabled() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..119e8aff38f --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the Kafka adapter's use of the shared DDL contract. */ +class KafkaTopicSchemaResolverTest { + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + void testCompatibleSchemaDoesNotEnableTable() { + assertThat(resolver.isEnabled(table().build())).isFalse(); + assertThatThrownBy(() -> resolver.resolve(table().build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + } + + @Test + void testInvalidEnabledContractIsNotTreatedAsDisabled() { + TableDescriptor table = + table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "schema_latest") + .build(); + assertThat(resolver.isEnabled(table)).isTrue(); + assertThatThrownBy(() -> resolver.resolve(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("schema_latest"); + } + + @Test + void testMalformedPersistedEnableFlagIsAnExplicitContractError() { + TableDescriptor table = table().property("table.kafka.enabled", "maybe").build(); + assertThatThrownBy(() -> resolver.isEnabled(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("table.kafka.enabled"); + } + + private static TableDescriptor.Builder table() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..6d476a5a63b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,948 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies fixed-column Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) throws Exception { + // The value looks like JSON but binary/string must not interpret or rescue its members. + byte[] value = bytes("{\"_kafka_rescue\":\"消息\",\"_kafka_key\":123}"); + byte[] key = {(byte) 0xff, 0, (byte) 0xc3}; + Record record = + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + key, + value, + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("nullable", bytes("discarded")), + new RecordHeader("duplicate", bytes("last")), + new RecordHeader("nullable", null))); + for (int mask = 0; mask < 16; mask++) { + TableInfo table = envelope(string, mask); + TableDescriptor before = table.toTableDescriptor(); + KafkaTableSchema contract = KafkaTableSchema.resolve(before); + read( + transcoder.transcode(7, Collections.singletonList(record), table), + table, + 1, + row -> { + assertThat(row.getFieldCount()) + .isEqualTo(table.getRowType().getFieldCount()); + assertValue(row, contract.valueProjection().positionAt(0), string, value); + if (contract.keyPosition() >= 0) { + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + } + if (contract.headersPosition() >= 0) { + Map headers = + headers(row.getMap(contract.headersPosition())); + assertThat(headers).hasSize(2).containsKey("nullable"); + assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); + assertThat(headers.get("nullable")).isNull(); + } + if (contract.metadataPosition() >= 0) { + assertMetadata(row, contract.metadataPosition(), 123L, 1, 7); + } + if (contract.rescuePosition() >= 0) { + assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); + } + }); + assertThat(table.toTableDescriptor()).isEqualTo(before); + assertThat(before.getProperties()).doesNotContainKey("table.kafka.convert.key.type"); + assertThat(before.getCustomProperties()).isEmpty(); + } + assertThat(record.timestampType()).isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.borrowedKey()).isEqualTo(key); + assertThat(record.borrowedValue()).isEqualTo(value); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { + TableInfo table = envelope(string, 7); + List records = + Arrays.asList( + new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), + new Record(0L, new byte[0], new byte[0], Collections.emptyList())); + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(0, records, table), + table, + 2, + row -> { + int value = table.getRowType().getFieldIndex("_kafka_value"); + int key = table.getRowType().getFieldIndex("_kafka_key"); + if (next.getAndIncrement() == 0) { + assertThat(row.isNullAt(value)).isTrue(); + assertThat(row.isNullAt(key)).isTrue(); + } else { + assertValue(row, value, string, new byte[0]); + assertThat(row.getBytes(key)).isEmpty(); + } + assertThat( + row.getMap(table.getRowType().getFieldIndex("_kafka_headers")) + .size()) + .isZero(); + assertThat(row.isNullAt(table.getRowType().getFieldIndex("_kafka_rescue"))) + .isTrue(); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + null, + bytes("valid"), + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_key"); + byte[] invalidUtf8Key = {(byte) 0xff}; + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + invalidUtf8Key, + bytes("valid"), + Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getBytes(0)).isEqualTo(invalidUtf8Key)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING() + .copy(false), + DataTypes.BYTES() + .copy( + nullableValue)) + .copy(false)) + .column("_kafka_value", DataTypes.STRING()) + .build(), + true) + .build()); + List overwrittenNull = + Arrays.asList( + new RecordHeader("same", null), + new RecordHeader("", new byte[0]), + new RecordHeader("same", bytes("last"))); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("v"), overwrittenNull)), + table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2); + assertThat(map.get("same")).isEqualTo(bytes("last")); + assertThat(map.get("")).isEmpty(); + }); + Record finalNull = + new Record( + 1L, + null, + bytes("v"), + Arrays.asList( + new RecordHeader("same", bytes("first")), + new RecordHeader("same", null))); + if (nullableValue) { + read( + transcoder.transcode(0, Collections.singletonList(finalNull), table), + table, + 1, + row -> + assertThat(headers(row.getMap(0))) + .containsOnlyKeys("same") + .containsEntry("same", null)); + } else { + assertThatThrownBy( + () -> + transcoder.transcode( + 0, Collections.singletonList(finalNull), table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_headers map values"); + } + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getMap(0).size()).isZero()); + } + + @Test + void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { + TableInfo table = valueTable(false); + List records = new ArrayList<>(); + for (TimestampType type : TimestampType.values()) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, Long.MAX_VALUE}) { + records.add( + new Record( + timestamp, + type, + new byte[] {(byte) 0xff}, + bytes("v"), + Collections.singletonList(new RecordHeader("ignored", null)))); + } + } + read( + transcoder.transcode(0, records, table), + table, + records.size(), + row -> { + assertThat(row.getFieldCount()).isEqualTo(1); + assertThat(row.getString(0).toString()).isEqualTo("v"); + }); + } + + @Test + void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception { + byte[] value = new byte[32768]; + new Random(42L).nextBytes(value); + value[0] = (byte) 0xff; + TableInfo table = envelope(false, 4); + BytesView encoded = + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, value, Collections.emptyList())), + table); + assertThat(encoded.getBytesLength()).isGreaterThan(4096); + read( + encoded, + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(1)).isEqualTo(value); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testValueNotNullConstraint(boolean string) { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + valueType(string).copy(false)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field '_kafka_value'"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataPreservesInputTimestampBatchTypeAndRequestedPartition(boolean string) + throws Exception { + long[] timestamps = {Record.NO_TIMESTAMP, 0L, 123L, 1728912345678L, Long.MAX_VALUE}; + TimestampType[] types = { + TimestampType.UNKNOWN, TimestampType.CREATE_TIME, TimestampType.LOG_APPEND_TIME + }; + int[] typeIds = {-1, 0, 1}; + List records = new ArrayList<>(); + for (TimestampType type : types) { + for (long timestamp : timestamps) { + records.add(new Record(timestamp, type, null, bytes("v"), Collections.emptyList())); + } + } + for (boolean nullable : new boolean[] {false, true}) { + TableInfo table = metadataTable(string, nullable, true); + TableDescriptor before = table.toTableDescriptor(); + // Reuse the same converter and records across requests without retaining partition + // state. + for (int partition : new int[] {7, 0, 3}) { + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(partition, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertMetadata( + row, + 0, + timestamps[i % timestamps.length], + typeIds[i / timestamps.length], + partition); + assertValue(row, 1, string, bytes("v")); + }); + } + assertThat(table.toTableDescriptor()).isEqualTo(before); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataTimestampNotNullFailureHasNoPartialOutputAndRemainsUsable(boolean string) + throws Exception { + Record valid = + new Record( + 0L, TimestampType.CREATE_TIME, null, bytes("v"), Collections.emptyList()); + for (boolean nullable : new boolean[] {false, true}) { + TableInfo table = metadataTable(string, nullable, false); + for (TimestampType type : TimestampType.values()) { + Record invalid = + new Record( + Record.NO_TIMESTAMP, + type, + null, + bytes("v"), + Collections.emptyList()); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 7, Arrays.asList(valid, invalid), table))); + assertThat(failure) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_metadata.timestamp"); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(3, Collections.singletonList(valid), table), + table, + 1, + row -> { + assertMetadata(row, 0, 0L, 0, 3); + assertValue(row, 1, string, bytes("v")); + }); + } + } + } + + @Test + void testLegacyRecordDoesNotInferBatchTimestampType() throws Exception { + TableInfo table = metadataTable(true, false, false); + read( + transcoder.transcode( + 7, + Collections.singletonList( + new Record(123L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertMetadata(row, 0, 123L, -1, 7)); + } + + @Test + void testRejectsNegativePartitionId() { + assertThatThrownBy( + () -> + transcoder.transcode( + -1, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + null, + bytes("v"), + Collections.emptyList())), + envelope(false, 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Kafka partition ID must be non-negative"); + } + + @Test + void testRejectsLegacyMetadataOffsetField() { + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor(schema, true).build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining( + "_kafka_metadata must be ROW"); + } + + @Test + void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { + List records = + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())); + Schema fixed = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()).build(); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + records, + table( + descriptor(fixed, true) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + records, + table( + descriptor( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING()) + .build(), + true) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + records, + table( + descriptor(fixed, true) + .customProperty( + "kafka.value.format", "string") + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("custom property"); + assertThatThrownBy( + () -> transcoder.transcode(0, Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void testJsonIsNotActivatedEvenForNullValues() { + TableInfo table = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build()); + for (byte[] value : new byte[][] {null, bytes("{}")}) { + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + null, + value, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("PARSE_JSON"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) + throws Exception { + TableInfo table = envelope(string, 15); + List records = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + records.add( + new Record( + i % 3 == 0 ? Record.NO_TIMESTAMP : i, + i % 3 == 0 + ? TimestampType.UNKNOWN + : (i % 3 == 1 + ? TimestampType.CREATE_TIME + : TimestampType.LOG_APPEND_TIME), + bytes("key-" + i), + bytes("消息-" + i), + Arrays.asList( + new RecordHeader("duplicate", null), + new RecordHeader("duplicate", bytes("header-" + i))))); + } + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(7, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertValue(row, 2, string, bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(1); + assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); + assertThat(row.isNullAt(1)).isTrue(); + assertMetadata(row, 4, i % 3 == 0 ? Record.NO_TIMESTAMP : i, i % 3 - 1, 7); + }); + } + + @Test + void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception { + TableInfo table = envelope(false, 2); + List input = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + input.add(new RecordHeader("header-" + i, bytes("value-" + i))); + } + read( + transcoder.transcode( + 0, + Collections.singletonList(new Record(1L, null, bytes("v"), input)), + table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2050); + for (int i = 0; i < 2050; i++) { + assertThat(map.get("header-" + i)).isEqualTo(bytes("value-" + i)); + } + }); + } + + @Test + void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { + TableInfo table = envelope(false, 7); + byte[] key = new byte[1]; + byte[] value = new byte[1]; + byte[] headerValue = new byte[1]; + GenericRow row = + GenericRow.of( + GenericMap.of(BinaryString.fromString("header"), headerValue), + null, + value, + key); + BytesView encoded = + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + for (int i = 0; i < 3; i++) { + key[0] = (byte) i; + value[0] = (byte) (i + 10); + headerValue[0] = (byte) (i + 20); + consumer.append(row); + } + Arrays.fill(key, (byte) -1); + Arrays.fill(value, (byte) -1); + Arrays.fill(headerValue, (byte) -1); + row.setField(2, null); + }, + table); + AtomicInteger next = new AtomicInteger(); + read( + encoded, + table, + 3, + actual -> { + int i = next.getAndIncrement(); + assertThat(actual.getBytes(3)).containsExactly((byte) i); + assertThat(actual.getBytes(2)).containsExactly((byte) (i + 10)); + assertThat(headers(actual.getMap(0)).get("header")) + .containsExactly((byte) (i + 20)); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable(boolean metadata) + throws Exception { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", DataTypes.STRING().copy(false)); + if (metadata) { + schema.column("_kafka_metadata", metadataType(true)); + } + TableInfo table = table(descriptor(schema.build(), true).build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + Record valid = + new Record( + Record.NO_TIMESTAMP, + TimestampType.LOG_APPEND_TIME, + null, + bytes("valid"), + Collections.emptyList()); + for (byte[] invalid : new byte[][] {null, {(byte) 0xc3, 0x28}, {(byte) 0xff}}) { + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 7, + Arrays.asList( + valid, + new Record( + 2L, + null, + invalid, + Collections.emptyList())), + plan))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + } + read( + transcoder.transcode(3, Collections.singletonList(valid), plan), + table, + 1, + row -> { + assertThat(row.getString(0).toString()).isEqualTo("valid"); + if (metadata) { + assertMetadata(row, 1, Record.NO_TIMESTAMP, 1, 3); + } + }); + } + + @Test + void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES().copy(false))) + .build(), + true) + .build()); + Record valid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", bytes("v")))); + Record invalid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", null))); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 0, Arrays.asList(valid, invalid), table))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(0, Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + @Test + void testStreamingSourceFailureClosesAllocator() { + IOException expected = new IOException("source failed after the first row"); + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + consumer.append( + GenericRow.of( + BinaryString.fromString( + "valid"))); + throw expected; + }, + valueTable(false))); + assertThat(failure).isSameAs(expected); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING().copy(!notNull)) + .build(), + true) + .build()); + } + + private static TableInfo envelope(boolean string, int mask) { + Schema.Builder schema = Schema.newBuilder(); + if ((mask & 2) != 0) { + schema.column("_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_value", valueType(string)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType(true)); + } + return table(descriptor(schema.build(), string).build()); + } + + private static DataType metadataType(boolean nullableTimestamp) { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + } + + private static TableInfo metadataTable( + boolean string, boolean nullableRow, boolean nullableTimestamp) { + return table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_metadata", + metadataType(nullableTimestamp).copy(nullableRow)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + } + + private static DataType valueType(boolean string) { + return string ? DataTypes.STRING() : DataTypes.BYTES(); + } + + private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(8) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, string ? "string" : "binary"); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("fluss", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static Map headers(InternalMap map) { + Map result = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + assertThat(map.keyArray().isNullAt(i)).isFalse(); + String key = map.keyArray().getString(i).toString(); + assertThat(result).doesNotContainKey(key); + result.put(key, map.valueArray().isNullAt(i) ? null : map.valueArray().getBytes(i)); + } + return result; + } + + private static void assertValue(InternalRow row, int position, boolean string, byte[] value) { + if (string) { + assertThat(row.getString(position).toString()) + .isEqualTo(new String(value, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(position)).isEqualTo(value); + } + } + + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + if (timestamp == Record.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.isNullAt(2)).isFalse(); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java new file mode 100644 index 00000000000..ac950afeeeb --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaCompiledWritePlanCacheTest { + + @Test + void testReusesSameIdentityAndSeparatesEquivalentAndChangedInstances() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo original = tableInfo("orders", 1L, 1, "v1"); + + KafkaTopicWritePlan first = + cache.getOrCompile(original, () -> compile(original, compileCount)); + KafkaTopicWritePlan reused = + cache.getOrCompile(original, () -> compile(original, compileCount)); + assertThat(reused).isSameAs(first); + assertThat(cache.getIfPresent(original)).isSameAs(first); + + // The authoritative TableInfo cache owns canonical instances. A separately constructed, + // structurally equivalent object is conservatively treated as a new metadata snapshot. + TableInfo equivalent = tableInfo("orders", 1L, 1, "v1"); + assertThat(cache.getIfPresent(equivalent)).isNull(); + KafkaTopicWritePlan equivalentPlan = + cache.getOrCompile(equivalent, () -> compile(equivalent, compileCount)); + assertThat(equivalentPlan).isNotSameAs(first); + + TableInfo propertyChange = tableInfo("orders", 1L, 1, "v2"); + KafkaTopicWritePlan propertyPlan = + cache.getOrCompile(propertyChange, () -> compile(propertyChange, compileCount)); + TableInfo schemaChange = tableInfo("orders", 1L, 2, "v2"); + KafkaTopicWritePlan schemaPlan = + cache.getOrCompile(schemaChange, () -> compile(schemaChange, compileCount)); + TableInfo recreated = tableInfo("orders", 2L, 2, "v2"); + KafkaTopicWritePlan recreatedPlan = + cache.getOrCompile(recreated, () -> compile(recreated, compileCount)); + + assertThat(propertyPlan).isNotSameAs(first); + assertThat(schemaPlan).isNotSameAs(propertyPlan); + assertThat(recreatedPlan).isNotSameAs(schemaPlan); + assertThat(compileCount).hasValue(5); + } + + @Test + void testEquivalentInstanceCannotInvalidateCurrentIdentity() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo current = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan currentPlan = + cache.getOrCompile(current, () -> compile(current, compileCount)); + + cache.invalidate(tableInfo("orders", 1L, 1, "v1")); + + assertThat(cache.getOrCompile(current, () -> compile(current, compileCount))) + .isSameAs(currentPlan); + assertThat(compileCount).hasValue(1); + } + + @Test + void testInvalidationAndCompileFailureRecovery() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan first = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + cache.invalidate(tableInfo); + KafkaTopicWritePlan second = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + assertThat(second).isNotSameAs(first); + + TableInfo failingTable = tableInfo("payments", 2L, 1, "v1"); + assertThatThrownBy( + () -> + cache.getOrCompile( + failingTable, + () -> { + compileCount.incrementAndGet(); + throw new IllegalStateException("compile failed"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("compile failed"); + assertThat(cache.getOrCompile(failingTable, () -> compile(failingTable, compileCount))) + .isNotNull(); + assertThat(compileCount).hasValue(4); + } + + @Test + void testConcurrentColdCompileIsSingleFlight() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + }))); + } + compilerStarted.await(); + releaseCompiler.countDown(); + + KafkaTopicWritePlan first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(compileCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightCompileToResurrect() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future compiling = + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + })); + compilerStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(tableInfo)); + releaseCompiler.countDown(); + + KafkaTopicWritePlan planBeforeInvalidation = compiling.get(); + invalidating.get(); + KafkaTopicWritePlan planAfterInvalidation = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + assertThat(planAfterInvalidation).isNotSameAs(planBeforeInvalidation); + assertThat(compileCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCompiledAssemblerIsSafeToShare() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan plan = cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + final int id = i; + futures.add( + executor.submit( + () -> + plan.rowAssembler() + .assemble( + new Object[] { + Integer.toString(id) + .getBytes( + StandardCharsets.UTF_8) + }, + new Record( + id, null, null, new ArrayList<>()), + 0) + .getField(0))); + } + for (int i = 0; i < futures.size(); i++) { + assertThat((byte[]) futures.get(i).get()) + .isEqualTo(Integer.toString(i).getBytes(StandardCharsets.UTF_8)); + } + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(2); + for (int i = 0; i < 10; i++) { + TableInfo tableInfo = tableInfo("topic_" + i, i + 1L, 1, "v1"); + cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger compileCount) { + compileCount.incrementAndGet(); + return compile(tableInfo); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo) { + KafkaTableSchema schema = + new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); + return new KafkaTopicWritePlan(tableInfo, schema); + } + + private static TableInfo tableInfo( + String tableName, long tableId, int schemaId, String propertyValue) { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + propertyValue.equals("v1") ? "binary" : "BINARY") + .build(); + return TableInfo.of( + TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test compiler.", e); + } + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index df2256985c2..37705f2e075 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -80,6 +80,7 @@ public final class NettyServer implements RpcServer { private EventLoopGroup selectorGroup; private volatile boolean isRunning; + private CompletableFuture closeFuture; public NettyServer( Configuration conf, @@ -104,7 +105,8 @@ public NettyServer( } @Override - public void start() throws IOException { + public synchronized void start() throws IOException { + checkState(closeFuture == null, "Netty server has already been closed."); checkState(bindChannels.isEmpty(), "Netty server has already been initialized."); int numNetworkThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_NETWORK_THREADS); int numWorkerThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS); @@ -298,30 +300,40 @@ public ScheduledExecutorService getScheduledExecutor() { @Override public CompletableFuture closeAsync() { - if (!isRunning) { - return CompletableFuture.completedFuture(null); - } - - isRunning = false; - - CompletableFuture acceptorShutdownFuture = shutdownGroup(acceptorGroup); - CompletableFuture selectorShutdownFuture = shutdownGroup(selectorGroup); - CompletableFuture channelShutdownFuture = - FutureUtils.completeAll( - bindChannels.stream() - .map(NettyUtils::shutdownChannel) - .collect(Collectors.toList())); - CompletableFuture workerShutdownFuture; - if (workerPool != null) { - workerShutdownFuture = workerPool.closeAsync(); - } else { - workerShutdownFuture = CompletableFuture.completedFuture(null); + synchronized (this) { + if (closeFuture != null) { + return closeFuture; + } + isRunning = false; + + CompletableFuture transportShutdownFuture = + CompletableFuture.allOf( + shutdownGroup(acceptorGroup), + shutdownGroup(selectorGroup), + FutureUtils.completeAll( + bindChannels.stream() + .map(NettyUtils::shutdownChannel) + .collect(Collectors.toList())), + workerPool.closeAsync()); + closeFuture = closeProtocolsAfter(transportShutdownFuture, protocols); + return closeFuture; } + } - return CompletableFuture.allOf( - acceptorShutdownFuture, - selectorShutdownFuture, - channelShutdownFuture, - workerShutdownFuture); + static CompletableFuture closeProtocolsAfter( + CompletableFuture shutdownFuture, List protocols) { + return FutureUtils.composeAfterwards( + shutdownFuture, + () -> { + List> closeFutures = new ArrayList<>(); + for (NetworkProtocolPlugin protocol : protocols) { + try { + closeFutures.add(protocol.closeAsync()); + } catch (Throwable failure) { + closeFutures.add(FutureUtils.completedExceptionally(failure)); + } + } + return FutureUtils.completeAll(closeFutures); + }); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java index f60202228ef..c8921473fad 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java @@ -25,6 +25,7 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; /** A network protocol plugin that provides the server side implementation of a network protocol. */ public interface NetworkProtocolPlugin extends Plugin { @@ -52,4 +53,9 @@ public interface NetworkProtocolPlugin extends Plugin { * protocol. */ RequestHandler createRequestHandler(RpcGatewayService service); + + /** Closes protocol-owned resources after transport and request processors have stopped. */ + default CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java index c16f996d713..72032ecd5ef 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java @@ -25,18 +25,25 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.util.NOPMetricsGroup; +import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.RpcServer; import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.NettyClient; import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import org.apache.fluss.utils.NetUtils; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.apache.fluss.utils.NetUtils.getAvailablePort; import static org.assertj.core.api.Assertions.assertThat; @@ -45,6 +52,24 @@ /** Test for {@link NettyServer}. */ public class NettyServerTest { + @Test + void testCannotStartAfterClose() { + MetricGroup metricGroup = NOPMetricsGroup.newInstance(); + NettyServer server = + new NettyServer( + new Configuration(), + Endpoint.fromListenersString("FLUSS://localhost:0"), + new TestingGatewayService(), + metricGroup, + RequestsMetrics.createCoordinatorServerRequestMetrics(metricGroup)); + + server.closeAsync().join(); + + assertThatThrownBy(server::start) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already been closed"); + } + @Test void testPortAsZero() throws Exception { List endpoints = Endpoint.fromListenersString("FLUSS://localhost:0"); @@ -111,4 +136,94 @@ void testOversizedRequestRejected() throws Exception { } } } + + @Test + void testProtocolsCloseOnlyAfterTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + NetworkProtocolPlugin protocol = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Collections.singletonList(protocol)); + assertThat(closeCalls).hasValue(0); + assertThat(result).isNotDone(); + + transportShutdown.complete(null); + result.join(); + assertThat(closeCalls).hasValue(1); + } + + @Test + void testAllProtocolsCloseAfterExceptionalTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + RuntimeException asyncFailure = new RuntimeException("async protocol close failure"); + RuntimeException synchronousFailure = + new RuntimeException("synchronous protocol close failure"); + NetworkProtocolPlugin asyncFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(asyncFailure); + return failed; + }); + NetworkProtocolPlugin synchronouslyFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + throw synchronousFailure; + }); + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Arrays.asList(asyncFailing, synchronouslyFailing)); + + transportShutdown.completeExceptionally(new RuntimeException("transport failure")); + + assertThatThrownBy(result::join) + .hasStackTraceContaining("transport failure") + .hasStackTraceContaining("async protocol close failure") + .hasStackTraceContaining("synchronous protocol close failure"); + assertThat(closeCalls).hasValue(2); + } + + private static NetworkProtocolPlugin testingProtocol( + Supplier> closeAction) { + return new NetworkProtocolPlugin() { + @Override + public String name() { + return "testing"; + } + + @Override + public void setup(Configuration conf) {} + + @Override + public List listenerNames() { + return Collections.emptyList(); + } + + @Override + public ChannelHandler createChannelHandler( + RequestChannel[] requestChannels, String listenerName) { + return null; + } + + @Override + public RequestHandler createRequestHandler(RpcGatewayService service) { + return null; + } + + @Override + public CompletableFuture closeAsync() { + return closeAction.get(); + } + }; + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 2cdd2c5bf97..3b93e2c8c50 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -548,17 +548,22 @@ private interface ThrowingRunnable { private void createDefaultDatabase() { MetadataManager metadataManager = new MetadataManager(zkClient, conf, lakeCatalogDynamicLoader); + createDefaultDatabases(metadataManager, conf); + } + + @VisibleForTesting + static void createDefaultDatabases(MetadataManager metadataManager, Configuration conf) { List databases = metadataManager.listDatabases(); if (databases.isEmpty()) { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } - // create Kafka default database if Kafka is enabled. if (conf.get(ConfigOptions.KAFKA_ENABLED)) { - String kafkaDB = conf.get(ConfigOptions.KAFKA_DATABASE); - if (!databases.contains(kafkaDB)) { - metadataManager.createDatabase(kafkaDB, DatabaseDescriptor.EMPTY, true); - LOG.info("Created default database '{}' for Kafka protocol.", kafkaDB); + String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + // The native default database follows the cluster initialization logic above. + if (!DEFAULT_DATABASE.equals(kafkaDatabase) && !databases.contains(kafkaDatabase)) { + metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); + LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 6bed0ecd3f0..df8713338a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 14b97c927eb..41259eed034 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -39,6 +39,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.DatabaseInfo; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LakeTableUtil; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -131,6 +132,12 @@ public void validateTableDescriptor(TableDescriptor tableDescriptor) { lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); } + /** Validates a descriptor and its Kafka topic identity before persistence. */ + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { + validateTableDescriptor(tableDescriptor); + KafkaTableSchema.validateTablePath(tablePath, tableDescriptor); + } + public void createDatabase( String databaseName, DatabaseDescriptor databaseDescriptor, boolean ignoreIfExists) throws DatabaseAlreadyExistException { @@ -852,7 +859,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index de40fbf01a8..fafbbf9a6c1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; @@ -122,6 +123,8 @@ public static void validateTableDescriptor( } } + KafkaTableSchema.validate(tableDescriptor); + // check distribution checkDistribution(tableDescriptor, maxBucketNum); checkPrimaryKey(tableDescriptor); @@ -146,6 +149,10 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { + TableDescriptor updated = + TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); + KafkaTableSchema.validate(updated); + KafkaTableSchema.validateTablePath(table.getTablePath(), updated); checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java new file mode 100644 index 00000000000..5e4cc8bc3a5 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode; +import org.apache.fluss.testutils.common.AllCallbackWrapper; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests startup database creation for native and Kafka default database configurations. */ +class KafkaDefaultDatabaseTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private MetadataManager manager; + + @BeforeEach + void setup() throws Exception { + ZooKeeperClient zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + if (zk.getStat(DatabasesZNode.path()).isPresent()) { + zk.getCuratorClient() + .delete() + .deletingChildrenIfNeeded() + .forPath(DatabasesZNode.path()); + } + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNativeDefaultFollowsClusterInitialization(boolean hasDatabase) { + if (hasDatabase) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + } + Configuration config = kafkaConfiguration(DEFAULT_DATABASE); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()) + .containsExactly(hasDatabase ? "existing" : DEFAULT_DATABASE); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreateMissingCustomDefaultAndPreserveExisting(boolean exists) { + String database = "kafka_default"; + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + DatabaseDescriptor descriptor = + DatabaseDescriptor.builder().comment("existing default").build(); + if (exists) { + manager.createDatabase(database, descriptor, false); + } + Configuration config = kafkaConfiguration(database); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactlyInAnyOrder("existing", database); + assertThat(manager.getDatabase(database).getDatabaseDescriptor().toJsonBytes()) + .isEqualTo((exists ? descriptor : DatabaseDescriptor.EMPTY).toJsonBytes()); + } + + @Test + void testDisabledKafkaDoesNotCreateConfiguredDatabase() { + Configuration config = kafkaConfiguration("kafka_default"); + config.set(ConfigOptions.KAFKA_ENABLED, false); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactly(DEFAULT_DATABASE); + } + + private static Configuration kafkaConfiguration(String database) { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + return config; + } +} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index ce76beca9a1..b183783fd27 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,7 +254,57 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.database | String | kafka | The database for Fluss Kafka. The default database is `kafka`. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, a configured database other than `fluss` is created if missing. The native `fluss` database is created only when the cluster has no databases. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. + +Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named +`sales.orders` maps to that table directly; `orders` maps to the same-named table in +`kafka.default.database` (by default, `fluss.orders`). The native `fluss` database is created only +when the cluster has no databases; Kafka reuses this database when its default is `fluss`. +If Kafka is enabled and a different default database is configured, the coordinator creates it +if missing. Existing databases are preserved; other databases and all tables must be created +explicitly. Reverse mapping always uses +the fully qualified name. All TabletServers must use the same default database. Changing it +redirects unqualified names, so migrate clients to fully qualified names before changing the +configuration. The prototype `kafka.database` option is no longer used. + +### Kafka table properties + +| Option | Type | Default | Description | +|---|---|---|---| +| table.kafka.enabled | Boolean | false | Explicitly expose a non-partitioned Arrow log table as a Kafka topic. CREATE and ALTER validate the contract before persisting it. | +| table.kafka.convert.value.type | String | (None) | Required when enabled: `binary`, `string`, or `parse_json`. | +| table.kafka.convert.value.layout | String | (None) | JSON uses `flat` when omitted; `nested` targets `_kafka_value ROW<...>`. Setting this property for binary/string is invalid. | +| table.kafka.convert.key.type | String | (None) | Only `binary` is supported, and requires `_kafka_key BYTES`. Declaring that column implies binary without adding a property. | + +Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` +column respectively. Flat JSON targets business columns at the table level; nested JSON targets +the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +JSON business schemas support nested ROW, ARRAY and MAP values. Every MAP key must be STRING, +including maps inside other containers. CREATE and ALTER reject unsupported types or a nesting +depth above 64, counted from each top-level JSON business field at depth zero; ROW fields, +ARRAY elements and MAP values each add one level. The JSON root and the physical nested +`_kafka_value ROW` container add no level, so equivalent flat/nested schemas have the same limit. +Scalar value and nullability checks apply when decoding records. +Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. +Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and +collects unknown or reserved JSON members; it requires no mapping property. Binary/string +conversion writes null to a declared rescue column. + +Enabling Kafka compatibility does not add columns or persist default Kafka properties. When +disabled, ordinary Fluss tables are not subject to Kafka-specific schema validation. Prototype +`kafka.key.*`, `kafka.value.*`, `kafka.metadata.*`, and `fluss.*` conversion properties are not +accepted on enabled tables. + +The `_kafka_metadata` contract is +`ROW`. +This optional physical column stores the input Kafka record timestamp, the batch timestamp type +(`-1` for no timestamp, `0` for CreateTime, `1` for LogAppendTime), and the requested partition. +Without a timestamp, the timestamp field is null; declared NOT NULL constraints still apply +when converting records. These values are available before Arrow encoding. Log offsets remain +native log metadata and are not part of this ROW, so enabling the column requires no append-time +offset backfill. CREATE and ALTER reject extra fields, including the former offset field, and +incorrect field names, ordering or types. +These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support.